@sensigo/realm 0.31.1 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/apply-resume.d.ts +34 -0
- package/dist/engine/apply-resume.d.ts.map +1 -0
- package/dist/engine/apply-resume.js +69 -0
- package/dist/engine/apply-resume.js.map +1 -0
- package/dist/engine/comparison-expr.d.ts.map +1 -1
- package/dist/engine/comparison-expr.js.map +1 -1
- package/dist/engine/defaulted-steps.d.ts +17 -0
- package/dist/engine/defaulted-steps.d.ts.map +1 -0
- package/dist/engine/defaulted-steps.js +26 -0
- package/dist/engine/defaulted-steps.js.map +1 -0
- package/dist/engine/execution-loop.d.ts +32 -0
- package/dist/engine/execution-loop.d.ts.map +1 -1
- package/dist/engine/execution-loop.js +704 -39
- package/dist/engine/execution-loop.js.map +1 -1
- package/dist/engine/lifecycle.d.ts +14 -0
- package/dist/engine/lifecycle.d.ts.map +1 -1
- package/dist/engine/lifecycle.js +14 -0
- package/dist/engine/lifecycle.js.map +1 -1
- package/dist/engine/run-health.d.ts +1 -1
- package/dist/engine/run-health.d.ts.map +1 -1
- package/dist/engine/run-health.js +26 -3
- package/dist/engine/run-health.js.map +1 -1
- package/dist/engine/settlement.d.ts +27 -0
- package/dist/engine/settlement.d.ts.map +1 -0
- package/dist/engine/settlement.js +414 -0
- package/dist/engine/settlement.js.map +1 -0
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -2
- package/dist/index.js.map +1 -1
- package/dist/store/json-file-store.d.ts +54 -3
- package/dist/store/json-file-store.d.ts.map +1 -1
- package/dist/store/json-file-store.js +156 -17
- package/dist/store/json-file-store.js.map +1 -1
- package/dist/store/store-interface.d.ts +47 -1
- package/dist/store/store-interface.d.ts.map +1 -1
- package/dist/types/run-record.d.ts +67 -0
- package/dist/types/run-record.d.ts.map +1 -1
- package/dist/types/settlement.d.ts +124 -0
- package/dist/types/settlement.d.ts.map +1 -0
- package/dist/types/settlement.js +2 -0
- package/dist/types/settlement.js.map +1 -0
- package/dist/types/workflow-definition.d.ts.map +1 -1
- package/dist/types/workflow-definition.js.map +1 -1
- package/dist/types/workflow-error.d.ts +1 -1
- package/dist/types/workflow-error.d.ts.map +1 -1
- package/dist/types/workflow-error.js.map +1 -1
- package/package.json +2 -2
|
@@ -3,6 +3,8 @@ import { WorkflowError } from '../types/workflow-error.js';
|
|
|
3
3
|
import { persistsField } from '../store/store-fidelity.js';
|
|
4
4
|
import { storeDeclaresSeal, storeDeclaresNonceCarriage } from '../store/trace-buffer-store.js';
|
|
5
5
|
import { partitionBufferedEntries } from './trace-adoption.js';
|
|
6
|
+
import { deriveDefaultedSteps } from './defaulted-steps.js';
|
|
7
|
+
import { selectFinalizers } from './settlement.js';
|
|
6
8
|
import { captureEvidence } from '../evidence/snapshot.js';
|
|
7
9
|
import { validateInputSchema, validateOutputSchema, validateTraceSchema, } from '../validation/input-schema.js';
|
|
8
10
|
import { normalizeTrace } from './trace-normalizer.js';
|
|
@@ -376,11 +378,11 @@ function mergeWarnings(traceWarnings, ...extraWarnings) {
|
|
|
376
378
|
return [...traceWarnings, ...defined];
|
|
377
379
|
}
|
|
378
380
|
/**
|
|
379
|
-
* Pure helper (issue #220 PR-2, D6):
|
|
380
|
-
*
|
|
381
|
-
* `defaulted_steps` set to their distinct step names
|
|
382
|
-
*
|
|
383
|
-
*
|
|
381
|
+
* Pure helper (issue #220 PR-2, D6): if `sealDraft`'s evidence has any entries default-settled
|
|
382
|
+
* (per {@link deriveDefaultedSteps}, issue #232 — the SHARED derivation the read surfaces also
|
|
383
|
+
* use), returns the record with `defaulted_steps` set to their distinct step names — else returns
|
|
384
|
+
* the SAME record reference unchanged, so a run with no default-settle anywhere is byte-identical
|
|
385
|
+
* to pre-PR-2 behavior (the damage-rail this preserves). Pure, no I/O.
|
|
384
386
|
*
|
|
385
387
|
* Applied by WRAPPING the `buildFinalizedSeal` call inside the TERMINAL branch of each seal
|
|
386
388
|
* ternary — `buildFinalizedSeal` itself stays byte-untouched (a chokepoint insertion was
|
|
@@ -388,17 +390,13 @@ function mergeWarnings(traceWarnings, ...extraWarnings) {
|
|
|
388
390
|
* fragile two-touch above the damage-rail fast-path). Callers must pass ONLY a sealed record from
|
|
389
391
|
* the `'complete'` branch — never a non-terminal draft, and never a fail/abort seal — so
|
|
390
392
|
* `defaulted_steps` never leaks onto a persisted non-terminal record that a later FAIL seal
|
|
391
|
-
* inherits (the FM-5 residual this guards).
|
|
393
|
+
* inherits (the FM-5 residual this guards). issue #232 note: this complete-only stamping is
|
|
394
|
+
* UNCHANGED — a run that fails/aborts still carries no persisted `defaulted_steps`; the failure-
|
|
395
|
+
* path disclosure gap is closed by the READ surfaces calling `deriveDefaultedSteps` directly, not
|
|
396
|
+
* by widening what gets persisted here.
|
|
392
397
|
*/
|
|
393
398
|
function stampDefaultedSteps(sealDraft) {
|
|
394
|
-
const steps =
|
|
395
|
-
const seen = new Set();
|
|
396
|
-
for (const snap of sealDraft.evidence) {
|
|
397
|
-
if (snap.diagnostics?.settled_by_default === true && !seen.has(snap.step_id)) {
|
|
398
|
-
seen.add(snap.step_id);
|
|
399
|
-
steps.push(snap.step_id);
|
|
400
|
-
}
|
|
401
|
-
}
|
|
399
|
+
const steps = deriveDefaultedSteps(sealDraft.evidence);
|
|
402
400
|
if (steps.length === 0)
|
|
403
401
|
return sealDraft;
|
|
404
402
|
return { ...sealDraft, defaulted_steps: steps };
|
|
@@ -595,6 +593,155 @@ function makeErrorEnvelope(options, run, err, definition, extraWarnings) {
|
|
|
595
593
|
}
|
|
596
594
|
return baseWithWarnings;
|
|
597
595
|
}
|
|
596
|
+
/**
|
|
597
|
+
* Design record §6: a claim-time refusal envelope's advisory line when the fresh run carries
|
|
598
|
+
* finalizer_ledger pendings — points the caller at the recovery verb. `undefined` when there is
|
|
599
|
+
* nothing pending (the common case — never emits an empty/placeholder advisory).
|
|
600
|
+
*/
|
|
601
|
+
function finalizerDrainAdvisory(run) {
|
|
602
|
+
const pendingCount = Object.values(run.finalizer_ledger ?? {}).filter((e) => e.status === 'pending').length;
|
|
603
|
+
if (pendingCount === 0)
|
|
604
|
+
return undefined;
|
|
605
|
+
return `${pendingCount} finalizer(s) not yet delivered — realm run drain ${run.id}`;
|
|
606
|
+
}
|
|
607
|
+
/**
|
|
608
|
+
* Re-arm disclosure (final-gate F10b, design record §6) — the settling caller's own comparison,
|
|
609
|
+
* NEVER inside the frozen `applySettlement` transform: any finalizer that was `'voided'` (an
|
|
610
|
+
* operator `--void`) in `before.finalizer_ledger` and is `'pending'` again in
|
|
611
|
+
* `after.finalizer_ledger` was just RE-ARMED by mintFresh on THIS terminal edge (a later
|
|
612
|
+
* fail-then-complete-differently — or any outcome that newly selects it — re-mints a clean pending
|
|
613
|
+
* entry per §4's mint rule; mintFresh has no memory of a prior void, by design). Called only when
|
|
614
|
+
* `result.transitioned === true` (mintFresh only ever runs on the terminal false→true edge).
|
|
615
|
+
*/
|
|
616
|
+
function computeReArmWarnings(before, after) {
|
|
617
|
+
const warnings = [];
|
|
618
|
+
for (const [name, afterEntry] of Object.entries(after ?? {})) {
|
|
619
|
+
const beforeEntry = before?.[name];
|
|
620
|
+
if (beforeEntry?.status === 'voided' && afterEntry.status === 'pending') {
|
|
621
|
+
warnings.push(`finalizer '${name}' was operator-voided; re-armed by this terminal edge`);
|
|
622
|
+
}
|
|
623
|
+
}
|
|
624
|
+
return warnings;
|
|
625
|
+
}
|
|
626
|
+
/**
|
|
627
|
+
* Builds the ResponseEnvelope for a `settle_step` REFUSAL (design record §7's result/code table) —
|
|
628
|
+
* shared by all three migrated seal sites (issue #279, increment 1, PR-B). `allEvidence` is
|
|
629
|
+
* attached ONLY for `claim_lost`: the dispatch DID run and produce evidence; it just was not
|
|
630
|
+
* recorded, so the caller should still see what happened. The four other reasons `settle_step` can
|
|
631
|
+
* actually return are enumerated explicitly; the remaining nine `SettlementRefusalReason` members
|
|
632
|
+
* are lease/mark-only and structurally unreachable here (a `default` throws rather than silently
|
|
633
|
+
* mis-rendering one).
|
|
634
|
+
*/
|
|
635
|
+
function buildSettlementRefusalEnvelope(options, definition, result, allEvidence, traceWarnings) {
|
|
636
|
+
const extraWarnings = traceWarnings.length > 0 ? traceWarnings : undefined;
|
|
637
|
+
switch (result.reason) {
|
|
638
|
+
case 'already_settled_by_other':
|
|
639
|
+
case 'settled_outcome_divergence': {
|
|
640
|
+
const persisted = result.run.settled?.[options.command]?.outcome;
|
|
641
|
+
const err = new WorkflowError(`Step '${options.command}' was already settled` +
|
|
642
|
+
(persisted !== undefined ? ` with outcome '${persisted}'` : '') +
|
|
643
|
+
` by a different attempt.`, {
|
|
644
|
+
code: 'STATE_STEP_ALREADY_SETTLED',
|
|
645
|
+
category: 'STATE',
|
|
646
|
+
agentAction: 'resolve_precondition',
|
|
647
|
+
retryable: false,
|
|
648
|
+
details: {
|
|
649
|
+
runId: options.runId,
|
|
650
|
+
step: options.command,
|
|
651
|
+
reason: result.reason,
|
|
652
|
+
...(persisted !== undefined ? { persisted_outcome: persisted } : {}),
|
|
653
|
+
},
|
|
654
|
+
});
|
|
655
|
+
return makeErrorEnvelope(options, result.run, err, definition, extraWarnings);
|
|
656
|
+
}
|
|
657
|
+
case 'claim_lost': {
|
|
658
|
+
const err = new WorkflowError(`Step '${options.command}': this attempt's outcome was NOT recorded — the claim was lost ` +
|
|
659
|
+
`(settled by another writer, or the run advanced).`, {
|
|
660
|
+
code: 'STATE_CLAIM_LOST',
|
|
661
|
+
category: 'STATE',
|
|
662
|
+
agentAction: 'resolve_precondition',
|
|
663
|
+
retryable: false,
|
|
664
|
+
details: { runId: options.runId, step: options.command },
|
|
665
|
+
});
|
|
666
|
+
return {
|
|
667
|
+
...makeErrorEnvelope(options, result.run, err, definition, extraWarnings),
|
|
668
|
+
evidence: allEvidence,
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
case 'run_terminal': {
|
|
672
|
+
const err = new WorkflowError(`Run '${options.runId}' is terminal; cannot settle step '${options.command}'.`, {
|
|
673
|
+
code: 'STATE_RUN_TERMINAL',
|
|
674
|
+
category: 'STATE',
|
|
675
|
+
agentAction: 'report_to_user',
|
|
676
|
+
retryable: false,
|
|
677
|
+
details: { runId: options.runId, run_phase: result.run.run_phase },
|
|
678
|
+
});
|
|
679
|
+
return makeErrorEnvelope(options, result.run, err, definition, extraWarnings);
|
|
680
|
+
}
|
|
681
|
+
case 'gate_mismatch': {
|
|
682
|
+
const err = new WorkflowError(`Step '${options.command}' is the currently open gate; resolve it via ` +
|
|
683
|
+
`submit_human_response instead of settling it directly.`, {
|
|
684
|
+
code: 'STATE_BLOCKED',
|
|
685
|
+
category: 'STATE',
|
|
686
|
+
agentAction: 'resolve_precondition',
|
|
687
|
+
retryable: false,
|
|
688
|
+
details: { runId: options.runId, step: options.command },
|
|
689
|
+
});
|
|
690
|
+
return makeErrorEnvelope(options, result.run, err, definition, extraWarnings);
|
|
691
|
+
}
|
|
692
|
+
default:
|
|
693
|
+
// run_not_terminal / ledger_not_pending / lease_held / lease_lost / rank_blocked /
|
|
694
|
+
// not_eligible / already_leased / already_marked are lease_finalizer/mark_finalizer-only —
|
|
695
|
+
// settle_step never returns them (design record §7).
|
|
696
|
+
throw new Error(`buildSettlementRefusalEnvelope: unreachable settle_step refusal reason '${result.reason}'`);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Ok-shaped envelope for a `settle_step` NOOP (`already_settled`) — the idempotent-retry case
|
|
701
|
+
* (design record §7: ok-shaped, calm context_hint, never `report_to_user`). Drain-aware: when the
|
|
702
|
+
* fresh run still carries pending finalizers, this retry attempts the SAME post-commit drain a
|
|
703
|
+
* fresh apply would have run — recovering an ambiguous-retry crash window (§6). A drain failure
|
|
704
|
+
* degrades to a warning (never an error status) — the settle itself is not in question here.
|
|
705
|
+
*/
|
|
706
|
+
async function buildAlreadySettledEnvelope(store, definition, options, result, traceWarnings) {
|
|
707
|
+
let run = result.run;
|
|
708
|
+
let drainWarnings = [];
|
|
709
|
+
const hasPending = Object.values(run.finalizer_ledger ?? {}).some((e) => e.status === 'pending');
|
|
710
|
+
if (hasPending) {
|
|
711
|
+
try {
|
|
712
|
+
const drainOutcome = await drainFinalizers(store, definition, options.registry, options.runId);
|
|
713
|
+
run = drainOutcome.run;
|
|
714
|
+
drainWarnings = drainOutcome.warnings;
|
|
715
|
+
}
|
|
716
|
+
catch (err) {
|
|
717
|
+
drainWarnings = [
|
|
718
|
+
`post-commit finalizer drain failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
719
|
+
];
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
const nextActions = run.terminal_state ? [] : buildNextActions(definition, run);
|
|
723
|
+
return {
|
|
724
|
+
command: options.command,
|
|
725
|
+
run_id: options.runId,
|
|
726
|
+
run_version: run.version,
|
|
727
|
+
status: 'ok',
|
|
728
|
+
data: {},
|
|
729
|
+
evidence: [],
|
|
730
|
+
warnings: mergeWarnings(traceWarnings, ...drainWarnings),
|
|
731
|
+
errors: [],
|
|
732
|
+
context_hint: `Step '${options.command}' was already settled (a duplicate/retried attempt) — no action was taken.`,
|
|
733
|
+
run_phase: run.run_phase,
|
|
734
|
+
next_actions: nextActions,
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
/**
|
|
738
|
+
* Design record §10 (I16, fail-closed dormancy): the ONE advisory warning every legacy
|
|
739
|
+
* (dormancy-fallback) seal-site envelope carries when `store.settleStep` is undeclared — never a
|
|
740
|
+
* hard requirement; the legacy read-then-update path remains fully functional. This dual branch
|
|
741
|
+
* persists until a major version (final-gate F4/R13).
|
|
742
|
+
*/
|
|
743
|
+
const DORMANCY_ADVISORY = 'settled via the legacy compatibility path — this store does not declare atomic settlement ' +
|
|
744
|
+
'(RunStore.settleStep); upgrade the store to close the fan-out seal race (issue #279)';
|
|
598
745
|
/**
|
|
599
746
|
* Validates eligibility, claims the step, executes it through the dispatcher with retry
|
|
600
747
|
* and timeout support, captures evidence, persists the updated run record, and returns
|
|
@@ -958,6 +1105,8 @@ export async function executeStep(store, definition, options) {
|
|
|
958
1105
|
if (err instanceof WorkflowError) {
|
|
959
1106
|
if (err.code === 'STATE_STEP_ALREADY_CLAIMED') {
|
|
960
1107
|
const freshRun = await store.get(options.runId).catch(() => run);
|
|
1108
|
+
// Design record §6: append the drain advisory when the fresh run carries pendings.
|
|
1109
|
+
const claimedAdvisory = finalizerDrainAdvisory(freshRun);
|
|
961
1110
|
return {
|
|
962
1111
|
command: options.command,
|
|
963
1112
|
run_id: options.runId,
|
|
@@ -965,7 +1114,7 @@ export async function executeStep(store, definition, options) {
|
|
|
965
1114
|
status: 'blocked',
|
|
966
1115
|
data: {},
|
|
967
1116
|
evidence: [],
|
|
968
|
-
warnings: [],
|
|
1117
|
+
warnings: claimedAdvisory !== undefined ? [claimedAdvisory] : [],
|
|
969
1118
|
errors: [],
|
|
970
1119
|
agent_action: 'resolve_precondition',
|
|
971
1120
|
context_hint: `Step '${options.command}' was already claimed by another process.`,
|
|
@@ -977,6 +1126,19 @@ export async function executeStep(store, definition, options) {
|
|
|
977
1126
|
},
|
|
978
1127
|
};
|
|
979
1128
|
}
|
|
1129
|
+
// Design record §6: STATE_STEP_NOT_ELIGIBLE (a claim-time eligibility re-check race) also
|
|
1130
|
+
// gets the drain advisory when the fresh run carries pendings — a fresh re-read since `run`
|
|
1131
|
+
// (Step 1's load) may be stale by the time claimStep's own re-check inside its lock raced.
|
|
1132
|
+
if (err.code === 'STATE_STEP_NOT_ELIGIBLE') {
|
|
1133
|
+
const freshRun = await store.get(options.runId).catch(() => run);
|
|
1134
|
+
const notEligibleAdvisory = finalizerDrainAdvisory(freshRun);
|
|
1135
|
+
const extraWarnings = notEligibleAdvisory !== undefined
|
|
1136
|
+
? [...traceWarnings, notEligibleAdvisory]
|
|
1137
|
+
: traceWarnings.length > 0
|
|
1138
|
+
? traceWarnings
|
|
1139
|
+
: undefined;
|
|
1140
|
+
return makeErrorEnvelope(options, freshRun, err, definition, extraWarnings);
|
|
1141
|
+
}
|
|
980
1142
|
return makeErrorEnvelope(options, run, err, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
981
1143
|
}
|
|
982
1144
|
return makeErrorEnvelope(options, run, new WorkflowError('Failed to claim step', {
|
|
@@ -1335,6 +1497,75 @@ export async function executeStep(store, definition, options) {
|
|
|
1335
1497
|
}),
|
|
1336
1498
|
status: 'skipped',
|
|
1337
1499
|
};
|
|
1500
|
+
// issue #279 (increment 1, PR-B): the migrated path — a store declaring settleStep
|
|
1501
|
+
// settles this abort atomically against FRESH state (not `pendingRun`, which may be
|
|
1502
|
+
// stale relative to a concurrent sibling settle). Dormancy: an undeclaring store falls
|
|
1503
|
+
// through to the byte-identical legacy path below (I16/#169 fail-closed dormancy).
|
|
1504
|
+
if (store.settleStep !== undefined) {
|
|
1505
|
+
const abortClaimToken = pendingRun.claims?.[options.command]?.token;
|
|
1506
|
+
const delta = {
|
|
1507
|
+
kind: 'settle_step',
|
|
1508
|
+
step: options.command,
|
|
1509
|
+
outcome: 'abort',
|
|
1510
|
+
...(abortClaimToken !== undefined ? { claimToken: abortClaimToken } : {}),
|
|
1511
|
+
evidence: [abortEvidence],
|
|
1512
|
+
abort: { stepId: options.command, abortMessage },
|
|
1513
|
+
};
|
|
1514
|
+
let result;
|
|
1515
|
+
try {
|
|
1516
|
+
result = await store.settleStep(options.runId, delta, definition);
|
|
1517
|
+
}
|
|
1518
|
+
catch (err) {
|
|
1519
|
+
// THROWN infra errors (lock exhaustion, run-not-found, I/O) — the complete-site's
|
|
1520
|
+
// existing catch shape, replicated at all three migrated sites.
|
|
1521
|
+
if (err instanceof WorkflowError) {
|
|
1522
|
+
return makeErrorEnvelope(options, pendingRun, err, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
1523
|
+
}
|
|
1524
|
+
const internal = new WorkflowError('Failed to persist run update', {
|
|
1525
|
+
code: 'ENGINE_STORE_FAILED',
|
|
1526
|
+
category: 'ENGINE',
|
|
1527
|
+
agentAction: 'stop',
|
|
1528
|
+
retryable: false,
|
|
1529
|
+
});
|
|
1530
|
+
return makeErrorEnvelope(options, pendingRun, internal, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
1531
|
+
}
|
|
1532
|
+
if (!result.applied) {
|
|
1533
|
+
if (result.reason === 'already_settled') {
|
|
1534
|
+
return buildAlreadySettledEnvelope(store, definition, options, { ...result, reason: 'already_settled' }, traceWarnings);
|
|
1535
|
+
}
|
|
1536
|
+
return buildSettlementRefusalEnvelope(options, definition, result, [abortEvidence], traceWarnings);
|
|
1537
|
+
}
|
|
1538
|
+
// applied: true — abort is UNCONDITIONALLY terminal (transitioned is always true here;
|
|
1539
|
+
// isTerminal(fresh) was already refused above inside applySettlement).
|
|
1540
|
+
let finalRun = result.run;
|
|
1541
|
+
let drainWarnings = [];
|
|
1542
|
+
const reArmWarnings = computeReArmWarnings(pendingRun.finalizer_ledger, result.run.finalizer_ledger);
|
|
1543
|
+
try {
|
|
1544
|
+
const drainOutcome = await drainFinalizers(store, definition, options.registry, options.runId);
|
|
1545
|
+
finalRun = drainOutcome.run;
|
|
1546
|
+
drainWarnings = drainOutcome.warnings;
|
|
1547
|
+
}
|
|
1548
|
+
catch (err) {
|
|
1549
|
+
// A drain failure is NEVER the step's own failure — the abort already committed.
|
|
1550
|
+
drainWarnings = [
|
|
1551
|
+
`post-commit finalizer drain failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
1552
|
+
];
|
|
1553
|
+
}
|
|
1554
|
+
return {
|
|
1555
|
+
command: options.command,
|
|
1556
|
+
run_id: options.runId,
|
|
1557
|
+
run_version: finalRun.version,
|
|
1558
|
+
status: 'ok',
|
|
1559
|
+
data: {},
|
|
1560
|
+
evidence: [abortEvidence],
|
|
1561
|
+
warnings: mergeWarnings(traceWarnings, ...reArmWarnings, ...drainWarnings),
|
|
1562
|
+
errors: [],
|
|
1563
|
+
context_hint: `Handler step '${options.command}' aborted the run: ${abortMessage}`,
|
|
1564
|
+
run_phase: finalRun.run_phase,
|
|
1565
|
+
next_actions: [],
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
// --- Legacy path (dormancy fallback — byte-identical to pre-#279 behavior) ---
|
|
1338
1569
|
const withHandlerSkipped = {
|
|
1339
1570
|
...pendingRun,
|
|
1340
1571
|
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
@@ -1382,7 +1613,9 @@ export async function executeStep(store, definition, options) {
|
|
|
1382
1613
|
evidence: [abortEvidence],
|
|
1383
1614
|
// Issue #140: was hardcoded `[]` — now threads traceWarnings (e.g. the programmatic
|
|
1384
1615
|
// on_timeout/idempotent gate advisory above) so it survives this settle path too.
|
|
1385
|
-
|
|
1616
|
+
// Issue #279 (increment 1, PR-B): + the ONE dormancy advisory (I16) — this IS the
|
|
1617
|
+
// legacy path (store.settleStep undeclared).
|
|
1618
|
+
warnings: mergeWarnings(traceWarnings, DORMANCY_ADVISORY),
|
|
1386
1619
|
errors: [],
|
|
1387
1620
|
context_hint: `Handler step '${options.command}' aborted the run: ${abortMessage}`,
|
|
1388
1621
|
run_phase: (persistedAbortRun ?? abortedRun).run_phase,
|
|
@@ -1744,6 +1977,141 @@ export async function executeStep(store, definition, options) {
|
|
|
1744
1977
|
next_actions: blockedNextActions,
|
|
1745
1978
|
};
|
|
1746
1979
|
}
|
|
1980
|
+
// issue #279 (increment 1, PR-B): the migrated path — settles this failure atomically against
|
|
1981
|
+
// FRESH state via the store's own settleStep. Dormancy: an undeclaring store falls through to
|
|
1982
|
+
// the byte-identical legacy path below (I16/#169 fail-closed dormancy).
|
|
1983
|
+
if (store.settleStep !== undefined) {
|
|
1984
|
+
const failClaimToken = pendingRun.claims?.[options.command]?.token;
|
|
1985
|
+
const delta = {
|
|
1986
|
+
kind: 'settle_step',
|
|
1987
|
+
step: options.command,
|
|
1988
|
+
outcome: 'fail',
|
|
1989
|
+
...(failClaimToken !== undefined ? { claimToken: failClaimToken } : {}),
|
|
1990
|
+
evidence: allEvidence,
|
|
1991
|
+
failureMessage: dispatchError.message,
|
|
1992
|
+
};
|
|
1993
|
+
let result;
|
|
1994
|
+
try {
|
|
1995
|
+
result = await store.settleStep(options.runId, delta, definition);
|
|
1996
|
+
}
|
|
1997
|
+
catch (err) {
|
|
1998
|
+
// THROWN infra errors — the same catch shape replicated at all three migrated sites.
|
|
1999
|
+
if (err instanceof WorkflowError) {
|
|
2000
|
+
return makeErrorEnvelope(options, pendingRun, err, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
2001
|
+
}
|
|
2002
|
+
const internal = new WorkflowError('Failed to persist run update', {
|
|
2003
|
+
code: 'ENGINE_STORE_FAILED',
|
|
2004
|
+
category: 'ENGINE',
|
|
2005
|
+
agentAction: 'stop',
|
|
2006
|
+
retryable: false,
|
|
2007
|
+
});
|
|
2008
|
+
return makeErrorEnvelope(options, pendingRun, internal, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
2009
|
+
}
|
|
2010
|
+
if (!result.applied) {
|
|
2011
|
+
if (result.reason === 'already_settled') {
|
|
2012
|
+
return buildAlreadySettledEnvelope(store, definition, options, { ...result, reason: 'already_settled' }, traceWarnings);
|
|
2013
|
+
}
|
|
2014
|
+
// WAL/sealFenced gates become result.applied (BU-12): claim_lost ⇒ the WAL SURVIVES
|
|
2015
|
+
// (reclaim's drain owns it) — no WAL cleanup attempted on ANY refusal.
|
|
2016
|
+
return buildSettlementRefusalEnvelope(options, definition, result, allEvidence, traceWarnings);
|
|
2017
|
+
}
|
|
2018
|
+
let finalRun = result.run;
|
|
2019
|
+
let drainWarnings = [];
|
|
2020
|
+
const reArmWarnings = result.transitioned
|
|
2021
|
+
? computeReArmWarnings(pendingRun.finalizer_ledger, result.run.finalizer_ledger)
|
|
2022
|
+
: [];
|
|
2023
|
+
if (result.transitioned) {
|
|
2024
|
+
try {
|
|
2025
|
+
const drainOutcome = await drainFinalizers(store, definition, options.registry, options.runId);
|
|
2026
|
+
finalRun = drainOutcome.run;
|
|
2027
|
+
drainWarnings = drainOutcome.warnings;
|
|
2028
|
+
}
|
|
2029
|
+
catch (err) {
|
|
2030
|
+
drainWarnings = [
|
|
2031
|
+
`post-commit finalizer drain failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2032
|
+
];
|
|
2033
|
+
}
|
|
2034
|
+
}
|
|
2035
|
+
// WAL cleanup — placement stays post-commit (unchanged), now gated on result.applied
|
|
2036
|
+
// (BU-12) rather than a separate persistedRun-defined check (the settle already committed
|
|
2037
|
+
// by the time we reach here, so there is no "did the persist succeed" ambiguity to gate on).
|
|
2038
|
+
let migratedWalCleanupWarning;
|
|
2039
|
+
{
|
|
2040
|
+
let performPlainDelete = true;
|
|
2041
|
+
if (adoptionPartition !== undefined &&
|
|
2042
|
+
adoptionPartition.preserved_foreign > 0 &&
|
|
2043
|
+
options.traceBufferStore !== undefined &&
|
|
2044
|
+
storeDeclaresSeal(options.traceBufferStore)) {
|
|
2045
|
+
try {
|
|
2046
|
+
const sealResult = await options.traceBufferStore.sealFenced(options.runId, options.command, buildSettleSealGuard(store, options.runId, options.command));
|
|
2047
|
+
if (sealResult.sealed) {
|
|
2048
|
+
performPlainDelete = false;
|
|
2049
|
+
migratedWalCleanupWarning =
|
|
2050
|
+
`${adoptionPartition.preserved_foreign} foreign line(s) preserved (sealed) — ` +
|
|
2051
|
+
'retrieve via `realm run export`';
|
|
2052
|
+
}
|
|
2053
|
+
else if (sealResult.reason === 'capped') {
|
|
2054
|
+
migratedWalCleanupWarning =
|
|
2055
|
+
'preservation cap reached — foreign lines destroyed, not preserved';
|
|
2056
|
+
}
|
|
2057
|
+
else {
|
|
2058
|
+
performPlainDelete = false;
|
|
2059
|
+
}
|
|
2060
|
+
}
|
|
2061
|
+
catch (err) {
|
|
2062
|
+
performPlainDelete = false;
|
|
2063
|
+
migratedWalCleanupWarning = `Failed to seal trace buffer after step failure: ${err instanceof Error ? err.message : String(err)}`;
|
|
2064
|
+
}
|
|
2065
|
+
}
|
|
2066
|
+
if (performPlainDelete) {
|
|
2067
|
+
try {
|
|
2068
|
+
await options.traceBufferStore?.delete(options.runId, options.command);
|
|
2069
|
+
}
|
|
2070
|
+
catch (walErr) {
|
|
2071
|
+
migratedWalCleanupWarning = `Failed to clean up trace buffer after step failure: ${walErr instanceof Error ? walErr.message : String(walErr)}`;
|
|
2072
|
+
}
|
|
2073
|
+
}
|
|
2074
|
+
}
|
|
2075
|
+
const migratedEffectiveAction = resolvePostDispatchAgentAction(dispatchError, finalRun.terminal_state);
|
|
2076
|
+
let migratedNextActions = [];
|
|
2077
|
+
if (migratedEffectiveAction !== 'stop') {
|
|
2078
|
+
try {
|
|
2079
|
+
migratedNextActions = buildNextActions(definition, finalRun);
|
|
2080
|
+
}
|
|
2081
|
+
catch {
|
|
2082
|
+
// buildNextActions can throw for unresolvable template references; fall back to [].
|
|
2083
|
+
}
|
|
2084
|
+
}
|
|
2085
|
+
const migratedContextHint = migratedEffectiveAction === 'stop'
|
|
2086
|
+
? `Step '${options.command}' failed. Run is terminated.`
|
|
2087
|
+
: migratedEffectiveAction === 'wait_and_proceed'
|
|
2088
|
+
? `Step '${options.command}' was rate-limited. Wait ${dispatchError.retry_after !== undefined ? `${dispatchError.retry_after} second(s)` : 'a moment'} then follow next_actions — no human intervention required.`
|
|
2089
|
+
: migratedEffectiveAction === 'wait_for_human'
|
|
2090
|
+
? `Step '${options.command}' failed due to external service unavailability. Wait for service recovery, then proceed with the steps in next_actions.`
|
|
2091
|
+
: `Step '${options.command}' failed. ${result.transitioned ? 'Run is terminated.' : 'Recovery steps are available in next_actions.'}`;
|
|
2092
|
+
return {
|
|
2093
|
+
command: options.command,
|
|
2094
|
+
run_id: options.runId,
|
|
2095
|
+
run_version: finalRun.version,
|
|
2096
|
+
status: 'error',
|
|
2097
|
+
data: {},
|
|
2098
|
+
evidence: allEvidence,
|
|
2099
|
+
warnings: mergeWarnings(traceWarnings, migratedWalCleanupWarning, ...reArmWarnings, ...drainWarnings),
|
|
2100
|
+
errors: [dispatchError.message],
|
|
2101
|
+
agent_action: migratedEffectiveAction,
|
|
2102
|
+
error_code: dispatchError.code,
|
|
2103
|
+
...(Object.keys(dispatchError.details).length > 0
|
|
2104
|
+
? { error_details: dispatchError.details }
|
|
2105
|
+
: {}),
|
|
2106
|
+
...(dispatchError.retry_after !== undefined
|
|
2107
|
+
? { retry_after: dispatchError.retry_after }
|
|
2108
|
+
: {}),
|
|
2109
|
+
context_hint: migratedContextHint,
|
|
2110
|
+
run_phase: finalRun.run_phase,
|
|
2111
|
+
next_actions: migratedNextActions,
|
|
2112
|
+
};
|
|
2113
|
+
}
|
|
2114
|
+
// --- Legacy path (dormancy fallback — byte-identical to pre-#279 behavior) ---
|
|
1747
2115
|
// Pure in-memory derivations — no I/O, no try required.
|
|
1748
2116
|
const afterFail = {
|
|
1749
2117
|
...pendingRun,
|
|
@@ -1879,7 +2247,9 @@ export async function executeStep(store, definition, options) {
|
|
|
1879
2247
|
status: 'error',
|
|
1880
2248
|
data: {},
|
|
1881
2249
|
evidence: allEvidence,
|
|
1882
|
-
|
|
2250
|
+
// Issue #279 (increment 1, PR-B): + the ONE dormancy advisory (I16) — this IS the legacy
|
|
2251
|
+
// path (store.settleStep undeclared).
|
|
2252
|
+
warnings: mergeWarnings(traceWarnings, storeCleanupWarning ?? walCleanupWarning, DORMANCY_ADVISORY),
|
|
1883
2253
|
errors: [dispatchError.message],
|
|
1884
2254
|
agent_action: effectiveAction,
|
|
1885
2255
|
// issue #140 (D3 §2, discriminator OBSERVABLE): additive-optional — lets a caller
|
|
@@ -2037,6 +2407,134 @@ export async function executeStep(store, definition, options) {
|
|
|
2037
2407
|
};
|
|
2038
2408
|
}
|
|
2039
2409
|
// Step 6: Move step from in_progress to completed, compute terminal state.
|
|
2410
|
+
// issue #279 (increment 1, PR-B): the migrated path — settles this completion atomically
|
|
2411
|
+
// against FRESH state via the store's own settleStep. Dormancy: an undeclaring store falls
|
|
2412
|
+
// through to the byte-identical legacy path below (I16/#169 fail-closed dormancy).
|
|
2413
|
+
if (store.settleStep !== undefined) {
|
|
2414
|
+
const completeClaimToken = pendingRun.claims?.[options.command]?.token;
|
|
2415
|
+
const delta = {
|
|
2416
|
+
kind: 'settle_step',
|
|
2417
|
+
step: options.command,
|
|
2418
|
+
outcome: 'complete',
|
|
2419
|
+
...(completeClaimToken !== undefined ? { claimToken: completeClaimToken } : {}),
|
|
2420
|
+
evidence: allEvidence,
|
|
2421
|
+
};
|
|
2422
|
+
let result;
|
|
2423
|
+
try {
|
|
2424
|
+
result = await store.settleStep(options.runId, delta, definition);
|
|
2425
|
+
}
|
|
2426
|
+
catch (err) {
|
|
2427
|
+
// THROWN infra errors — the same catch shape replicated at all three migrated sites.
|
|
2428
|
+
if (err instanceof WorkflowError) {
|
|
2429
|
+
return makeErrorEnvelope(options, pendingRun, err, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
2430
|
+
}
|
|
2431
|
+
const internal = new WorkflowError('Failed to persist run update', {
|
|
2432
|
+
code: 'ENGINE_STORE_FAILED',
|
|
2433
|
+
category: 'ENGINE',
|
|
2434
|
+
agentAction: 'stop',
|
|
2435
|
+
retryable: false,
|
|
2436
|
+
});
|
|
2437
|
+
return makeErrorEnvelope(options, pendingRun, internal, definition, traceWarnings.length > 0 ? traceWarnings : undefined);
|
|
2438
|
+
}
|
|
2439
|
+
if (!result.applied) {
|
|
2440
|
+
if (result.reason === 'already_settled') {
|
|
2441
|
+
return buildAlreadySettledEnvelope(store, definition, options, { ...result, reason: 'already_settled' }, traceWarnings);
|
|
2442
|
+
}
|
|
2443
|
+
// WAL/sealFenced gates become result.applied (BU-12): claim_lost ⇒ the WAL SURVIVES.
|
|
2444
|
+
return buildSettlementRefusalEnvelope(options, definition, result, allEvidence, traceWarnings);
|
|
2445
|
+
}
|
|
2446
|
+
let finalRun = result.run;
|
|
2447
|
+
let drainWarnings = [];
|
|
2448
|
+
const reArmWarnings = result.transitioned
|
|
2449
|
+
? computeReArmWarnings(pendingRun.finalizer_ledger, result.run.finalizer_ledger)
|
|
2450
|
+
: [];
|
|
2451
|
+
if (result.transitioned) {
|
|
2452
|
+
try {
|
|
2453
|
+
const drainOutcome = await drainFinalizers(store, definition, options.registry, options.runId);
|
|
2454
|
+
finalRun = drainOutcome.run;
|
|
2455
|
+
drainWarnings = drainOutcome.warnings;
|
|
2456
|
+
}
|
|
2457
|
+
catch (err) {
|
|
2458
|
+
drainWarnings = [
|
|
2459
|
+
`post-commit finalizer drain failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
2460
|
+
];
|
|
2461
|
+
}
|
|
2462
|
+
}
|
|
2463
|
+
// WAL cleanup — placement stays post-commit (unchanged), gated on result.applied (BU-12).
|
|
2464
|
+
let migratedSuccessWalCleanupWarning;
|
|
2465
|
+
{
|
|
2466
|
+
let performPlainDelete = true;
|
|
2467
|
+
if (adoptionPartition !== undefined &&
|
|
2468
|
+
adoptionPartition.preserved_foreign > 0 &&
|
|
2469
|
+
options.traceBufferStore !== undefined &&
|
|
2470
|
+
storeDeclaresSeal(options.traceBufferStore)) {
|
|
2471
|
+
try {
|
|
2472
|
+
const sealResult = await options.traceBufferStore.sealFenced(options.runId, options.command, buildSettleSealGuard(store, options.runId, options.command));
|
|
2473
|
+
if (sealResult.sealed) {
|
|
2474
|
+
performPlainDelete = false;
|
|
2475
|
+
migratedSuccessWalCleanupWarning =
|
|
2476
|
+
`${adoptionPartition.preserved_foreign} foreign line(s) preserved (sealed) — ` +
|
|
2477
|
+
'retrieve via `realm run export`';
|
|
2478
|
+
}
|
|
2479
|
+
else if (sealResult.reason === 'capped') {
|
|
2480
|
+
migratedSuccessWalCleanupWarning =
|
|
2481
|
+
'preservation cap reached — foreign lines destroyed, not preserved';
|
|
2482
|
+
}
|
|
2483
|
+
else {
|
|
2484
|
+
performPlainDelete = false;
|
|
2485
|
+
}
|
|
2486
|
+
}
|
|
2487
|
+
catch (err) {
|
|
2488
|
+
performPlainDelete = false;
|
|
2489
|
+
migratedSuccessWalCleanupWarning = `Failed to seal trace buffer after step completion: ${err instanceof Error ? err.message : String(err)}`;
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
2492
|
+
if (performPlainDelete) {
|
|
2493
|
+
try {
|
|
2494
|
+
await options.traceBufferStore?.delete(options.runId, options.command);
|
|
2495
|
+
}
|
|
2496
|
+
catch (err) {
|
|
2497
|
+
migratedSuccessWalCleanupWarning = `Failed to clean up trace buffer after step completion: ${err instanceof Error ? err.message : String(err)}`;
|
|
2498
|
+
}
|
|
2499
|
+
}
|
|
2500
|
+
}
|
|
2501
|
+
const migratedDefaultedStepsDurabilityWarning = finalRun.defaulted_steps !== undefined &&
|
|
2502
|
+
finalRun.defaulted_steps.length > 0 &&
|
|
2503
|
+
!persistsField(store, 'defaulted_steps')
|
|
2504
|
+
? 'run-level defaultedness marker (defaulted_steps) not durable on this store'
|
|
2505
|
+
: undefined;
|
|
2506
|
+
const migratedNextActions = finalRun.terminal_state
|
|
2507
|
+
? []
|
|
2508
|
+
: buildNextActions(definition, finalRun);
|
|
2509
|
+
const migratedOrientation = finalRun.terminal_state
|
|
2510
|
+
? `Run completed (phase: '${finalRun.run_phase}'). Call get_run_state with run_id '${options.runId}' to retrieve the full evidence record.`
|
|
2511
|
+
: migratedNextActions.length > 0
|
|
2512
|
+
? `Step '${options.command}' completed. ${migratedNextActions.length} step(s) now available.`
|
|
2513
|
+
: `Step '${options.command}' completed. Waiting for other steps to complete.`;
|
|
2514
|
+
return {
|
|
2515
|
+
command: options.command,
|
|
2516
|
+
run_id: options.runId,
|
|
2517
|
+
run_version: finalRun.version,
|
|
2518
|
+
status: 'ok',
|
|
2519
|
+
data: output,
|
|
2520
|
+
evidence: allEvidence,
|
|
2521
|
+
warnings: mergeWarnings(traceWarnings, currentWarn, migratedSuccessWalCleanupWarning, migratedDefaultedStepsDurabilityWarning, ...reArmWarnings, ...drainWarnings),
|
|
2522
|
+
errors: [],
|
|
2523
|
+
context_hint: migratedOrientation,
|
|
2524
|
+
run_phase: finalRun.run_phase,
|
|
2525
|
+
next_actions: migratedNextActions,
|
|
2526
|
+
...(carriageActive && adoptionPartition !== undefined
|
|
2527
|
+
? {
|
|
2528
|
+
adopted_own: adoptionPartition.adopted_own,
|
|
2529
|
+
adopted_anonymous: adoptionPartition.adopted_anonymous,
|
|
2530
|
+
preserved_foreign: adoptionPartition.preserved_foreign,
|
|
2531
|
+
}
|
|
2532
|
+
: {}),
|
|
2533
|
+
...(settledByDefault ? { settled_by_default: true } : {}),
|
|
2534
|
+
...(finalRun.defaulted_steps?.length ? { defaulted_steps: finalRun.defaulted_steps } : {}),
|
|
2535
|
+
};
|
|
2536
|
+
}
|
|
2537
|
+
// --- Legacy path (dormancy fallback — byte-identical to pre-#279 behavior) ---
|
|
2040
2538
|
const afterComplete = {
|
|
2041
2539
|
...pendingRun,
|
|
2042
2540
|
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
@@ -2158,7 +2656,9 @@ export async function executeStep(store, definition, options) {
|
|
|
2158
2656
|
status: 'ok',
|
|
2159
2657
|
data: output,
|
|
2160
2658
|
evidence: allEvidence,
|
|
2161
|
-
|
|
2659
|
+
// Issue #279 (increment 1, PR-B): + the ONE dormancy advisory (I16) — this IS the legacy
|
|
2660
|
+
// path (store.settleStep undeclared).
|
|
2661
|
+
warnings: mergeWarnings(traceWarnings, currentWarn, successWalCleanupWarning, defaultedStepsDurabilityWarning, DORMANCY_ADVISORY),
|
|
2162
2662
|
errors: [],
|
|
2163
2663
|
context_hint: orientation,
|
|
2164
2664
|
run_phase: savedRun.run_phase,
|
|
@@ -2462,13 +2962,6 @@ async function executeGuardStep(stepName, stepDef, definition, run) {
|
|
|
2462
2962
|
},
|
|
2463
2963
|
};
|
|
2464
2964
|
}
|
|
2465
|
-
/** Normalizes a finalizer's `on_outcome` to a set of triggers. */
|
|
2466
|
-
function finalizerTriggers(stepDef) {
|
|
2467
|
-
const raw = stepDef.on_outcome;
|
|
2468
|
-
if (raw === undefined)
|
|
2469
|
-
return new Set();
|
|
2470
|
-
return new Set(Array.isArray(raw) ? raw : [raw]);
|
|
2471
|
-
}
|
|
2472
2965
|
/**
|
|
2473
2966
|
* Drains the finalizers matching a run's terminal `outcome` and returns the FINALIZED
|
|
2474
2967
|
* RunRecord — the workflow-level try/catch/finally at the seal. Modeled on
|
|
@@ -2496,22 +2989,15 @@ async function buildFinalizedSeal(definition, sealDraft, outcome, registry) {
|
|
|
2496
2989
|
const hasFinalizers = Object.values(definition.steps).some((s) => s.execution === 'finalizer');
|
|
2497
2990
|
if (!hasFinalizers)
|
|
2498
2991
|
return sealDraft;
|
|
2992
|
+
// issue #279 (increment 1, PR-A extraction): the grouping/ordering logic itself now lives in
|
|
2993
|
+
// settlement.ts's selectFinalizers, shared with `mintFresh` — this call passes the SAME inputs
|
|
2994
|
+
// the inline loop used to compute over, so the returned name list is byte-identical to what
|
|
2995
|
+
// `[...groupA, ...groupB]` produced before the extraction.
|
|
2499
2996
|
const settled = new Set([...sealDraft.completed_steps, ...sealDraft.failed_steps]);
|
|
2500
|
-
const
|
|
2501
|
-
const groupB = [];
|
|
2502
|
-
for (const [name, step] of Object.entries(definition.steps)) {
|
|
2503
|
-
if (step.execution !== 'finalizer')
|
|
2504
|
-
continue;
|
|
2505
|
-
if (settled.has(name))
|
|
2506
|
-
continue; // at-most-once per run (resume / re-drive safety)
|
|
2507
|
-
const triggers = finalizerTriggers(step);
|
|
2508
|
-
if (triggers.has(outcome))
|
|
2509
|
-
groupA.push([name, step]);
|
|
2510
|
-
else if (triggers.has('always'))
|
|
2511
|
-
groupB.push([name, step]);
|
|
2512
|
-
}
|
|
2997
|
+
const selected = selectFinalizers(definition, settled, outcome);
|
|
2513
2998
|
let record = sealDraft;
|
|
2514
|
-
for (const
|
|
2999
|
+
for (const name of selected) {
|
|
3000
|
+
const step = definition.steps[name];
|
|
2515
3001
|
const now = new Date();
|
|
2516
3002
|
const evidenceByStep = buildEvidenceByStep(record);
|
|
2517
3003
|
const timeoutMs = (step.timeout_seconds ?? DRAIN_CEILING_SECONDS) * 1000;
|
|
@@ -2585,6 +3071,185 @@ async function buildFinalizedSeal(definition, sealDraft, outcome, registry) {
|
|
|
2585
3071
|
}
|
|
2586
3072
|
return { ...record, terminal_state: true };
|
|
2587
3073
|
}
|
|
3074
|
+
/** Every currently-`'pending'` finalizer in `run`'s ledger, ascending by rank — the order the
|
|
3075
|
+
* drain loop below consumes (design record §6). */
|
|
3076
|
+
function pendingByRank(run) {
|
|
3077
|
+
return Object.entries(run.finalizer_ledger ?? {})
|
|
3078
|
+
.filter(([, e]) => e.status === 'pending')
|
|
3079
|
+
.sort(([, a], [, b]) => a.rank - b.rank)
|
|
3080
|
+
.map(([name]) => name);
|
|
3081
|
+
}
|
|
3082
|
+
/** Bounded retry count for `mark_finalizer` on a thrown `STATE_RUN_BUSY` (design record §6: "bounded
|
|
3083
|
+
* retry on thrown STATE_RUN_BUSY"). Small and fixed — lock contention self-heals quickly; this is
|
|
3084
|
+
* not a backoff schedule, just enough attempts to ride out a transient contender. */
|
|
3085
|
+
const DRAIN_MARK_BUSY_RETRIES = 3;
|
|
3086
|
+
/**
|
|
3087
|
+
* Post-commit finalizer drain (design record §6, D2, issue #279 increment 1 PR-B). Drains the
|
|
3088
|
+
* LOWEST-ranked pending finalizer, re-reading the record returned by the LAST settle/lease/mark at
|
|
3089
|
+
* every step — NEVER a pass-start snapshot (the DRAIN_REREADS_LEDGER pin: a snapshot taken once at
|
|
3090
|
+
* the top of this function would go stale the instant the first lease/mark commits, silently
|
|
3091
|
+
* reintroducing exactly the kind of pre-commit assumption #279 exists to eliminate).
|
|
3092
|
+
*
|
|
3093
|
+
* Lives here (not in settlement.ts) so the module-private `withTimeout`/`callHandler` — already
|
|
3094
|
+
* proven by `buildFinalizedSeal` above — are reused without exporting them; `settlement.ts` stays
|
|
3095
|
+
* pure/no-I/O (design record §7 CS-purity).
|
|
3096
|
+
*
|
|
3097
|
+
* Registry pre-check: an absent handler leaves the entry pending and discloses why, rather than
|
|
3098
|
+
* burning it (leasing it, failing the call, and marking it 'failed' would destroy the "recoverable
|
|
3099
|
+
* on a capable runner" property a still-pending entry carries). Rank-monotonic: a pass HALTS at the
|
|
3100
|
+
* first entry it cannot lease/execute, so a lower-ranked absent-handler or held-lease entry
|
|
3101
|
+
* withholds every higher-ranked (later) finalizer behind it (R11) — this is deliberate: rank order
|
|
3102
|
+
* is a DELIVERY order guarantee, not just a preference.
|
|
3103
|
+
*
|
|
3104
|
+
* `not_eligible` at lease OR mark time is a contract violation (an unknown finalizer id — mintFresh
|
|
3105
|
+
* only ever mints real workflow finalizer names) and aborts the pass LOUD (throws), distinct from
|
|
3106
|
+
* `ledger_not_pending` at lease time, which ADVANCES (someone else already resolved it). A
|
|
3107
|
+
* non-BUSY infra throw from `mark_finalizer` also aborts loud, with the remaining-pending list in
|
|
3108
|
+
* the message. The executed-but-not-recorded warning (three reasons: `lease_lost`,
|
|
3109
|
+
* `ledger_not_pending`-AFTER-execution, `run_not_terminal`) halts the pass rather than continuing —
|
|
3110
|
+
* each of those three reasons refuses the mark WITHOUT mutating the ledger entry's `'pending'`
|
|
3111
|
+
* status, so continuing would re-select the SAME entry next iteration (an infinite loop); halting
|
|
3112
|
+
* is the only forward-progress-safe response until an operator or a later terminal edge resolves it.
|
|
3113
|
+
*/
|
|
3114
|
+
export async function drainFinalizers(store, definition, registry, runId) {
|
|
3115
|
+
// .bind(store): a bare `store.settleStep` reference loses its `this` binding — the store's own
|
|
3116
|
+
// method body (e.g. JsonFileStore's `this.ensureDir()`/`this.filePath()`) would throw on
|
|
3117
|
+
// `this === undefined` once called through the detached reference below.
|
|
3118
|
+
const settleStep = store.settleStep?.bind(store);
|
|
3119
|
+
if (settleStep === undefined) {
|
|
3120
|
+
// Defensive — every caller only invokes this once it has already confirmed the store
|
|
3121
|
+
// declares settleStep. A fresh read is still the correct degenerate response.
|
|
3122
|
+
return { run: await store.get(runId), warnings: [] };
|
|
3123
|
+
}
|
|
3124
|
+
const warnings = [];
|
|
3125
|
+
let run = await store.get(runId);
|
|
3126
|
+
for (;;) {
|
|
3127
|
+
const pending = pendingByRank(run);
|
|
3128
|
+
if (pending.length === 0)
|
|
3129
|
+
break;
|
|
3130
|
+
const finalizerName = pending[0];
|
|
3131
|
+
const stepDef = definition.steps[finalizerName];
|
|
3132
|
+
const handlerName = stepDef?.handler;
|
|
3133
|
+
const handler = handlerName !== undefined ? registry?.getHandler(handlerName) : undefined;
|
|
3134
|
+
// Registry pre-check — never burn an absent handler; rank-monotonic HALT (R11).
|
|
3135
|
+
if (stepDef === undefined || handlerName === undefined || handler === undefined) {
|
|
3136
|
+
warnings.push(`finalizer '${finalizerName}' left pending — handler not available on this surface`);
|
|
3137
|
+
break;
|
|
3138
|
+
}
|
|
3139
|
+
const leaseToken = crypto.randomUUID();
|
|
3140
|
+
const leaseSeconds = stepDef.timeout_seconds ?? DRAIN_CEILING_SECONDS;
|
|
3141
|
+
const leaseResult = await settleStep(runId, { kind: 'lease_finalizer', finalizer: finalizerName, leaseToken, leaseSeconds }, definition);
|
|
3142
|
+
if (!leaseResult.applied) {
|
|
3143
|
+
if (leaseResult.reason === 'lease_held' || leaseResult.reason === 'rank_blocked') {
|
|
3144
|
+
run = leaseResult.run;
|
|
3145
|
+
break; // HALT the pass — a peer holds this lease, or a lower rank is still pending.
|
|
3146
|
+
}
|
|
3147
|
+
if (leaseResult.reason === 'ledger_not_pending') {
|
|
3148
|
+
run = leaseResult.run;
|
|
3149
|
+
continue; // ADVANCE — a peer already resolved this entry.
|
|
3150
|
+
}
|
|
3151
|
+
// not_eligible — unknown finalizer id; a contract violation (mintFresh never mints one).
|
|
3152
|
+
throw new Error(`drainFinalizers: lease refused '${leaseResult.reason}' for finalizer '${finalizerName}' ` +
|
|
3153
|
+
`on run '${runId}' — remaining pending: ${pendingByRank(leaseResult.run).join(', ')}`);
|
|
3154
|
+
}
|
|
3155
|
+
run = leaseResult.run;
|
|
3156
|
+
// callHandler under withTimeout, OUTSIDE any critical section (the store's CS ends the
|
|
3157
|
+
// instant the lease-apply write above returned).
|
|
3158
|
+
const startedAt = new Date();
|
|
3159
|
+
const evidenceByStep = buildEvidenceByStep(run);
|
|
3160
|
+
const callOptions = {
|
|
3161
|
+
runId,
|
|
3162
|
+
command: finalizerName,
|
|
3163
|
+
input: {},
|
|
3164
|
+
dispatcher: async () => ({}),
|
|
3165
|
+
...(registry !== undefined ? { registry } : {}),
|
|
3166
|
+
};
|
|
3167
|
+
let markResult;
|
|
3168
|
+
let evidenceSnapshot;
|
|
3169
|
+
try {
|
|
3170
|
+
const callResult = await withTimeout((signal) => callHandler(stepDef, callOptions, run, evidenceByStep, signal), leaseSeconds * 1000, finalizerName);
|
|
3171
|
+
if (callResult.kind === 'abort') {
|
|
3172
|
+
markResult = 'failed';
|
|
3173
|
+
evidenceSnapshot = captureEvidence({
|
|
3174
|
+
stepId: finalizerName,
|
|
3175
|
+
startedAt,
|
|
3176
|
+
completedAt: new Date(),
|
|
3177
|
+
input: {},
|
|
3178
|
+
output: { aborted: true, abort_message: callResult.message },
|
|
3179
|
+
error: `Finalizer '${finalizerName}' returned abort: ${callResult.message}`,
|
|
3180
|
+
});
|
|
3181
|
+
}
|
|
3182
|
+
else {
|
|
3183
|
+
markResult = 'completed';
|
|
3184
|
+
evidenceSnapshot = captureEvidence({
|
|
3185
|
+
stepId: finalizerName,
|
|
3186
|
+
startedAt,
|
|
3187
|
+
completedAt: new Date(),
|
|
3188
|
+
input: {},
|
|
3189
|
+
output: callResult.output,
|
|
3190
|
+
...(callResult.kind === 'warn' ? { warn: callResult.message } : {}),
|
|
3191
|
+
});
|
|
3192
|
+
}
|
|
3193
|
+
}
|
|
3194
|
+
catch (err) {
|
|
3195
|
+
markResult = 'failed';
|
|
3196
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
3197
|
+
evidenceSnapshot = captureEvidence({
|
|
3198
|
+
stepId: finalizerName,
|
|
3199
|
+
startedAt,
|
|
3200
|
+
completedAt: new Date(),
|
|
3201
|
+
input: {},
|
|
3202
|
+
output: {},
|
|
3203
|
+
error: `Finalizer '${finalizerName}' failed: ${message}`,
|
|
3204
|
+
});
|
|
3205
|
+
}
|
|
3206
|
+
// mark_finalizer — same lease token; bounded retry on a THROWN STATE_RUN_BUSY only.
|
|
3207
|
+
let markOutcome;
|
|
3208
|
+
for (let attempt = 1; attempt <= DRAIN_MARK_BUSY_RETRIES; attempt++) {
|
|
3209
|
+
try {
|
|
3210
|
+
markOutcome = await settleStep(runId, {
|
|
3211
|
+
kind: 'mark_finalizer',
|
|
3212
|
+
finalizer: finalizerName,
|
|
3213
|
+
leaseToken,
|
|
3214
|
+
result: markResult,
|
|
3215
|
+
evidence: evidenceSnapshot,
|
|
3216
|
+
}, definition);
|
|
3217
|
+
break;
|
|
3218
|
+
}
|
|
3219
|
+
catch (err) {
|
|
3220
|
+
const isBusy = err instanceof WorkflowError && err.code === 'STATE_RUN_BUSY';
|
|
3221
|
+
if (isBusy && attempt < DRAIN_MARK_BUSY_RETRIES)
|
|
3222
|
+
continue;
|
|
3223
|
+
// Non-BUSY infra throw (or BUSY exhausted) ⇒ abort the pass loud, remaining-pending named.
|
|
3224
|
+
throw new Error(`drainFinalizers: mark_finalizer failed for '${finalizerName}' on run '${runId}' — ` +
|
|
3225
|
+
`remaining pending: ${pendingByRank(run).join(', ')}: ` +
|
|
3226
|
+
`${err instanceof Error ? err.message : String(err)}`, { cause: err });
|
|
3227
|
+
}
|
|
3228
|
+
}
|
|
3229
|
+
/* istanbul ignore next -- the for-loop above always either assigns markOutcome or throws */
|
|
3230
|
+
if (markOutcome === undefined) {
|
|
3231
|
+
throw new Error(`drainFinalizers: mark_finalizer produced no outcome for '${finalizerName}' on run '${runId}'`);
|
|
3232
|
+
}
|
|
3233
|
+
if (!markOutcome.applied) {
|
|
3234
|
+
if (markOutcome.reason === 'not_eligible') {
|
|
3235
|
+
throw new Error(`drainFinalizers: mark refused 'not_eligible' for finalizer '${finalizerName}' on run ` +
|
|
3236
|
+
`'${runId}' — contract violation (mintFresh never mints an unknown id)`);
|
|
3237
|
+
}
|
|
3238
|
+
// lease_lost / ledger_not_pending(-after-execution) / run_not_terminal: the handler DID
|
|
3239
|
+
// execute, but the outcome could not be durably recorded — the three-reason
|
|
3240
|
+
// executed-but-not-recorded warning (design record §6). Halt: none of these three mutate
|
|
3241
|
+
// the entry's 'pending' status, so continuing would re-select the SAME entry forever.
|
|
3242
|
+
run = markOutcome.run;
|
|
3243
|
+
warnings.push(`finalizer '${finalizerName}' executed but its outcome may not have been recorded ` +
|
|
3244
|
+
`(${markOutcome.reason}) — it may re-execute at the next terminal edge`);
|
|
3245
|
+
break;
|
|
3246
|
+
}
|
|
3247
|
+
run = markOutcome.run;
|
|
3248
|
+
// result:'failed' is settled too (a recorded terminal outcome for this finalizer) — the loop
|
|
3249
|
+
// continues to the next rank regardless of markResult.
|
|
3250
|
+
}
|
|
3251
|
+
return { run, warnings };
|
|
3252
|
+
}
|
|
2588
3253
|
async function executeChainInternal(store, definition, options, depth, chainedSteps,
|
|
2589
3254
|
/**
|
|
2590
3255
|
* issue #197 PR-2 (chain-replacement disposition, accepted): a settled DEPTH-0 step (typically
|