@sensigo/realm 0.28.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/eligibility.d.ts +50 -0
- package/dist/engine/eligibility.d.ts.map +1 -1
- package/dist/engine/eligibility.js +72 -0
- package/dist/engine/eligibility.js.map +1 -1
- package/dist/engine/execution-loop.d.ts +10 -0
- package/dist/engine/execution-loop.d.ts.map +1 -1
- package/dist/engine/execution-loop.js +595 -228
- package/dist/engine/execution-loop.js.map +1 -1
- package/dist/engine/reclaim-step.d.ts +19 -2
- package/dist/engine/reclaim-step.d.ts.map +1 -1
- package/dist/engine/reclaim-step.js +41 -4
- package/dist/engine/reclaim-step.js.map +1 -1
- package/dist/engine/run-health.d.ts +49 -0
- package/dist/engine/run-health.d.ts.map +1 -0
- package/dist/engine/run-health.js +68 -0
- package/dist/engine/run-health.js.map +1 -0
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -3
- package/dist/index.js.map +1 -1
- package/dist/observability/failed-attempt-record.d.ts +12 -0
- package/dist/observability/failed-attempt-record.d.ts.map +1 -1
- package/dist/observability/failed-attempt-record.js +24 -4
- package/dist/observability/failed-attempt-record.js.map +1 -1
- package/dist/store/json-file-store.d.ts.map +1 -1
- package/dist/store/json-file-store.js +2 -0
- package/dist/store/json-file-store.js.map +1 -1
- package/dist/store/store-interface.d.ts +21 -6
- package/dist/store/store-interface.d.ts.map +1 -1
- package/dist/types/response-envelope.d.ts +20 -0
- package/dist/types/response-envelope.d.ts.map +1 -1
- package/dist/types/run-record.d.ts +42 -0
- package/dist/types/run-record.d.ts.map +1 -1
- package/dist/types/workflow-definition.d.ts +29 -1
- package/dist/types/workflow-definition.d.ts.map +1 -1
- package/dist/types/workflow-definition.js +1 -0
- package/dist/types/workflow-definition.js.map +1 -1
- package/dist/types/workflow-error.d.ts +1 -1
- package/dist/types/workflow-error.d.ts.map +1 -1
- package/dist/types/workflow-error.js.map +1 -1
- package/dist/workflow/diagnostics.d.ts +15 -13
- package/dist/workflow/diagnostics.d.ts.map +1 -1
- package/dist/workflow/diagnostics.js +13 -7
- package/dist/workflow/diagnostics.js.map +1 -1
- package/dist/workflow/yaml-loader.d.ts.map +1 -1
- package/dist/workflow/yaml-loader.js +203 -10
- package/dist/workflow/yaml-loader.js.map +1 -1
- package/package.json +1 -1
|
@@ -46,6 +46,16 @@ function withTimeout(dispatch, ms, stepName) {
|
|
|
46
46
|
}
|
|
47
47
|
/** Maximum nesting depth allowed in an input_map tree. */
|
|
48
48
|
const MAX_INPUT_MAP_DEPTH = 10;
|
|
49
|
+
/**
|
|
50
|
+
* Issue #220: default per-step threshold for bounded validation-rejection exhaustion — the
|
|
51
|
+
* multiple-of-(schemaRetries+1) alignment formula at k=2 (two full default `realm agent` repair
|
|
52
|
+
* drives, schemaRetries defaulting to 2 ⇒ 3 attempts/drive ⇒ termination lands at drive
|
|
53
|
+
* boundaries at defaults). Exported so `realm agent`'s drive-time coherence warn (run-agent.ts)
|
|
54
|
+
* and a per-step `validation_exhaustion.threshold` override (yaml-loader.ts) share ONE source of
|
|
55
|
+
* truth. Every countable agent step (see `countRejection` below) is auto-enrolled at this
|
|
56
|
+
* threshold — there is no reachable default-off posture in PR-1.
|
|
57
|
+
*/
|
|
58
|
+
export const DEFAULT_VALIDATION_EXHAUSTION_THRESHOLD = 6;
|
|
49
59
|
/**
|
|
50
60
|
* Resolves an input_map declaration into a concrete params object.
|
|
51
61
|
* Falls back to options.input when input_map is absent.
|
|
@@ -365,6 +375,34 @@ function mergeWarnings(traceWarnings, ...extraWarnings) {
|
|
|
365
375
|
return [];
|
|
366
376
|
return [...traceWarnings, ...defined];
|
|
367
377
|
}
|
|
378
|
+
/**
|
|
379
|
+
* Pure helper (issue #220 PR-2, D6): scans `sealDraft`'s evidence for entries stamped
|
|
380
|
+
* `diagnostics.settled_by_default === true` and, if any exist, returns the record with
|
|
381
|
+
* `defaulted_steps` set to their distinct step names (declaration order of first occurrence) —
|
|
382
|
+
* else returns the SAME record reference unchanged, so a run with no default-settle anywhere is
|
|
383
|
+
* byte-identical to pre-PR-2 behavior (the damage-rail this preserves). Pure, no I/O.
|
|
384
|
+
*
|
|
385
|
+
* Applied by WRAPPING the `buildFinalizedSeal` call inside the TERMINAL branch of each seal
|
|
386
|
+
* ternary — `buildFinalizedSeal` itself stays byte-untouched (a chokepoint insertion was
|
|
387
|
+
* considered and rejected in the design record: it reaches fail/abort seals too and would be a
|
|
388
|
+
* fragile two-touch above the damage-rail fast-path). Callers must pass ONLY a sealed record from
|
|
389
|
+
* the `'complete'` branch — never a non-terminal draft, and never a fail/abort seal — so
|
|
390
|
+
* `defaulted_steps` never leaks onto a persisted non-terminal record that a later FAIL seal
|
|
391
|
+
* inherits (the FM-5 residual this guards).
|
|
392
|
+
*/
|
|
393
|
+
function stampDefaultedSteps(sealDraft) {
|
|
394
|
+
const steps = [];
|
|
395
|
+
const seen = new Set();
|
|
396
|
+
for (const snap of sealDraft.evidence) {
|
|
397
|
+
if (snap.diagnostics?.settled_by_default === true && !seen.has(snap.step_id)) {
|
|
398
|
+
seen.add(snap.step_id);
|
|
399
|
+
steps.push(snap.step_id);
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
if (steps.length === 0)
|
|
403
|
+
return sealDraft;
|
|
404
|
+
return { ...sealDraft, defaulted_steps: steps };
|
|
405
|
+
}
|
|
368
406
|
/**
|
|
369
407
|
* Compensating un-claim (issue #207 PR-2, D3 §5): built from `pendingRun` — the record OUR OWN
|
|
370
408
|
* `claimStep` call returned, never a fresh get — removing the step from `in_progress_steps` AND
|
|
@@ -647,25 +685,148 @@ export async function executeStep(store, definition, options) {
|
|
|
647
685
|
// stripped input. effectiveOptions is identical to options when _debug was absent.
|
|
648
686
|
const effectiveOptions = { ...options, input: effectiveInput };
|
|
649
687
|
let inputTokenEstimate = Math.ceil(JSON.stringify(effectiveInput).length / 4);
|
|
688
|
+
// issue #220: bounded validation-rejection exhaustion — locals shared by countRejection below
|
|
689
|
+
// and by both its call sites (Step 2b/2c) and the trace enforce-gate further down. `exhaustion`
|
|
690
|
+
// stays null unless/until a counted rejection's JUST-persisted count reaches its threshold;
|
|
691
|
+
// once armed, EVERY subsequent validation return in this call YIELDS instead of returning.
|
|
692
|
+
let exhaustion = null;
|
|
693
|
+
let counted = false; // at-most-once arm per invocation
|
|
694
|
+
let persisted; // the count actually PERSISTED this invocation, if any
|
|
695
|
+
const countWarnings = [];
|
|
696
|
+
/**
|
|
697
|
+
* issue #220 (design record §2) — the single chokepoint every counted rejection passes through.
|
|
698
|
+
* Counts ONLY `{VALIDATION_INPUT_SCHEMA, VALIDATION_OUTPUT_SCHEMA}` on `execution: 'agent'`
|
|
699
|
+
* steps (CLOSED set: for agent steps, Step 2b/2c validates `options.input`, which is
|
|
700
|
+
* model-authored by construction — every counted rejection is model-attributable bytes.
|
|
701
|
+
* `VALIDATION_TRACE_SCHEMA` is EXCLUDED v1 — a WAL-merged trace may carry preserved foreign
|
|
702
|
+
* lines (#185/#197), so counting it would poison a step on someone else's bytes; the
|
|
703
|
+
* nonce-refusal class is pre-engine today [structurally thrown in the MCP wrapper before this
|
|
704
|
+
* function ever runs] — re-adjudicate this exclusion if that gate ever moves into core).
|
|
705
|
+
* Read-modify-write from the Step-1 `run` through `store.update()`'s CAS, with the write's
|
|
706
|
+
* return value DISCARDED — the envelope keeps building from the stale Step-1 `run`
|
|
707
|
+
* (bump-and-report; this is the #217 repair-gate contract's own ordering guarantee — see
|
|
708
|
+
* run-agent.ts's cross-ref comment at the repair gate). CAS failure retries ONCE on a fresh
|
|
709
|
+
* `get()` (the extension-identity precedent above), re-checking countability on the FRESH
|
|
710
|
+
* record [P-B1]: an unguarded retry could write onto a terminal/claimed/gate-waiting record,
|
|
711
|
+
* worst case CAS-failing a concurrent VALID submission's settle into a human-judged claim
|
|
712
|
+
* wedge. Any further failure — or a failed countability re-check — drops and swallows
|
|
713
|
+
* (undercount-safe); envelope delivery is unconditional regardless of what this function does.
|
|
714
|
+
*/
|
|
715
|
+
async function countRejection(err) {
|
|
716
|
+
if (counted)
|
|
717
|
+
return; // at-most-once per invocation
|
|
718
|
+
if (stepDef?.execution !== 'agent')
|
|
719
|
+
return; // non-agent steps are never counted
|
|
720
|
+
if (err.code !== 'VALIDATION_INPUT_SCHEMA' && err.code !== 'VALIDATION_OUTPUT_SCHEMA')
|
|
721
|
+
return;
|
|
722
|
+
counted = true;
|
|
723
|
+
const threshold = stepDef.validation_exhaustion?.threshold ?? DEFAULT_VALIDATION_EXHAUSTION_THRESHOLD;
|
|
724
|
+
const attempted = (run.validation_rejections?.[options.command] ?? 0) + 1;
|
|
725
|
+
try {
|
|
726
|
+
// FIRST CAS — expected version = the Step-1 read. Return value DISCARDED: the envelope
|
|
727
|
+
// built at the call site keeps using the stale `run`, never this write's result.
|
|
728
|
+
await store.update({
|
|
729
|
+
...run,
|
|
730
|
+
validation_rejections: {
|
|
731
|
+
...(run.validation_rejections ?? {}),
|
|
732
|
+
[options.command]: attempted,
|
|
733
|
+
},
|
|
734
|
+
});
|
|
735
|
+
persisted = attempted;
|
|
736
|
+
}
|
|
737
|
+
catch {
|
|
738
|
+
// CAS loser — retry ONCE on fresh state, WITH a countability re-check on the fresh record.
|
|
739
|
+
try {
|
|
740
|
+
const fresh = await store.get(options.runId);
|
|
741
|
+
if (!fresh.terminal_state &&
|
|
742
|
+
fresh.pending_gate === undefined &&
|
|
743
|
+
findEligibleSteps(definition, fresh).includes(options.command)) {
|
|
744
|
+
const freshN = (fresh.validation_rejections?.[options.command] ?? 0) + 1;
|
|
745
|
+
// CAS on fresh.version closes the re-check's own TOCTOU: a claim landing after the
|
|
746
|
+
// re-check above but before this write fails THIS write too (caught below).
|
|
747
|
+
await store.update({
|
|
748
|
+
...fresh,
|
|
749
|
+
validation_rejections: {
|
|
750
|
+
...(fresh.validation_rejections ?? {}),
|
|
751
|
+
[options.command]: freshN,
|
|
752
|
+
},
|
|
753
|
+
});
|
|
754
|
+
persisted = freshN;
|
|
755
|
+
}
|
|
756
|
+
else {
|
|
757
|
+
persisted = undefined; // drop-and-swallow (undercount-safe)
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
catch {
|
|
761
|
+
persisted = undefined; // double-CAS-failure swallow
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
// [design record §2, P-S4] fires on the drop path AND the double-failure path alike — nothing
|
|
765
|
+
// was persisted THIS invocation either way.
|
|
766
|
+
if (persisted === undefined) {
|
|
767
|
+
countWarnings.push(`rejection count not persisted — record remains at ${attempted - 1}`);
|
|
768
|
+
}
|
|
769
|
+
// [design record §1, softened per the final gate] a store round-tripping everything but not
|
|
770
|
+
// declaring this field yet is not called broken definitively — "MAY be unavailable", not "is".
|
|
771
|
+
if (!persistsField(store, 'validation_rejections')) {
|
|
772
|
+
countWarnings.push('rejection counting not declared durable on this store — exhaustion terminalization MAY ' +
|
|
773
|
+
'be unavailable');
|
|
774
|
+
}
|
|
775
|
+
// Countdown observable (Temporal-style) — `rejections` is the JUST-PERSISTED count when one
|
|
776
|
+
// exists, else the attempted (unpersisted) value, so a caller always sees SOME number.
|
|
777
|
+
err.details['rejections'] = persisted ?? attempted;
|
|
778
|
+
err.details['threshold'] = threshold;
|
|
779
|
+
if (persisted !== undefined && persisted >= threshold) {
|
|
780
|
+
exhaustion = new WorkflowError(`Step '${options.command}' exhausted its validation-rejection budget (${persisted}/${threshold})`, {
|
|
781
|
+
code: 'VALIDATION_EXHAUSTED',
|
|
782
|
+
category: 'VALIDATION',
|
|
783
|
+
// observationally inert on this path (Step-5 terminal translation supplies 'stop');
|
|
784
|
+
// kept for error-catalog coherence
|
|
785
|
+
agentAction: 'stop',
|
|
786
|
+
retryable: false,
|
|
787
|
+
details: {
|
|
788
|
+
step_id: options.command,
|
|
789
|
+
rejections: persisted,
|
|
790
|
+
threshold,
|
|
791
|
+
last_error: err.message,
|
|
792
|
+
last_ajv_errors: err.details['errors'],
|
|
793
|
+
},
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
}
|
|
650
797
|
// Step 2b: Validate input schema.
|
|
651
798
|
if (stepDef?.input_schema !== undefined) {
|
|
652
799
|
try {
|
|
653
800
|
validateInputSchema(effectiveInput, stepDef.input_schema, options.command);
|
|
654
801
|
}
|
|
655
802
|
catch (err) {
|
|
656
|
-
|
|
803
|
+
await countRejection(err);
|
|
804
|
+
if (exhaustion === null) {
|
|
805
|
+
return makeErrorEnvelope(options, run, err, definition, countWarnings.length > 0 ? countWarnings : undefined);
|
|
806
|
+
}
|
|
807
|
+
// FALL THROUGH — exhaustion armed; Step 2c below is gated on `exhaustion === null` (skipped
|
|
808
|
+
// whole), and every downstream gate yields toward terminalization instead of returning.
|
|
657
809
|
}
|
|
658
810
|
}
|
|
659
|
-
// Step 2c: Validate output schema (agent steps only).
|
|
811
|
+
// Step 2c: Validate output schema (agent steps only). issue #220: the WHOLE block is gated on
|
|
812
|
+
// `exhaustion === null` — once armed by Step 2b above, 2c does not run at all (no validation,
|
|
813
|
+
// no catch), matching W5's own conjunct set exactly [P-S1]: a both-schemas step must never
|
|
814
|
+
// double-count or report the wrong last_error.
|
|
660
815
|
// For agent steps dispatch is a pass-through, so options.input IS the agent's
|
|
661
816
|
// submitted output. Validating here (pre-claim) is equivalent to
|
|
662
817
|
// "post-generation, pre-commit" — the standard output guardrail position.
|
|
663
|
-
if (
|
|
818
|
+
if (exhaustion === null &&
|
|
819
|
+
stepDef?.execution === 'agent' &&
|
|
820
|
+
stepDef.output_schema !== undefined) {
|
|
664
821
|
try {
|
|
665
822
|
validateOutputSchema(effectiveInput, stepDef.output_schema, options.command);
|
|
666
823
|
}
|
|
667
824
|
catch (err) {
|
|
668
|
-
|
|
825
|
+
await countRejection(err);
|
|
826
|
+
if (exhaustion === null) {
|
|
827
|
+
return makeErrorEnvelope(options, run, err, definition, countWarnings.length > 0 ? countWarnings : undefined);
|
|
828
|
+
}
|
|
829
|
+
// FALL THROUGH — exhaustion armed.
|
|
669
830
|
}
|
|
670
831
|
}
|
|
671
832
|
// Step 2d: PRE-claim WAL read — issue #185 Fix 2: this read serves the enforce-gate ONLY.
|
|
@@ -677,6 +838,12 @@ export async function executeStep(store, definition, options) {
|
|
|
677
838
|
// #185 Finding 2). A rare line landing in exactly that window bypasses THIS enforce check —
|
|
678
839
|
// documented, accepted (see the post-claim block).
|
|
679
840
|
//
|
|
841
|
+
// issue #220 carve-out: the ABOVE "stays pre-claim so an invalid trace doesn't consume a claim"
|
|
842
|
+
// guarantee is for the COMMON case only — when validation exhaustion is already armed by an
|
|
843
|
+
// earlier Step 2b/2c rejection, the enforce-gate below deliberately YIELDS its return instead
|
|
844
|
+
// (still never deletes/consumes the WAL) so a persistently-invalid-trace agent under `enforce`
|
|
845
|
+
// can be terminalized rather than wedging forever purely on this unrelated gate.
|
|
846
|
+
//
|
|
680
847
|
// walEntries is declared at this outer scope because it is REASSIGNED to the post-claim read
|
|
681
848
|
// below and referenced at the captureEvidence call site further down this function.
|
|
682
849
|
const traceWarnings = [];
|
|
@@ -748,7 +915,24 @@ export async function executeStep(store, definition, options) {
|
|
|
748
915
|
}
|
|
749
916
|
catch (err) {
|
|
750
917
|
// On enforce rejection: do NOT delete the WAL — agent retries with WAL preserved.
|
|
751
|
-
|
|
918
|
+
if (exhaustion === null) {
|
|
919
|
+
return makeErrorEnvelope(options, run, err, definition);
|
|
920
|
+
}
|
|
921
|
+
// issue #220: exhaustion is already armed by an earlier Step 2b/2c rejection — the
|
|
922
|
+
// enforce-gate YIELDS its return (rather than returning) so a persistently-invalid
|
|
923
|
+
// trace under `enforce` can still be terminalized instead of wedging forever on this
|
|
924
|
+
// unrelated gate (the exact class #220 exists to kill). Re-run the WARN-shape pass
|
|
925
|
+
// (never enforce) purely so `preClaimSchemaResult`/the summary still reflect what
|
|
926
|
+
// actually happened to the trace; `validateTraceSchema('warn', ...)` never throws.
|
|
927
|
+
const warnResult = validateTraceSchema(preClaimNormalized.entries, stepDef.trace_schema, options.command, 'warn');
|
|
928
|
+
preClaimSchemaResult = {
|
|
929
|
+
schema_applied: true,
|
|
930
|
+
validation_mode: 'warn',
|
|
931
|
+
validation_errors: warnResult.errorCount,
|
|
932
|
+
};
|
|
933
|
+
if (warnResult.errorCount > 0) {
|
|
934
|
+
traceWarnings.push(warnResult.warning);
|
|
935
|
+
}
|
|
752
936
|
}
|
|
753
937
|
}
|
|
754
938
|
else {
|
|
@@ -1057,252 +1241,382 @@ export async function executeStep(store, definition, options) {
|
|
|
1057
1241
|
const rateLimiterRegistry = options.registry ?? new ExtensionRegistry();
|
|
1058
1242
|
let output = {};
|
|
1059
1243
|
let dispatchError = null;
|
|
1244
|
+
// issue #220 PR-2: true iff THIS invocation settles the step via its declared default_output
|
|
1245
|
+
// substitution (exhaustion armed above AND the step opted into mode: 'default') — computed once,
|
|
1246
|
+
// function-scoped (not inside the `if (bypassDispatch)` block below, which closes long before
|
|
1247
|
+
// D5's Step-6 envelope build reads this local) so it survives to every downstream read site.
|
|
1248
|
+
const settledByDefault = exhaustion !== null && stepDef?.validation_exhaustion?.mode === 'default';
|
|
1060
1249
|
let attemptsUsed = 0;
|
|
1061
1250
|
const allEvidence = [];
|
|
1062
1251
|
let currentWarn;
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1252
|
+
// issue #220: once exhaustion is armed by an earlier Step 2b/2c/enforce-gate rejection, bypass
|
|
1253
|
+
// the ENTIRE dispatch loop below AND the retry-wrap block that follows it — claimStep's own
|
|
1254
|
+
// under-lock re-check (STATE_STEP_NOT_ELIGIBLE) is the safety net that cleanly aborts
|
|
1255
|
+
// terminalization if a concurrent valid submission or abandon beat this invocation to the claim
|
|
1256
|
+
// (see the claim's own catch above). Without this bypass, a hand-built max_attempts:0 definition
|
|
1257
|
+
// would wrap VALIDATION_EXHAUSTED into STEP_RETRY_EXHAUSTED, destroying the discriminator.
|
|
1258
|
+
const bypassDispatch = exhaustion !== null;
|
|
1259
|
+
if (!bypassDispatch) {
|
|
1260
|
+
for (let attemptNum = 1; attemptNum <= maxAttempts; attemptNum++) {
|
|
1261
|
+
// SITE (a) — issue #140, loop-top, BEFORE attemptsUsed is assigned: attempt 1 ALWAYS
|
|
1262
|
+
// proceeds regardless of capMs (the `attemptNum > 1` conjunct) — this guard exists solely for
|
|
1263
|
+
// the clock-anomaly window between `capStart` above and here (a suspend/resume or NTP forward
|
|
1264
|
+
// jump), never to gate the very first attempt.
|
|
1265
|
+
if (capMs !== undefined && attemptNum > 1 && remainingMs() <= 0) {
|
|
1266
|
+
capExhausted = true;
|
|
1267
|
+
break;
|
|
1268
|
+
}
|
|
1269
|
+
attemptsUsed = attemptNum;
|
|
1270
|
+
const startedAt = new Date();
|
|
1271
|
+
let attemptOutput = {};
|
|
1272
|
+
let attemptError = null;
|
|
1273
|
+
let resolvedParams;
|
|
1274
|
+
// Per-attempt effective timeout (issue #140): uniform full-clip to whatever cap budget
|
|
1275
|
+
// remains. Clip floor `max(0, remainingMs())` ensures a clock anomaly (see SITE (a) above)
|
|
1276
|
+
// never passes a negative ms to withTimeout. capMs undefined ⇒ effectiveMs === timeoutMs,
|
|
1277
|
+
// byte-identical to pre-#140 behavior (every non-retry-configured, or unopted-uncapped-by-
|
|
1278
|
+
// total_timeout_seconds-being-absent-pre-amendment, auto step). `clippedToMs` records the
|
|
1279
|
+
// per-attempt evidence value ONLY when the cap actually reduced the bound below the step's
|
|
1280
|
+
// own declared/default timeout — never on an uncapped or not-yet-biting attempt.
|
|
1281
|
+
const effectiveMs = timeoutMs !== undefined
|
|
1282
|
+
? capMs !== undefined
|
|
1283
|
+
? Math.min(timeoutMs, Math.max(0, remainingMs()))
|
|
1284
|
+
: timeoutMs
|
|
1285
|
+
: undefined;
|
|
1286
|
+
const clippedToMs = capMs !== undefined && effectiveMs !== undefined && effectiveMs < timeoutMs
|
|
1287
|
+
? effectiveMs
|
|
1288
|
+
: undefined;
|
|
1289
|
+
try {
|
|
1290
|
+
const makeCall = (signal) => {
|
|
1291
|
+
if (stepDef?.execution === 'auto' && stepDef.uses_service !== undefined) {
|
|
1292
|
+
return callAdapter(stepDef, definition, effectiveOptions, pendingRun, rateLimiterRegistry, signal);
|
|
1293
|
+
}
|
|
1294
|
+
else if (stepDef?.execution === 'auto' && stepDef.handler !== undefined) {
|
|
1295
|
+
return callHandler(stepDef, effectiveOptions, pendingRun, evidenceByStep, signal).then((result) => {
|
|
1296
|
+
if (result.kind === 'abort') {
|
|
1297
|
+
return {
|
|
1298
|
+
output: {},
|
|
1299
|
+
resolvedParams: undefined,
|
|
1300
|
+
handlerAbort: { message: result.message },
|
|
1301
|
+
};
|
|
1302
|
+
}
|
|
1303
|
+
if (result.kind === 'warn') {
|
|
1304
|
+
return {
|
|
1305
|
+
output: result.output,
|
|
1306
|
+
resolvedParams: result.resolvedParams,
|
|
1307
|
+
handlerWarn: result.message,
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
return { output: result.output, resolvedParams: result.resolvedParams };
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
else {
|
|
1314
|
+
return options
|
|
1315
|
+
.dispatcher(options.command, effectiveInput, pendingRun, signal)
|
|
1316
|
+
.then((result) => ({ output: result, resolvedParams: undefined }));
|
|
1317
|
+
}
|
|
1318
|
+
};
|
|
1319
|
+
const callResult = effectiveMs !== undefined
|
|
1320
|
+
? await withTimeout((signal) => makeCall(signal), effectiveMs, options.command)
|
|
1321
|
+
: await makeCall();
|
|
1322
|
+
// Handle graceful abort from a handler returning { abort: { message } }.
|
|
1323
|
+
if (callResult.handlerAbort !== undefined) {
|
|
1324
|
+
const abortMessage = callResult.handlerAbort.message;
|
|
1325
|
+
const now = new Date();
|
|
1326
|
+
const abortEvidence = {
|
|
1327
|
+
...captureEvidence({
|
|
1328
|
+
stepId: options.command,
|
|
1329
|
+
startedAt: now,
|
|
1330
|
+
completedAt: now,
|
|
1331
|
+
input: effectiveInput,
|
|
1332
|
+
output: { aborted: true, abort_message: abortMessage },
|
|
1333
|
+
error: abortMessage,
|
|
1334
|
+
...(debugOutput !== undefined ? { debugOutput } : {}),
|
|
1335
|
+
}),
|
|
1336
|
+
status: 'skipped',
|
|
1337
|
+
};
|
|
1338
|
+
const withHandlerSkipped = {
|
|
1339
|
+
...pendingRun,
|
|
1340
|
+
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
1341
|
+
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
1342
|
+
claims: omitClaim(pendingRun.claims, options.command),
|
|
1343
|
+
evidence: [...pendingRun.evidence, abortEvidence],
|
|
1344
|
+
skipped_steps: [...pendingRun.skipped_steps, options.command],
|
|
1345
|
+
};
|
|
1346
|
+
// #111: merge is load-bearing — it preserves any cascade details propagateSkips derives
|
|
1347
|
+
// for OTHER now-unreachable steps alongside this step's own handler_abort tag.
|
|
1348
|
+
const handlerAbortPropagated = propagateSkips(withHandlerSkipped, definition);
|
|
1349
|
+
const withAllSkipped = {
|
|
1350
|
+
...withHandlerSkipped,
|
|
1351
|
+
skipped_steps: handlerAbortPropagated.skipped,
|
|
1352
|
+
skip_details: {
|
|
1353
|
+
...handlerAbortPropagated.details,
|
|
1354
|
+
[options.command]: { kind: 'handler_abort' },
|
|
1355
|
+
},
|
|
1356
|
+
};
|
|
1357
|
+
const abortDraft = {
|
|
1358
|
+
...withAllSkipped,
|
|
1359
|
+
terminal_state: true,
|
|
1360
|
+
terminal_reason: `Handler '${options.command}' aborted the run: ${abortMessage}`,
|
|
1361
|
+
aborted_at: {
|
|
1362
|
+
step_id: options.command,
|
|
1363
|
+
abort_message: abortMessage,
|
|
1364
|
+
},
|
|
1365
|
+
};
|
|
1366
|
+
// NEW: a handler-abort now drains the abort/always finalizers before sealing
|
|
1367
|
+
// (previously it sealed-and-returned immediately). Single seal write below.
|
|
1368
|
+
const abortedRun = await buildFinalizedSeal(definition, abortDraft, 'abort', options.registry);
|
|
1369
|
+
let persistedAbortRun;
|
|
1370
|
+
try {
|
|
1371
|
+
persistedAbortRun = await store.update(abortedRun);
|
|
1372
|
+
}
|
|
1373
|
+
catch {
|
|
1374
|
+
// Persist failed — return the in-memory run version.
|
|
1375
|
+
}
|
|
1376
|
+
return {
|
|
1377
|
+
command: options.command,
|
|
1378
|
+
run_id: options.runId,
|
|
1379
|
+
run_version: (persistedAbortRun ?? abortedRun).version,
|
|
1380
|
+
status: 'ok',
|
|
1381
|
+
data: {},
|
|
1382
|
+
evidence: [abortEvidence],
|
|
1383
|
+
// Issue #140: was hardcoded `[]` — now threads traceWarnings (e.g. the programmatic
|
|
1384
|
+
// on_timeout/idempotent gate advisory above) so it survives this settle path too.
|
|
1385
|
+
warnings: mergeWarnings(traceWarnings),
|
|
1386
|
+
errors: [],
|
|
1387
|
+
context_hint: `Handler step '${options.command}' aborted the run: ${abortMessage}`,
|
|
1388
|
+
run_phase: (persistedAbortRun ?? abortedRun).run_phase,
|
|
1389
|
+
next_actions: [],
|
|
1390
|
+
};
|
|
1096
1391
|
}
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
resolvedParams: undefined,
|
|
1103
|
-
handlerAbort: { message: result.message },
|
|
1104
|
-
};
|
|
1105
|
-
}
|
|
1106
|
-
if (result.kind === 'warn') {
|
|
1107
|
-
return {
|
|
1108
|
-
output: result.output,
|
|
1109
|
-
resolvedParams: result.resolvedParams,
|
|
1110
|
-
handlerWarn: result.message,
|
|
1111
|
-
};
|
|
1112
|
-
}
|
|
1113
|
-
return { output: result.output, resolvedParams: result.resolvedParams };
|
|
1114
|
-
});
|
|
1392
|
+
attemptOutput = callResult.output;
|
|
1393
|
+
resolvedParams = callResult.resolvedParams;
|
|
1394
|
+
currentWarn = callResult.handlerWarn;
|
|
1395
|
+
if (resolvedParams !== undefined) {
|
|
1396
|
+
inputTokenEstimate = Math.ceil(JSON.stringify(resolvedParams).length / 4);
|
|
1115
1397
|
}
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1398
|
+
}
|
|
1399
|
+
catch (err) {
|
|
1400
|
+
if (err instanceof WorkflowError) {
|
|
1401
|
+
attemptError = err;
|
|
1120
1402
|
}
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
const now = new Date();
|
|
1129
|
-
const abortEvidence = {
|
|
1130
|
-
...captureEvidence({
|
|
1403
|
+
else {
|
|
1404
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1405
|
+
attemptError = new WorkflowError(`Dispatcher failed: ${message}`, {
|
|
1406
|
+
code: 'ENGINE_INTERNAL',
|
|
1407
|
+
category: 'ENGINE',
|
|
1408
|
+
agentAction: 'stop',
|
|
1409
|
+
retryable: false,
|
|
1131
1410
|
stepId: options.command,
|
|
1132
|
-
|
|
1133
|
-
completedAt: now,
|
|
1134
|
-
input: effectiveInput,
|
|
1135
|
-
output: { aborted: true, abort_message: abortMessage },
|
|
1136
|
-
error: abortMessage,
|
|
1137
|
-
...(debugOutput !== undefined ? { debugOutput } : {}),
|
|
1138
|
-
}),
|
|
1139
|
-
status: 'skipped',
|
|
1140
|
-
};
|
|
1141
|
-
const withHandlerSkipped = {
|
|
1142
|
-
...pendingRun,
|
|
1143
|
-
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
1144
|
-
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
1145
|
-
claims: omitClaim(pendingRun.claims, options.command),
|
|
1146
|
-
evidence: [...pendingRun.evidence, abortEvidence],
|
|
1147
|
-
skipped_steps: [...pendingRun.skipped_steps, options.command],
|
|
1148
|
-
};
|
|
1149
|
-
// #111: merge is load-bearing — it preserves any cascade details propagateSkips derives
|
|
1150
|
-
// for OTHER now-unreachable steps alongside this step's own handler_abort tag.
|
|
1151
|
-
const handlerAbortPropagated = propagateSkips(withHandlerSkipped, definition);
|
|
1152
|
-
const withAllSkipped = {
|
|
1153
|
-
...withHandlerSkipped,
|
|
1154
|
-
skipped_steps: handlerAbortPropagated.skipped,
|
|
1155
|
-
skip_details: {
|
|
1156
|
-
...handlerAbortPropagated.details,
|
|
1157
|
-
[options.command]: { kind: 'handler_abort' },
|
|
1158
|
-
},
|
|
1159
|
-
};
|
|
1160
|
-
const abortDraft = {
|
|
1161
|
-
...withAllSkipped,
|
|
1162
|
-
terminal_state: true,
|
|
1163
|
-
terminal_reason: `Handler '${options.command}' aborted the run: ${abortMessage}`,
|
|
1164
|
-
aborted_at: {
|
|
1165
|
-
step_id: options.command,
|
|
1166
|
-
abort_message: abortMessage,
|
|
1167
|
-
},
|
|
1168
|
-
};
|
|
1169
|
-
// NEW: a handler-abort now drains the abort/always finalizers before sealing
|
|
1170
|
-
// (previously it sealed-and-returned immediately). Single seal write below.
|
|
1171
|
-
const abortedRun = await buildFinalizedSeal(definition, abortDraft, 'abort', options.registry);
|
|
1172
|
-
let persistedAbortRun;
|
|
1173
|
-
try {
|
|
1174
|
-
persistedAbortRun = await store.update(abortedRun);
|
|
1175
|
-
}
|
|
1176
|
-
catch {
|
|
1177
|
-
// Persist failed — return the in-memory run version.
|
|
1411
|
+
});
|
|
1178
1412
|
}
|
|
1179
|
-
return {
|
|
1180
|
-
command: options.command,
|
|
1181
|
-
run_id: options.runId,
|
|
1182
|
-
run_version: (persistedAbortRun ?? abortedRun).version,
|
|
1183
|
-
status: 'ok',
|
|
1184
|
-
data: {},
|
|
1185
|
-
evidence: [abortEvidence],
|
|
1186
|
-
// Issue #140: was hardcoded `[]` — now threads traceWarnings (e.g. the programmatic
|
|
1187
|
-
// on_timeout/idempotent gate advisory above) so it survives this settle path too.
|
|
1188
|
-
warnings: mergeWarnings(traceWarnings),
|
|
1189
|
-
errors: [],
|
|
1190
|
-
context_hint: `Handler step '${options.command}' aborted the run: ${abortMessage}`,
|
|
1191
|
-
run_phase: (persistedAbortRun ?? abortedRun).run_phase,
|
|
1192
|
-
next_actions: [],
|
|
1193
|
-
};
|
|
1194
1413
|
}
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1414
|
+
const completedAt = new Date();
|
|
1415
|
+
const profile = stepDef?.agent_profile;
|
|
1416
|
+
const profileData = profile !== undefined ? definition.resolved_profiles?.[profile] : undefined;
|
|
1417
|
+
const baseSnap = captureEvidence({
|
|
1418
|
+
stepId: options.command,
|
|
1419
|
+
startedAt,
|
|
1420
|
+
completedAt,
|
|
1421
|
+
input: effectiveInput,
|
|
1422
|
+
output: attemptOutput,
|
|
1423
|
+
...(attemptError !== null ? { error: attemptError.message } : {}),
|
|
1424
|
+
diagnostics: {
|
|
1425
|
+
input_token_estimate: inputTokenEstimate,
|
|
1426
|
+
precondition_trace: preconditionTrace,
|
|
1427
|
+
// issue #220: success-settle stamp — a free diagnostic proving this step needed N prior
|
|
1428
|
+
// rejections before finally succeeding ("succeeded after N rejections"). Only stamped on
|
|
1429
|
+
// a SUCCESS settle. `run` here is the Step-1 read, so this reflects rejections accrued
|
|
1430
|
+
// in PRIOR invocations only — a rejection in THIS invocation never reaches this call site
|
|
1431
|
+
// (countRejection only runs from the Step 2b/2c catch, which always either returns or
|
|
1432
|
+
// falls through toward terminalization, never toward dispatch).
|
|
1433
|
+
...(attemptError === null && (run.validation_rejections?.[options.command] ?? 0) > 0
|
|
1434
|
+
? { validation_rejections: run.validation_rejections[options.command] }
|
|
1435
|
+
: {}),
|
|
1436
|
+
},
|
|
1437
|
+
...(profileData !== undefined
|
|
1438
|
+
? { agentProfile: profile, agentProfileHash: profileData.content_hash }
|
|
1439
|
+
: {}),
|
|
1440
|
+
...(resolvedParams !== undefined ? { resolvedParams } : {}),
|
|
1441
|
+
...(currentWarn !== undefined ? { warn: currentWarn } : {}),
|
|
1442
|
+
...(debugOutput !== undefined ? { debugOutput } : {}),
|
|
1443
|
+
...(options.stepMeta?.toolCalls !== undefined
|
|
1444
|
+
? { toolCalls: options.stepMeta.toolCalls }
|
|
1445
|
+
: {}),
|
|
1446
|
+
// issue #140: PER-ATTEMPT value (may be smaller than the outer effectiveTimeoutSeconds once
|
|
1447
|
+
// the cap starts clipping) — byte-identical to the pre-#140 outer value whenever capMs is
|
|
1448
|
+
// undefined or hasn't bitten yet.
|
|
1449
|
+
...(effectiveMs !== undefined ? { effectiveTimeoutSeconds: effectiveMs / 1000 } : {}),
|
|
1450
|
+
...(clippedToMs !== undefined ? { clippedToMs } : {}),
|
|
1451
|
+
// Gate trace to agent steps only — drop silently for auto/adapter/handler steps.
|
|
1452
|
+
// When pre-normalized (WAL merge + schema validation ran), pass the pre-normalized
|
|
1453
|
+
// result to avoid double normalization. Also handle WAL-only case (options.trace may
|
|
1454
|
+
// be undefined while walEntries contributed entries via preNormalizedTrace).
|
|
1455
|
+
...(stepDef?.execution === 'agent' && (options.trace !== undefined || walEntries.length > 0)
|
|
1456
|
+
? preNormalizedTrace !== undefined
|
|
1457
|
+
? { normalizedTrace: preNormalizedTrace }
|
|
1458
|
+
: { trace: options.trace ?? [] }
|
|
1459
|
+
: {}),
|
|
1460
|
+
});
|
|
1461
|
+
const snap = retryConfig !== undefined ? { ...baseSnap, attempt: attemptNum } : baseSnap;
|
|
1462
|
+
allEvidence.push(snap);
|
|
1463
|
+
if (attemptError === null) {
|
|
1464
|
+
output = attemptOutput;
|
|
1465
|
+
dispatchError = null;
|
|
1466
|
+
break;
|
|
1200
1467
|
}
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1468
|
+
dispatchError = attemptError;
|
|
1469
|
+
// SITE (b) — issue #140, post-attempt, AFTER dispatchError is set, BEFORE willRetry: the
|
|
1470
|
+
// primary capExhausted setter (site (a) above only catches the loop-top clock-anomaly window;
|
|
1471
|
+
// site (c) below re-checks once more right before an actual sleep). Fires on ANY dispatch
|
|
1472
|
+
// error once the cap is spent, not just STEP_TIMEOUT — the cap bounds the step's total
|
|
1473
|
+
// budget, regardless of which error exhausted it.
|
|
1474
|
+
if (capMs !== undefined && remainingMs() <= 0) {
|
|
1475
|
+
capExhausted = true;
|
|
1476
|
+
}
|
|
1477
|
+
const willRetry = (retryConfig !== undefined && attemptError.retryable && attemptNum < maxAttempts) ||
|
|
1478
|
+
// issue #140 (AMENDED): a STEP_TIMEOUT may ALSO retry in place when the step opted in via
|
|
1479
|
+
// `retry.on_timeout: true` AND attested `idempotent: true` (the concurrency-safety gate).
|
|
1480
|
+
// ALL SIX conjuncts are required; `capMs !== undefined` enforces opted⇒capped
|
|
1481
|
+
// structurally — this disjunct is inert off the enforced auto class even for a hand-built
|
|
1482
|
+
// definition bypassing the loader's E1 gate on a non-auto step, since capMs is undefined
|
|
1483
|
+
// there (shouldEnforceTimeout false ⇒ enforceTimeout false ⇒ capMs undefined) — see R11 in
|
|
1484
|
+
// the design record.
|
|
1485
|
+
(attemptError.code === 'STEP_TIMEOUT' &&
|
|
1486
|
+
capMs !== undefined &&
|
|
1487
|
+
retryConfig?.on_timeout === true &&
|
|
1488
|
+
stepDef.idempotent === true &&
|
|
1489
|
+
!capExhausted &&
|
|
1490
|
+
attemptNum < maxAttempts);
|
|
1491
|
+
if (willRetry) {
|
|
1492
|
+
const baseBackoff = computeBackoff(retryConfig, attemptNum);
|
|
1493
|
+
const retryAfterMs = attemptError instanceof WorkflowError && attemptError.retry_after !== undefined
|
|
1494
|
+
? attemptError.retry_after * 1000
|
|
1495
|
+
: 0;
|
|
1496
|
+
const waitMs = Math.max(baseBackoff, retryAfterMs);
|
|
1497
|
+
// SITE (c) — issue #140, sleep guard, BEFORE every backoff/retry_after sleep (`>=`: an
|
|
1498
|
+
// exact-fit sleep is doomed too — never sleep into a wall). dispatchError already holds the
|
|
1499
|
+
// ACTUAL last error (e.g. a 429 with retry_after in its details); the post-loop wrap gate
|
|
1500
|
+
// below decides whether/how to wrap it — this site only decides whether to sleep at all.
|
|
1501
|
+
if (capMs !== undefined && sleepWouldExceedCap(Date.now() - capStart, waitMs, capMs)) {
|
|
1502
|
+
capExhausted = true;
|
|
1503
|
+
break;
|
|
1504
|
+
}
|
|
1505
|
+
await delayMs(waitMs);
|
|
1205
1506
|
}
|
|
1206
1507
|
else {
|
|
1207
|
-
|
|
1208
|
-
attemptError = new WorkflowError(`Dispatcher failed: ${message}`, {
|
|
1209
|
-
code: 'ENGINE_INTERNAL',
|
|
1210
|
-
category: 'ENGINE',
|
|
1211
|
-
agentAction: 'stop',
|
|
1212
|
-
retryable: false,
|
|
1213
|
-
stepId: options.command,
|
|
1214
|
-
});
|
|
1508
|
+
break;
|
|
1215
1509
|
}
|
|
1216
1510
|
}
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1511
|
+
} // end issue #220 `if (!bypassDispatch)` — the dispatch loop
|
|
1512
|
+
if (bypassDispatch && settledByDefault) {
|
|
1513
|
+
// issue #220 PR-2 (D4): declared fail-open. The step opted into `validation_exhaustion.mode:
|
|
1514
|
+
// 'default'` and its schema-rejection budget is exhausted — SETTLE the step SUCCESSFULLY with
|
|
1515
|
+
// the declared `default_output` instead of terminalizing. `dispatchError` stays `null` here
|
|
1516
|
+
// (deliberately NOT set) so every existing downstream success path runs UNMODIFIED: Step 5
|
|
1517
|
+
// (dispatch-failure handling) is skipped, Step 5b's gate fires for `human_confirmed` steps on
|
|
1518
|
+
// the default_output preview (pin z falls out of this structurally — D7), and Step 6's
|
|
1519
|
+
// complete-settle records the step in `completed_steps`. Step 6 does NOT read the `output`
|
|
1520
|
+
// local at all — the step's durable output travels via the EVIDENCE SNAPSHOT's
|
|
1521
|
+
// `output_summary` (what `buildEvidenceByStep`/eligibility read for downstream steps) — so
|
|
1522
|
+
// this branch does exactly two things and no more: (1) set `output` for the envelope/gate
|
|
1523
|
+
// preview; (2) push ONE synthesized SUCCESS evidence snapshot mirroring the dispatch-loop's
|
|
1524
|
+
// own success `captureEvidence` call (the same one PR-1's FAILURE snapshot above was modeled
|
|
1525
|
+
// on, for the success shape instead).
|
|
1526
|
+
const defaultOutput = stepDef.validation_exhaustion.default_output;
|
|
1527
|
+
output = defaultOutput;
|
|
1528
|
+
const settledAt = new Date();
|
|
1529
|
+
const defaultProfile = stepDef?.agent_profile;
|
|
1530
|
+
const defaultProfileData = defaultProfile !== undefined ? definition.resolved_profiles?.[defaultProfile] : undefined;
|
|
1531
|
+
const defaultSnap = captureEvidence({
|
|
1221
1532
|
stepId: options.command,
|
|
1222
|
-
startedAt,
|
|
1223
|
-
completedAt,
|
|
1533
|
+
startedAt: settledAt,
|
|
1534
|
+
completedAt: settledAt,
|
|
1224
1535
|
input: effectiveInput,
|
|
1225
|
-
output:
|
|
1226
|
-
...(attemptError !== null ? { error: attemptError.message } : {}),
|
|
1536
|
+
output: defaultOutput,
|
|
1227
1537
|
diagnostics: {
|
|
1228
1538
|
input_token_estimate: inputTokenEstimate,
|
|
1229
1539
|
precondition_trace: preconditionTrace,
|
|
1540
|
+
settled_by_default: true,
|
|
1541
|
+
validation_rejections: exhaustion.details['rejections'],
|
|
1230
1542
|
},
|
|
1231
|
-
...(
|
|
1232
|
-
? { agentProfile:
|
|
1543
|
+
...(defaultProfileData !== undefined
|
|
1544
|
+
? { agentProfile: defaultProfile, agentProfileHash: defaultProfileData.content_hash }
|
|
1545
|
+
: {}),
|
|
1546
|
+
...(options.stepMeta?.toolCalls !== undefined
|
|
1547
|
+
? { toolCalls: options.stepMeta.toolCalls }
|
|
1233
1548
|
: {}),
|
|
1234
|
-
...(resolvedParams !== undefined ? { resolvedParams } : {}),
|
|
1235
|
-
...(currentWarn !== undefined ? { warn: currentWarn } : {}),
|
|
1236
1549
|
...(debugOutput !== undefined ? { debugOutput } : {}),
|
|
1550
|
+
// Gate trace to agent steps only — drop silently for auto/adapter/handler steps. Mirrors
|
|
1551
|
+
// the real dispatch-loop capture's own trace-spread conjunct exactly.
|
|
1552
|
+
...(stepDef?.execution === 'agent' && (options.trace !== undefined || walEntries.length > 0)
|
|
1553
|
+
? preNormalizedTrace !== undefined
|
|
1554
|
+
? { normalizedTrace: preNormalizedTrace }
|
|
1555
|
+
: { trace: options.trace ?? [] }
|
|
1556
|
+
: {}),
|
|
1557
|
+
});
|
|
1558
|
+
allEvidence.push(defaultSnap);
|
|
1559
|
+
// Human-readable disclosure (record §4 — the fourth default-settle disclosure element the
|
|
1560
|
+
// record enumerates) + the store-honesty advisory (§5c nuance 1: `countWarnings` are normally
|
|
1561
|
+
// delivered ONLY on counted-rejection RETURN envelopes and DROPPED on the fall-through
|
|
1562
|
+
// terminalization path — deliberately re-threaded here, for the default-settle success
|
|
1563
|
+
// envelope only, so an undeclared-but-persisting store's advisory is not silently lost).
|
|
1564
|
+
traceWarnings.push(`Step '${options.command}' settled with its declared default_output after ` +
|
|
1565
|
+
`${exhaustion.details['rejections']} schema rejection(s) ` +
|
|
1566
|
+
`(validation_exhaustion.mode: 'default')`);
|
|
1567
|
+
traceWarnings.push(...countWarnings);
|
|
1568
|
+
}
|
|
1569
|
+
else if (bypassDispatch) {
|
|
1570
|
+
// issue #220: terminalize. `dispatchError` is PRE-SET to the minted VALIDATION_EXHAUSTED error
|
|
1571
|
+
// HERE, BEFORE the retry-wrap block below — that block's own `!bypassDispatch` guard is what
|
|
1572
|
+
// then keeps it from being wrapped: without dispatchError being non-null at this exact point,
|
|
1573
|
+
// a hand-built `max_attempts: 0` definition's retry-wrap condition
|
|
1574
|
+
// (`attemptsUsed(0) === maxAttempts(0)`) would trivially hold and wrap the terminal code into
|
|
1575
|
+
// STEP_RETRY_EXHAUSTED, robbing Step 5 of the discriminator it needs. ONE synthesized evidence
|
|
1576
|
+
// snapshot mirrors the real dispatch-loop capture at its own `captureEvidence` call site
|
|
1577
|
+
// VERBATIM — including the normalizedTrace/trace spread — so the WAL delete at Step 5 never
|
|
1578
|
+
// destroys adopted post-claim lines unrecorded (issue #185 Finding 2 would otherwise be
|
|
1579
|
+
// reintroduced inside this feature).
|
|
1580
|
+
dispatchError = exhaustion;
|
|
1581
|
+
const exhaustedAt = new Date();
|
|
1582
|
+
// issue #220 correction (deliverable 1 — snapshot completeness): these two derivation lines
|
|
1583
|
+
// replicate the dispatch-loop's own `profile`/`profileData` consts VERBATIM (that block's
|
|
1584
|
+
// own locals are out of scope here, but `stepDef`/`definition` — the two inputs they're
|
|
1585
|
+
// derived from — are both still in scope at this bypass block).
|
|
1586
|
+
const exhaustedProfile = stepDef?.agent_profile;
|
|
1587
|
+
const exhaustedProfileData = exhaustedProfile !== undefined ? definition.resolved_profiles?.[exhaustedProfile] : undefined;
|
|
1588
|
+
const exhaustedSnap = captureEvidence({
|
|
1589
|
+
stepId: options.command,
|
|
1590
|
+
startedAt: exhaustedAt,
|
|
1591
|
+
completedAt: exhaustedAt,
|
|
1592
|
+
input: effectiveInput,
|
|
1593
|
+
output: {},
|
|
1594
|
+
error: exhaustion.message,
|
|
1595
|
+
diagnostics: {
|
|
1596
|
+
input_token_estimate: inputTokenEstimate,
|
|
1597
|
+
precondition_trace: preconditionTrace,
|
|
1598
|
+
validation_rejections: exhaustion.details['rejections'],
|
|
1599
|
+
},
|
|
1600
|
+
...(exhaustedProfileData !== undefined
|
|
1601
|
+
? { agentProfile: exhaustedProfile, agentProfileHash: exhaustedProfileData.content_hash }
|
|
1602
|
+
: {}),
|
|
1237
1603
|
...(options.stepMeta?.toolCalls !== undefined
|
|
1238
1604
|
? { toolCalls: options.stepMeta.toolCalls }
|
|
1239
1605
|
: {}),
|
|
1240
|
-
|
|
1241
|
-
//
|
|
1242
|
-
//
|
|
1243
|
-
|
|
1244
|
-
...(clippedToMs !== undefined ? { clippedToMs } : {}),
|
|
1245
|
-
// Gate trace to agent steps only — drop silently for auto/adapter/handler steps.
|
|
1246
|
-
// When pre-normalized (WAL merge + schema validation ran), pass the pre-normalized
|
|
1247
|
-
// result to avoid double normalization. Also handle WAL-only case (options.trace may
|
|
1248
|
-
// be undefined while walEntries contributed entries via preNormalizedTrace).
|
|
1606
|
+
...(debugOutput !== undefined ? { debugOutput } : {}),
|
|
1607
|
+
// Gate trace to agent steps only — drop silently for auto/adapter/handler steps. Mirrors
|
|
1608
|
+
// the real dispatch-loop capture's own trace-spread conjunct exactly (execution-loop.ts's
|
|
1609
|
+
// per-attempt captureEvidence call, further above).
|
|
1249
1610
|
...(stepDef?.execution === 'agent' && (options.trace !== undefined || walEntries.length > 0)
|
|
1250
1611
|
? preNormalizedTrace !== undefined
|
|
1251
1612
|
? { normalizedTrace: preNormalizedTrace }
|
|
1252
1613
|
: { trace: options.trace ?? [] }
|
|
1253
1614
|
: {}),
|
|
1254
1615
|
});
|
|
1255
|
-
|
|
1256
|
-
allEvidence.push(snap);
|
|
1257
|
-
if (attemptError === null) {
|
|
1258
|
-
output = attemptOutput;
|
|
1259
|
-
dispatchError = null;
|
|
1260
|
-
break;
|
|
1261
|
-
}
|
|
1262
|
-
dispatchError = attemptError;
|
|
1263
|
-
// SITE (b) — issue #140, post-attempt, AFTER dispatchError is set, BEFORE willRetry: the
|
|
1264
|
-
// primary capExhausted setter (site (a) above only catches the loop-top clock-anomaly window;
|
|
1265
|
-
// site (c) below re-checks once more right before an actual sleep). Fires on ANY dispatch
|
|
1266
|
-
// error once the cap is spent, not just STEP_TIMEOUT — the cap bounds the step's total
|
|
1267
|
-
// budget, regardless of which error exhausted it.
|
|
1268
|
-
if (capMs !== undefined && remainingMs() <= 0) {
|
|
1269
|
-
capExhausted = true;
|
|
1270
|
-
}
|
|
1271
|
-
const willRetry = (retryConfig !== undefined && attemptError.retryable && attemptNum < maxAttempts) ||
|
|
1272
|
-
// issue #140 (AMENDED): a STEP_TIMEOUT may ALSO retry in place when the step opted in via
|
|
1273
|
-
// `retry.on_timeout: true` AND attested `idempotent: true` (the concurrency-safety gate).
|
|
1274
|
-
// ALL SIX conjuncts are required; `capMs !== undefined` enforces opted⇒capped
|
|
1275
|
-
// structurally — this disjunct is inert off the enforced auto class even for a hand-built
|
|
1276
|
-
// definition bypassing the loader's E1 gate on a non-auto step, since capMs is undefined
|
|
1277
|
-
// there (shouldEnforceTimeout false ⇒ enforceTimeout false ⇒ capMs undefined) — see R11 in
|
|
1278
|
-
// the design record.
|
|
1279
|
-
(attemptError.code === 'STEP_TIMEOUT' &&
|
|
1280
|
-
capMs !== undefined &&
|
|
1281
|
-
retryConfig?.on_timeout === true &&
|
|
1282
|
-
stepDef.idempotent === true &&
|
|
1283
|
-
!capExhausted &&
|
|
1284
|
-
attemptNum < maxAttempts);
|
|
1285
|
-
if (willRetry) {
|
|
1286
|
-
const baseBackoff = computeBackoff(retryConfig, attemptNum);
|
|
1287
|
-
const retryAfterMs = attemptError instanceof WorkflowError && attemptError.retry_after !== undefined
|
|
1288
|
-
? attemptError.retry_after * 1000
|
|
1289
|
-
: 0;
|
|
1290
|
-
const waitMs = Math.max(baseBackoff, retryAfterMs);
|
|
1291
|
-
// SITE (c) — issue #140, sleep guard, BEFORE every backoff/retry_after sleep (`>=`: an
|
|
1292
|
-
// exact-fit sleep is doomed too — never sleep into a wall). dispatchError already holds the
|
|
1293
|
-
// ACTUAL last error (e.g. a 429 with retry_after in its details); the post-loop wrap gate
|
|
1294
|
-
// below decides whether/how to wrap it — this site only decides whether to sleep at all.
|
|
1295
|
-
if (capMs !== undefined && sleepWouldExceedCap(Date.now() - capStart, waitMs, capMs)) {
|
|
1296
|
-
capExhausted = true;
|
|
1297
|
-
break;
|
|
1298
|
-
}
|
|
1299
|
-
await delayMs(waitMs);
|
|
1300
|
-
}
|
|
1301
|
-
else {
|
|
1302
|
-
break;
|
|
1303
|
-
}
|
|
1616
|
+
allEvidence.push(exhaustedSnap);
|
|
1304
1617
|
}
|
|
1305
|
-
if (
|
|
1618
|
+
if (!bypassDispatch &&
|
|
1619
|
+
dispatchError !== null &&
|
|
1306
1620
|
retryConfig !== undefined &&
|
|
1307
1621
|
(attemptsUsed === maxAttempts || capExhausted)) {
|
|
1308
1622
|
const lastError = dispatchError;
|
|
@@ -1621,7 +1935,12 @@ export async function executeStep(store, definition, options) {
|
|
|
1621
1935
|
}
|
|
1622
1936
|
throw err;
|
|
1623
1937
|
}
|
|
1624
|
-
|
|
1938
|
+
// issue #220 §4c (PR-3, D4): widened to admit a `$`-leading reference (e.g.
|
|
1939
|
+
// `{{ $settlement.step.field }}`) so a typo'd $settlement path in a gate message is
|
|
1940
|
+
// DETECTED as an unresolved placeholder here, not silently left unmatched by this regex
|
|
1941
|
+
// (renderTemplate itself already leaves an unresolved ref's placeholder text verbatim in
|
|
1942
|
+
// `raw` — this is purely a detection-side widening, no change to render behavior).
|
|
1943
|
+
const unresolved = [...raw.matchAll(/\{\{\s*([\w.$-]+)\s*\}\}/g)].map((m) => m[1]);
|
|
1625
1944
|
if (unresolved.length > 0) {
|
|
1626
1945
|
return makeErrorEnvelope(options, pendingRun, new WorkflowError(`gate.message has unresolvable references: ${unresolved.join(', ')}`, {
|
|
1627
1946
|
code: 'GATE_MESSAGE_UNRESOLVABLE',
|
|
@@ -1747,9 +2066,20 @@ export async function executeStep(store, definition, options) {
|
|
|
1747
2066
|
...(isComplete ? { terminal_reason: `Workflow completed.` } : {}),
|
|
1748
2067
|
};
|
|
1749
2068
|
// On the terminal transition, drain the complete/always finalizers before the single seal.
|
|
2069
|
+
// issue #220 PR-2 (D6): stamp defaulted_steps onto the SEALED terminal record only — the
|
|
2070
|
+
// non-terminal branch (`completeDraft`) is never stamped (FM-5 guard: it must never leak onto a
|
|
2071
|
+
// record a later FAIL seal inherits).
|
|
1750
2072
|
const finalRun = isComplete
|
|
1751
|
-
? await buildFinalizedSeal(definition, completeDraft, 'complete', options.registry)
|
|
2073
|
+
? stampDefaultedSteps(await buildFinalizedSeal(definition, completeDraft, 'complete', options.registry))
|
|
1752
2074
|
: completeDraft;
|
|
2075
|
+
// issue #220 PR-2 (D6 write-site consumer): when the stamped seal record's defaulted_steps is
|
|
2076
|
+
// non-empty but this store doesn't declare it durable, disclose the gap explicitly rather than
|
|
2077
|
+
// silently losing the run-level marker on round-trip.
|
|
2078
|
+
const defaultedStepsDurabilityWarning = finalRun.defaulted_steps !== undefined &&
|
|
2079
|
+
finalRun.defaulted_steps.length > 0 &&
|
|
2080
|
+
!persistsField(store, 'defaulted_steps')
|
|
2081
|
+
? 'run-level defaultedness marker (defaulted_steps) not durable on this store'
|
|
2082
|
+
: undefined;
|
|
1753
2083
|
let savedRun;
|
|
1754
2084
|
try {
|
|
1755
2085
|
savedRun = await store.update(finalRun);
|
|
@@ -1828,7 +2158,7 @@ export async function executeStep(store, definition, options) {
|
|
|
1828
2158
|
status: 'ok',
|
|
1829
2159
|
data: output,
|
|
1830
2160
|
evidence: allEvidence,
|
|
1831
|
-
warnings: mergeWarnings(traceWarnings, currentWarn, successWalCleanupWarning),
|
|
2161
|
+
warnings: mergeWarnings(traceWarnings, currentWarn, successWalCleanupWarning, defaultedStepsDurabilityWarning),
|
|
1832
2162
|
errors: [],
|
|
1833
2163
|
context_hint: orientation,
|
|
1834
2164
|
run_phase: savedRun.run_phase,
|
|
@@ -1844,6 +2174,13 @@ export async function executeStep(store, definition, options) {
|
|
|
1844
2174
|
preserved_foreign: adoptionPartition.preserved_foreign,
|
|
1845
2175
|
}
|
|
1846
2176
|
: {}),
|
|
2177
|
+
// issue #220 PR-2 (D5/D6): settled_by_default set true on EXACTLY this success envelope, when
|
|
2178
|
+
// THIS invocation settled via the declared default_output substitution. defaulted_steps read
|
|
2179
|
+
// off `finalRun` — the STAMPED PRE-PERSIST seal record — never off the round-tripped
|
|
2180
|
+
// `savedRun`, since a non-persisting store would silently drop the field from that round-trip
|
|
2181
|
+
// and the qualifier must state the run's TRUE defaultedness.
|
|
2182
|
+
...(settledByDefault ? { settled_by_default: true } : {}),
|
|
2183
|
+
...(finalRun.defaulted_steps?.length ? { defaulted_steps: finalRun.defaulted_steps } : {}),
|
|
1847
2184
|
};
|
|
1848
2185
|
}
|
|
1849
2186
|
/**
|
|
@@ -1953,9 +2290,17 @@ export async function submitHumanResponse(store, definition, options) {
|
|
|
1953
2290
|
...(isComplete ? { terminal_reason: `Workflow completed.` } : {}),
|
|
1954
2291
|
};
|
|
1955
2292
|
// On the gate-completion terminal transition, drain complete/always finalizers before seal.
|
|
2293
|
+
// issue #220 PR-2 (D6): stamp defaulted_steps onto the SEALED terminal record only (never the
|
|
2294
|
+
// non-terminal `gateDraft` — the FM-5 guard).
|
|
1956
2295
|
const finalRun = isComplete
|
|
1957
|
-
? await buildFinalizedSeal(definition, gateDraft, 'complete', options.registry)
|
|
2296
|
+
? stampDefaultedSteps(await buildFinalizedSeal(definition, gateDraft, 'complete', options.registry))
|
|
1958
2297
|
: gateDraft;
|
|
2298
|
+
// issue #220 PR-2 (D6 write-site consumer): see the Step-6 twin above.
|
|
2299
|
+
const defaultedStepsDurabilityWarning = finalRun.defaulted_steps !== undefined &&
|
|
2300
|
+
finalRun.defaulted_steps.length > 0 &&
|
|
2301
|
+
!persistsField(store, 'defaulted_steps')
|
|
2302
|
+
? 'run-level defaultedness marker (defaulted_steps) not durable on this store'
|
|
2303
|
+
: undefined;
|
|
1959
2304
|
let savedRun;
|
|
1960
2305
|
try {
|
|
1961
2306
|
savedRun = await store.update(finalRun);
|
|
@@ -1984,11 +2329,17 @@ export async function submitHumanResponse(store, definition, options) {
|
|
|
1984
2329
|
status: 'ok',
|
|
1985
2330
|
data,
|
|
1986
2331
|
evidence: [],
|
|
1987
|
-
|
|
2332
|
+
// issue #220 PR-2 (D5): submitHumanResponse is a SEPARATE function with no D4
|
|
2333
|
+
// `settledByDefault` local in scope — it does NOT set the per-settle `settled_by_default`
|
|
2334
|
+
// envelope flag (do NOT add an evidence scan to recompute it). Its disclosure surface is the
|
|
2335
|
+
// run-level `defaulted_steps` marker below, plus whatever the gate-open envelope already
|
|
2336
|
+
// warned the human with.
|
|
2337
|
+
warnings: mergeWarnings([], defaultedStepsDurabilityWarning),
|
|
1988
2338
|
errors: [],
|
|
1989
2339
|
context_hint: orientation,
|
|
1990
2340
|
run_phase: savedRun.run_phase,
|
|
1991
2341
|
next_actions: nextActions,
|
|
2342
|
+
...(finalRun.defaulted_steps?.length ? { defaulted_steps: finalRun.defaulted_steps } : {}),
|
|
1992
2343
|
};
|
|
1993
2344
|
}
|
|
1994
2345
|
const MAX_CHAIN_DEPTH = 50;
|
|
@@ -2313,9 +2664,20 @@ depth0Warnings) {
|
|
|
2313
2664
|
? 'complete'
|
|
2314
2665
|
: 'fail'
|
|
2315
2666
|
: undefined;
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2667
|
+
// issue #220 PR-2 (D6): stamp defaulted_steps ONLY on the 'complete' seal — a guard that FAILS
|
|
2668
|
+
// or ABORTS the run does not get the qualifier (the FM-5 guard: never on a non-complete
|
|
2669
|
+
// terminal, and never on the non-terminal `guardResult` passthrough).
|
|
2670
|
+
const guardSealed = guardOutcome === 'complete'
|
|
2671
|
+
? stampDefaultedSteps(await buildFinalizedSeal(definition, guardResult, guardOutcome, options.registry))
|
|
2672
|
+
: guardOutcome !== undefined
|
|
2673
|
+
? await buildFinalizedSeal(definition, guardResult, guardOutcome, options.registry)
|
|
2674
|
+
: guardResult;
|
|
2675
|
+
// issue #220 PR-2 (D6 write-site consumer): see the Step-6 twin above.
|
|
2676
|
+
const guardDefaultedStepsDurabilityWarning = guardSealed.defaulted_steps !== undefined &&
|
|
2677
|
+
guardSealed.defaulted_steps.length > 0 &&
|
|
2678
|
+
!persistsField(store, 'defaulted_steps')
|
|
2679
|
+
? 'run-level defaultedness marker (defaulted_steps) not durable on this store'
|
|
2680
|
+
: undefined;
|
|
2319
2681
|
// Persist the guard step result.
|
|
2320
2682
|
let persistedGuardRun;
|
|
2321
2683
|
try {
|
|
@@ -2357,11 +2719,16 @@ depth0Warnings) {
|
|
|
2357
2719
|
data: {},
|
|
2358
2720
|
// The guard's own evidence entry, captured before the finalizer drain appended any.
|
|
2359
2721
|
evidence: guardOwnEvidence,
|
|
2360
|
-
warnings: [],
|
|
2722
|
+
warnings: mergeWarnings([], guardDefaultedStepsDurabilityWarning),
|
|
2361
2723
|
errors: [],
|
|
2362
2724
|
context_hint: contextHint,
|
|
2363
2725
|
run_phase: persistedGuardRun.run_phase,
|
|
2364
2726
|
next_actions: [],
|
|
2727
|
+
// issue #220 PR-2 (D6): read off `guardSealed` — the stamped PRE-PERSIST record — never
|
|
2728
|
+
// the round-tripped `persistedGuardRun`, so a non-persisting store can't silently drop it.
|
|
2729
|
+
...(guardSealed.defaulted_steps?.length
|
|
2730
|
+
? { defaulted_steps: guardSealed.defaulted_steps }
|
|
2731
|
+
: {}),
|
|
2365
2732
|
};
|
|
2366
2733
|
}
|
|
2367
2734
|
run = persistedGuardRun;
|