@sensigo/realm 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engine/claim-liveness.d.ts +59 -11
- package/dist/engine/claim-liveness.d.ts.map +1 -1
- package/dist/engine/claim-liveness.js +83 -19
- package/dist/engine/claim-liveness.js.map +1 -1
- package/dist/engine/execution-loop.d.ts.map +1 -1
- package/dist/engine/execution-loop.js +141 -13
- package/dist/engine/execution-loop.js.map +1 -1
- package/dist/evidence/snapshot.d.ts +11 -1
- package/dist/evidence/snapshot.d.ts.map +1 -1
- package/dist/evidence/snapshot.js +1 -0
- package/dist/evidence/snapshot.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/types/run-record.d.ts +27 -5
- package/dist/types/run-record.d.ts.map +1 -1
- package/dist/types/workflow-definition.d.ts +52 -7
- package/dist/types/workflow-definition.d.ts.map +1 -1
- package/dist/types/workflow-definition.js +15 -0
- package/dist/types/workflow-definition.js.map +1 -1
- package/dist/workflow/diagnostics.d.ts +15 -6
- package/dist/workflow/diagnostics.d.ts.map +1 -1
- package/dist/workflow/diagnostics.js +16 -8
- package/dist/workflow/diagnostics.js.map +1 -1
- package/dist/workflow/yaml-loader.d.ts.map +1 -1
- package/dist/workflow/yaml-loader.js +108 -6
- package/dist/workflow/yaml-loader.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { RunRecord, ClaimRecord } from '../types/run-record.js';
|
|
2
|
-
import type { WorkflowDefinition, StepDefinition } from '../types/workflow-definition.js';
|
|
2
|
+
import type { WorkflowDefinition, StepDefinition, RetryConfig } from '../types/workflow-definition.js';
|
|
3
3
|
/**
|
|
4
4
|
* Default execution timeout (issue A3) enforced on every `execution: 'auto'` step that declares no
|
|
5
5
|
* `timeout_seconds` — bounds a hung adapter/handler call so it fails loudly (`STEP_TIMEOUT`)
|
|
@@ -33,6 +33,40 @@ export declare const RECLAIM_MARGIN_SECONDS = 60;
|
|
|
33
33
|
* deadline-carrying claim — finalizer-bearing steps get `deadline: null` → `claim_unknown_age`.)
|
|
34
34
|
*/
|
|
35
35
|
export declare const RECLAIM_FLOOR_SECONDS = 900;
|
|
36
|
+
/**
|
|
37
|
+
* Computes the worst-case wall-clock (SECONDS) across every attempt of a retry-configured step:
|
|
38
|
+
* `max_attempts × perAttemptSec` + the sum of the DECLARED backoff delays between attempts (n-1 of
|
|
39
|
+
* them, for attemptNum 1..n-1 — matches the retry loop's own schedule in execution-loop.ts;
|
|
40
|
+
* computeBackoff returns ms). Pure — the declared schedule only; does not model a runtime
|
|
41
|
+
* rate-limit `retry_after` override (execution-loop.ts applies that separately). This is the
|
|
42
|
+
* SHARED formula (issue #140) consumed by BOTH `computeClaimDeadline` below (the claim DETECTION
|
|
43
|
+
* horizon) and `resolveCapMs` (the engine's ENFORCEMENT cap) so the two can never independently
|
|
44
|
+
* drift apart — see the congruence-by-construction test in claim-liveness.test.ts.
|
|
45
|
+
*/
|
|
46
|
+
export declare function worstCaseScheduleSeconds(retry: RetryConfig, perAttemptSec: number): number;
|
|
47
|
+
/**
|
|
48
|
+
* Resolves the total-time cap (MILLISECONDS) enforced across every attempt of a retry-configured
|
|
49
|
+
* `execution: 'auto'` step (issue #140, AMENDED default): the explicit `retry.total_timeout_seconds`
|
|
50
|
+
* when declared — `!== undefined`, NEVER truthiness, so a hand-built `total_timeout_seconds: 0` is
|
|
51
|
+
* honored as a present (already-exhausted) cap rather than falling through to the default — else
|
|
52
|
+
* the shared `worstCaseScheduleSeconds` formula above. The default equals the step's own declared
|
|
53
|
+
* schedule, so the cap binds only when a runtime wait pushes an attempt's actual wall-clock
|
|
54
|
+
* MATERIALLY past it. Call only when enforcement applies and `retryConfig` is defined
|
|
55
|
+
* (execution-loop.ts gates the call on `enforceTimeout && retryConfig !== undefined`) — `timeoutMs`
|
|
56
|
+
* is guaranteed defined in that case. Seconds-in (`total_timeout_seconds`, `timeoutMs / 1000`),
|
|
57
|
+
* MILLISECONDS-out — the same convention `timeoutMs` itself uses.
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveCapMs(retry: RetryConfig, timeoutMs: number): number;
|
|
60
|
+
/**
|
|
61
|
+
* Pure predicate for the engine's site-(c) sleep guard (issue #140 correction, S2): whether
|
|
62
|
+
* sleeping `waitMs` more, on top of `elapsedMs` already spent, would meet or exceed the total-time
|
|
63
|
+
* cap `capMs`. `>=`, not `>` — an exact-fit sleep is doomed too (never sleep into a wall). Module
|
|
64
|
+
* export ONLY (not re-exported from `packages/core/src/index.ts`) — this exists purely so the
|
|
65
|
+
* boundary itself can be unit-pinned without depending on real-timer elapsed-time precision (a
|
|
66
|
+
* real clock is never observed to read EXACTLY `capMs - waitMs`, so a test driving this through
|
|
67
|
+
* `executeStep` cannot discriminate `>=` from `>` at the boundary — seeing this in isolation can).
|
|
68
|
+
*/
|
|
69
|
+
export declare function sleepWouldExceedCap(elapsedMs: number, waitMs: number, capMs: number): boolean;
|
|
36
70
|
/**
|
|
37
71
|
* Computes the claim deadline for `stepName` at claim time. A CONCRETE deadline is returned ONLY
|
|
38
72
|
* for a reliably time-boundable claim: an `execution: 'auto'` (handler-driven) step in a workflow
|
|
@@ -42,16 +76,30 @@ export declare const RECLAIM_FLOOR_SECONDS = 900;
|
|
|
42
76
|
* any step in a finalizer-bearing workflow may hold its claim through the terminal drain.
|
|
43
77
|
*
|
|
44
78
|
* The horizon is the WORST-CASE wall-clock across every retry attempt (issue #101 follow-up —
|
|
45
|
-
* Design Reviewer finding #4 from the A3 debate)
|
|
46
|
-
* step's real claim span
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
79
|
+
* Design Reviewer finding #4 from the A3 debate), via the shared `worstCaseScheduleSeconds`
|
|
80
|
+
* formula above: a single-attempt bound understates a retrying step's real claim span, so a
|
|
81
|
+
* legitimately-retrying step could be labeled `claim_stale` — and, if `idempotent`, become
|
|
82
|
+
* eligible for a premature `realm run reclaim --all --force` re-drive — while still on an early
|
|
83
|
+
* attempt. For `n = 1` (no `retry:`, or `max_attempts: 1`) this reduces EXACTLY to the prior
|
|
84
|
+
* single-attempt formula `max(RECLAIM_FLOOR, perAttemptSec + MARGIN)`, so non-retry steps'
|
|
85
|
+
* horizons are byte-unchanged.
|
|
86
|
+
*
|
|
87
|
+
* Issue #140 AMENDMENT + correction (B2): the horizon now consumes the SAME resolved cap the
|
|
88
|
+
* engine enforces — `(explicit total_timeout_seconds ?? worstCaseScheduleSeconds) + margin`,
|
|
89
|
+
* via `resolveCapMs` (record §3: `(cap ?? worstCase) + margin`). This is NOT merely "the runtime
|
|
90
|
+
* enforcement side changed; this function's formula did not" (an earlier, INCORRECT claim this
|
|
91
|
+
* paragraph made) — the horizon's own formula changed too, and had to: a step declaring an
|
|
92
|
+
* EXPLICIT `total_timeout_seconds` ABOVE its worst-case schedule (the legitimate long-wait
|
|
93
|
+
* opt-in this feature exists to support) would otherwise sleep past a horizon computed from the
|
|
94
|
+
* worst-case schedule ALONE, well before its declared cap — a premature `claim_stale` mid-
|
|
95
|
+
* legitimate-sleep, exactly the silent-duplicate hazard this program closes. Consuming
|
|
96
|
+
* `resolveCapMs` closes it: retry-no-cap still reduces to the worst-case schedule (unchanged,
|
|
97
|
+
* every pre-existing pin stays byte-identical), no-retry still reduces to `perAttemptSec`
|
|
98
|
+
* (unchanged), and retry-WITH-an-explicit-cap now widens the horizon to `cap + margin` — the
|
|
99
|
+
* record's mandate. The prior "does not attempt to model `retry_after`" caveat remains retired
|
|
100
|
+
* for the DEFAULT-cap population for the same reason as before (enforcement now tracks the
|
|
101
|
+
* declared schedule there too); it is now ALSO honestly retired for the explicit-cap population,
|
|
102
|
+
* which this correction is what actually makes true.
|
|
55
103
|
*/
|
|
56
104
|
export declare function computeClaimDeadline(definition: WorkflowDefinition, stepName: string, now: Date): string | null;
|
|
57
105
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claim-liveness.d.ts","sourceRoot":"","sources":["../../src/engine/claim-liveness.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"claim-liveness.d.ts","sourceRoot":"","sources":["../../src/engine/claim-liveness.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrE,OAAO,KAAK,EACV,kBAAkB,EAClB,cAAc,EACd,WAAW,EACZ,MAAM,iCAAiC,CAAC;AAGzC;;;;;;;GAOG;AACH,eAAO,MAAM,iCAAiC,OAAO,CAAC;AAEtD;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,cAAc,GAAG,OAAO,CAElE;AAKD;;;GAGG;AACH,eAAO,MAAM,4BAA4B,MAAM,CAAC;AAChD,qGAAqG;AACrG,eAAO,MAAM,sBAAsB,KAAK,CAAC;AACzC;;;;;;GAMG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,GAAG,MAAM,CAa1F;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAE1E;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAE7F;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,wBAAgB,oBAAoB,CAClC,UAAU,EAAE,kBAAkB,EAC9B,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,IAAI,GACR,MAAM,GAAG,IAAI,CAgBf;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CACvB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,SAAS,EAC/C,QAAQ,EAAE,MAAM,GACf,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAI7B;AAED;;;;;;GAMG;AACH,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,aAAa,GAAG,mBAAmB,CAAC;AAEzE,wGAAwG;AACxG,wBAAgB,aAAa,CAAC,KAAK,EAAE,WAAW,GAAG,SAAS,EAAE,GAAG,EAAE,IAAI,GAAG,UAAU,CAGnF;AAED,6FAA6F;AAC7F,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB;AAED;;;GAGG;AACH,wBAAgB,wBAAwB,CACtC,GAAG,EAAE,SAAS,EACd,GAAG,GAAE,IAAiB,GACrB,mBAAmB,EAAE,CAKvB"}
|
|
@@ -36,6 +36,58 @@ export const RECLAIM_MARGIN_SECONDS = 60; // 1 min
|
|
|
36
36
|
* deadline-carrying claim — finalizer-bearing steps get `deadline: null` → `claim_unknown_age`.)
|
|
37
37
|
*/
|
|
38
38
|
export const RECLAIM_FLOOR_SECONDS = 900; // 15 min
|
|
39
|
+
/**
|
|
40
|
+
* Computes the worst-case wall-clock (SECONDS) across every attempt of a retry-configured step:
|
|
41
|
+
* `max_attempts × perAttemptSec` + the sum of the DECLARED backoff delays between attempts (n-1 of
|
|
42
|
+
* them, for attemptNum 1..n-1 — matches the retry loop's own schedule in execution-loop.ts;
|
|
43
|
+
* computeBackoff returns ms). Pure — the declared schedule only; does not model a runtime
|
|
44
|
+
* rate-limit `retry_after` override (execution-loop.ts applies that separately). This is the
|
|
45
|
+
* SHARED formula (issue #140) consumed by BOTH `computeClaimDeadline` below (the claim DETECTION
|
|
46
|
+
* horizon) and `resolveCapMs` (the engine's ENFORCEMENT cap) so the two can never independently
|
|
47
|
+
* drift apart — see the congruence-by-construction test in claim-liveness.test.ts.
|
|
48
|
+
*/
|
|
49
|
+
export function worstCaseScheduleSeconds(retry, perAttemptSec) {
|
|
50
|
+
// issue #140 correction (B1): the loader ADMITS an absent `max_attempts` (only validates its
|
|
51
|
+
// shape IF present — see yaml-loader.ts's retry-block walk; W1's own comment says so), so a
|
|
52
|
+
// loader-legal `retry: {backoff: fixed, base_delay_ms: 1000}` step reaches here with
|
|
53
|
+
// `retry.max_attempts === undefined` despite the TYPE declaring it required. An unguarded read
|
|
54
|
+
// is a NaN cascade: NaN capMs arms every guard, NaN effectiveMs instant-timeouts attempt 1 via
|
|
55
|
+
// setTimeout's own NaN→0 coercion, and `new Date(NaN).toISOString()` THROWS inside
|
|
56
|
+
// computeClaimDeadline (called from claimStep) — every execute_step on such a step would crash.
|
|
57
|
+
// Mirror execution-loop's own `retryConfig?.max_attempts ?? 1` default idiom exactly.
|
|
58
|
+
const n = retry.max_attempts ?? 1;
|
|
59
|
+
let backoffSec = 0;
|
|
60
|
+
for (let a = 1; a < n; a++)
|
|
61
|
+
backoffSec += computeBackoff(retry, a) / 1000;
|
|
62
|
+
return n * perAttemptSec + backoffSec;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Resolves the total-time cap (MILLISECONDS) enforced across every attempt of a retry-configured
|
|
66
|
+
* `execution: 'auto'` step (issue #140, AMENDED default): the explicit `retry.total_timeout_seconds`
|
|
67
|
+
* when declared — `!== undefined`, NEVER truthiness, so a hand-built `total_timeout_seconds: 0` is
|
|
68
|
+
* honored as a present (already-exhausted) cap rather than falling through to the default — else
|
|
69
|
+
* the shared `worstCaseScheduleSeconds` formula above. The default equals the step's own declared
|
|
70
|
+
* schedule, so the cap binds only when a runtime wait pushes an attempt's actual wall-clock
|
|
71
|
+
* MATERIALLY past it. Call only when enforcement applies and `retryConfig` is defined
|
|
72
|
+
* (execution-loop.ts gates the call on `enforceTimeout && retryConfig !== undefined`) — `timeoutMs`
|
|
73
|
+
* is guaranteed defined in that case. Seconds-in (`total_timeout_seconds`, `timeoutMs / 1000`),
|
|
74
|
+
* MILLISECONDS-out — the same convention `timeoutMs` itself uses.
|
|
75
|
+
*/
|
|
76
|
+
export function resolveCapMs(retry, timeoutMs) {
|
|
77
|
+
return (retry.total_timeout_seconds ?? worstCaseScheduleSeconds(retry, timeoutMs / 1000)) * 1000;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Pure predicate for the engine's site-(c) sleep guard (issue #140 correction, S2): whether
|
|
81
|
+
* sleeping `waitMs` more, on top of `elapsedMs` already spent, would meet or exceed the total-time
|
|
82
|
+
* cap `capMs`. `>=`, not `>` — an exact-fit sleep is doomed too (never sleep into a wall). Module
|
|
83
|
+
* export ONLY (not re-exported from `packages/core/src/index.ts`) — this exists purely so the
|
|
84
|
+
* boundary itself can be unit-pinned without depending on real-timer elapsed-time precision (a
|
|
85
|
+
* real clock is never observed to read EXACTLY `capMs - waitMs`, so a test driving this through
|
|
86
|
+
* `executeStep` cannot discriminate `>=` from `>` at the boundary — seeing this in isolation can).
|
|
87
|
+
*/
|
|
88
|
+
export function sleepWouldExceedCap(elapsedMs, waitMs, capMs) {
|
|
89
|
+
return elapsedMs + waitMs >= capMs;
|
|
90
|
+
}
|
|
39
91
|
/**
|
|
40
92
|
* Computes the claim deadline for `stepName` at claim time. A CONCRETE deadline is returned ONLY
|
|
41
93
|
* for a reliably time-boundable claim: an `execution: 'auto'` (handler-driven) step in a workflow
|
|
@@ -45,16 +97,30 @@ export const RECLAIM_FLOOR_SECONDS = 900; // 15 min
|
|
|
45
97
|
* any step in a finalizer-bearing workflow may hold its claim through the terminal drain.
|
|
46
98
|
*
|
|
47
99
|
* The horizon is the WORST-CASE wall-clock across every retry attempt (issue #101 follow-up —
|
|
48
|
-
* Design Reviewer finding #4 from the A3 debate)
|
|
49
|
-
* step's real claim span
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
100
|
+
* Design Reviewer finding #4 from the A3 debate), via the shared `worstCaseScheduleSeconds`
|
|
101
|
+
* formula above: a single-attempt bound understates a retrying step's real claim span, so a
|
|
102
|
+
* legitimately-retrying step could be labeled `claim_stale` — and, if `idempotent`, become
|
|
103
|
+
* eligible for a premature `realm run reclaim --all --force` re-drive — while still on an early
|
|
104
|
+
* attempt. For `n = 1` (no `retry:`, or `max_attempts: 1`) this reduces EXACTLY to the prior
|
|
105
|
+
* single-attempt formula `max(RECLAIM_FLOOR, perAttemptSec + MARGIN)`, so non-retry steps'
|
|
106
|
+
* horizons are byte-unchanged.
|
|
107
|
+
*
|
|
108
|
+
* Issue #140 AMENDMENT + correction (B2): the horizon now consumes the SAME resolved cap the
|
|
109
|
+
* engine enforces — `(explicit total_timeout_seconds ?? worstCaseScheduleSeconds) + margin`,
|
|
110
|
+
* via `resolveCapMs` (record §3: `(cap ?? worstCase) + margin`). This is NOT merely "the runtime
|
|
111
|
+
* enforcement side changed; this function's formula did not" (an earlier, INCORRECT claim this
|
|
112
|
+
* paragraph made) — the horizon's own formula changed too, and had to: a step declaring an
|
|
113
|
+
* EXPLICIT `total_timeout_seconds` ABOVE its worst-case schedule (the legitimate long-wait
|
|
114
|
+
* opt-in this feature exists to support) would otherwise sleep past a horizon computed from the
|
|
115
|
+
* worst-case schedule ALONE, well before its declared cap — a premature `claim_stale` mid-
|
|
116
|
+
* legitimate-sleep, exactly the silent-duplicate hazard this program closes. Consuming
|
|
117
|
+
* `resolveCapMs` closes it: retry-no-cap still reduces to the worst-case schedule (unchanged,
|
|
118
|
+
* every pre-existing pin stays byte-identical), no-retry still reduces to `perAttemptSec`
|
|
119
|
+
* (unchanged), and retry-WITH-an-explicit-cap now widens the horizon to `cap + margin` — the
|
|
120
|
+
* record's mandate. The prior "does not attempt to model `retry_after`" caveat remains retired
|
|
121
|
+
* for the DEFAULT-cap population for the same reason as before (enforcement now tracks the
|
|
122
|
+
* declared schedule there too); it is now ALSO honestly retired for the explicit-cap population,
|
|
123
|
+
* which this correction is what actually makes true.
|
|
58
124
|
*/
|
|
59
125
|
export function computeClaimDeadline(definition, stepName, now) {
|
|
60
126
|
const step = definition.steps[stepName];
|
|
@@ -63,16 +129,14 @@ export function computeClaimDeadline(definition, stepName, now) {
|
|
|
63
129
|
const hasFinalizers = Object.values(definition.steps).some((s) => s.execution === 'finalizer');
|
|
64
130
|
if (step.execution !== 'auto' || hasFinalizers)
|
|
65
131
|
return null;
|
|
66
|
-
const n = step.retry?.max_attempts ?? 1;
|
|
67
132
|
const perAttemptSec = step.timeout_seconds ?? DEFAULT_EXECUTION_TIMEOUT_SECONDS;
|
|
68
|
-
//
|
|
69
|
-
//
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
const worstCaseSec = n * perAttemptSec + backoffSec;
|
|
133
|
+
// B2: consume the resolved cap (explicit total_timeout_seconds, when declared, else the shared
|
|
134
|
+
// worst-case formula) — NOT worstCaseScheduleSeconds directly — so an explicit cap ABOVE the
|
|
135
|
+
// worst-case schedule widens the horizon instead of leaving it silently short. resolveCapMs
|
|
136
|
+
// returns MILLISECONDS; this function works in SECONDS throughout, hence the /1000.
|
|
137
|
+
const worstCaseSec = step.retry !== undefined
|
|
138
|
+
? resolveCapMs(step.retry, perAttemptSec * 1000) / 1000
|
|
139
|
+
: perAttemptSec;
|
|
76
140
|
const horizonSeconds = Math.max(RECLAIM_FLOOR_SECONDS, worstCaseSec + RECLAIM_MARGIN_SECONDS);
|
|
77
141
|
return new Date(now.getTime() + horizonSeconds * 1000).toISOString();
|
|
78
142
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"claim-liveness.js","sourceRoot":"","sources":["../../src/engine/claim-liveness.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"claim-liveness.js","sourceRoot":"","sources":["../../src/engine/claim-liveness.ts"],"names":[],"mappings":"AAiBA,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,IAAI,CAAC,CAAC,SAAS;AAEhE;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAoB;IACvD,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AACnC,CAAC;AAED,+FAA+F;AAC/F,oFAAoF;AAEpF;;;GAGG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC,CAAC,QAAQ;AACzD,qGAAqG;AACrG,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC,CAAC,QAAQ;AAClD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC,CAAC,SAAS;AAEnD;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CAAC,KAAkB,EAAE,aAAqB;IAChF,6FAA6F;IAC7F,4FAA4F;IAC5F,qFAAqF;IACrF,+FAA+F;IAC/F,+FAA+F;IAC/F,mFAAmF;IACnF,gGAAgG;IAChG,sFAAsF;IACtF,MAAM,CAAC,GAAG,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;IAClC,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,UAAU,IAAI,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;IAC1E,OAAO,CAAC,GAAG,aAAa,GAAG,UAAU,CAAC;AACxC,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,YAAY,CAAC,KAAkB,EAAE,SAAiB;IAChE,OAAO,CAAC,KAAK,CAAC,qBAAqB,IAAI,wBAAwB,CAAC,KAAK,EAAE,SAAS,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC;AACnG,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,SAAiB,EAAE,MAAc,EAAE,KAAa;IAClF,OAAO,SAAS,GAAG,MAAM,IAAI,KAAK,CAAC;AACrC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,MAAM,UAAU,oBAAoB,CAClC,UAA8B,EAC9B,QAAgB,EAChB,GAAS;IAET,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC;IAC/F,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,aAAa;QAAE,OAAO,IAAI,CAAC;IAC5D,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,IAAI,iCAAiC,CAAC;IAChF,+FAA+F;IAC/F,6FAA6F;IAC7F,4FAA4F;IAC5F,oFAAoF;IACpF,MAAM,YAAY,GAChB,IAAI,CAAC,KAAK,KAAK,SAAS;QACtB,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,CAAC,GAAG,IAAI;QACvD,CAAC,CAAC,aAAa,CAAC;IACpB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,YAAY,GAAG,sBAAsB,CAAC,CAAC;IAC9F,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,cAAc,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;AACvE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CACvB,MAA+C,EAC/C,QAAgB;IAEhB,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC;AAWD,wGAAwG;AACxG,MAAM,UAAU,aAAa,CAAC,KAA8B,EAAE,GAAS;IACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAC/E,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;AACxF,CAAC;AASD;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CACtC,GAAc,EACd,MAAY,IAAI,IAAI,EAAE;IAEtB,OAAO,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,IAAI,EAAE,CAAC;IACvF,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-loop.d.ts","sourceRoot":"","sources":["../../src/engine/execution-loop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,SAAS,EAIT,eAAe,EAEhB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAE5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,KAAK,EAAE,gBAAgB,EAAiB,MAAM,gCAAgC,CAAC;AAWtF,OAAO,EAAE,eAAe,EAA0C,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"execution-loop.d.ts","sourceRoot":"","sources":["../../src/engine/execution-loop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,SAAS,EAIT,eAAe,EAEhB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAE5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAE5D,OAAO,KAAK,EAAE,gBAAgB,EAAiB,MAAM,gCAAgC,CAAC;AAWtF,OAAO,EAAE,eAAe,EAA0C,MAAM,gBAAgB,CAAC;AAczF,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAoB9D,MAAM,MAAM,cAAc,GAAG,CAC3B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,GAAG,EAAE,SAAS,EACd,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtC,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,cAAc,EAAE,CAAA;KAAE,CAAC;IAC5C;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC;;;;;;;;;OASG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,uCAAuC;IACvC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,cAAc,EAAE,CAAA;KAAE,CAAC;IAC5C,oCAAoC;IACpC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,+CAA+C;IAC/C,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,0CAA0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAkaD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,kBAAkB,EAAE,GAAG,EAAE,SAAS,GAAG,UAAU,EAAE,CAwB7F;AAgND;;;;GAIG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,aAAa,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,gBAAgB,CAkBlB;AAiCD;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,kBAAkB,EAC9B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,gBAAgB,CAAC,CAm8C3B;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,kBAAkB,EAC9B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAqL3B;AA6dD;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,kBAAkB,EAC9B,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAqE3B;AAGD,OAAO,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC"}
|
|
@@ -7,7 +7,7 @@ import { captureEvidence } from '../evidence/snapshot.js';
|
|
|
7
7
|
import { validateInputSchema, validateOutputSchema, validateTraceSchema, } from '../validation/input-schema.js';
|
|
8
8
|
import { normalizeTrace } from './trace-normalizer.js';
|
|
9
9
|
import { TERMINAL_PHASES, isTerminalPhase, DRAIN_CEILING_SECONDS } from './lifecycle.js';
|
|
10
|
-
import { omitClaim, shouldEnforceTimeout, DEFAULT_EXECUTION_TIMEOUT_SECONDS, } from './claim-liveness.js';
|
|
10
|
+
import { omitClaim, shouldEnforceTimeout, DEFAULT_EXECUTION_TIMEOUT_SECONDS, resolveCapMs, sleepWouldExceedCap, } from './claim-liveness.js';
|
|
11
11
|
import { computeBackoff } from './backoff.js';
|
|
12
12
|
import { checkPreconditions, evaluateAllPreconditions, evaluateGuardConditions, } from './precondition.js';
|
|
13
13
|
import { ExtensionRegistry } from '../extensions/registry.js';
|
|
@@ -36,7 +36,9 @@ function withTimeout(dispatch, ms, stepName) {
|
|
|
36
36
|
category: 'ENGINE',
|
|
37
37
|
agentAction: 'report_to_user',
|
|
38
38
|
retryable: false,
|
|
39
|
-
|
|
39
|
+
// issue #140: details gain `stepId` alongside the pre-existing `stepName` — additive,
|
|
40
|
+
// never removes stepName (byte-identical shape for any existing consumer of that key).
|
|
41
|
+
details: { stepName, stepId: stepName, timeout_ms: ms },
|
|
40
42
|
}));
|
|
41
43
|
}, ms);
|
|
42
44
|
});
|
|
@@ -995,14 +997,58 @@ export async function executeStep(store, definition, options) {
|
|
|
995
997
|
// A3: every `execution: 'auto'` step is bounded — authored timeout_seconds if declared, else the
|
|
996
998
|
// generous DEFAULT_EXECUTION_TIMEOUT_SECONDS default. Resolved ONCE here (before the retry loop
|
|
997
999
|
// below), not per-attempt. Agent/guard steps (shouldEnforceTimeout false) are untouched: agent
|
|
998
|
-
// dispatch stays the instant-return no-op it always was, never wrapped in withTimeout
|
|
999
|
-
//
|
|
1000
|
-
//
|
|
1000
|
+
// dispatch stays the instant-return no-op it always was, never wrapped in withTimeout — this
|
|
1001
|
+
// A3 invariant is preserved verbatim by issue #140 below (the finalizer-drain withTimeout at
|
|
1002
|
+
// buildFinalizedSeal is a separate, DRAIN_CEILING-bounded wrap, outside this cap's scope).
|
|
1003
|
+
// effectiveTimeoutSeconds is the single source of truth for the step's OWN declared/default
|
|
1004
|
+
// per-attempt bound; timeoutMs is derived from it so the two can never diverge.
|
|
1005
|
+
//
|
|
1006
|
+
// Issue #140 (retryable timeout + total-time cap): capMs/capStart/capExhausted resolve ONCE
|
|
1007
|
+
// here too (same as timeoutMs), gated on `enforceTimeout && retryConfig !== undefined` — every
|
|
1008
|
+
// retry-configured auto step now gets a default total-time cap (resolveCapMs), whether or not it
|
|
1009
|
+
// opts into `retry.on_timeout`. What is NOT resolved once anymore is the PER-ATTEMPT bound
|
|
1010
|
+
// actually passed to withTimeout: each attempt clips to whatever budget remains (see
|
|
1011
|
+
// `effectiveMs` inside the loop below) — a later attempt's evidence `effective_timeout_seconds`
|
|
1012
|
+
// can therefore be SMALLER than this outer `effectiveTimeoutSeconds` once the cap starts biting.
|
|
1013
|
+
//
|
|
1014
|
+
// Zombie-stacking split (A3 caveat, sharpened by #140's `on_timeout` in-place retry): a timeout
|
|
1015
|
+
// frees the RUNNER, not the work — `withTimeout` races the dispatch against a timer and moves on
|
|
1016
|
+
// the instant the timer wins, but does not stop a handler/adapter that ignores the abort signal
|
|
1017
|
+
// (or a remote server still processing an already-aborted request). Whether an abandoned call
|
|
1018
|
+
// can STACK behind a subsequent retry attempt splits on the handler's own shape: a SYNCHRONOUS
|
|
1019
|
+
// (blocking) handler can never stack one — it monopolizes the event loop, so nothing else
|
|
1020
|
+
// (including the next attempt's own dispatch) can even begin until it returns or the process is
|
|
1021
|
+
// killed. An ASYNCHRONOUS handler that ignores its abort signal CAN stack: each timed-out-and-
|
|
1022
|
+
// retried attempt leaves its own abandoned promise running, so up to `max_attempts − 1` zombies
|
|
1023
|
+
// can accumulate behind the one currently-live attempt (attempts 1..max_attempts−1 each
|
|
1024
|
+
// potentially zombie-and-retry; the final attempt is the "+1" that is never itself abandoned by
|
|
1025
|
+
// this loop). This was always true pre-#140 for a step whose retry consumed a NORMAL retryable
|
|
1026
|
+
// error mid-flight of a slow-but-not-yet-timed-out prior attempt; #140 sharpens it because
|
|
1027
|
+
// `on_timeout` now lets a STEP_TIMEOUT itself mint a retry, so a maximally adversarial handler
|
|
1028
|
+
// can produce the full max_attempts−1 zombie count from timeouts alone. The zombie count stays
|
|
1029
|
+
// cap-BOUNDED (the total-time cap bounds how many attempts the ENGINE'S retry loop can mint —
|
|
1030
|
+
// it does not, and cannot, reach into an already-abandoned zombie running on a remote server
|
|
1031
|
+
// outside realm's control).
|
|
1001
1032
|
const enforceTimeout = stepDef !== undefined && shouldEnforceTimeout(stepDef);
|
|
1002
1033
|
const effectiveTimeoutSeconds = enforceTimeout
|
|
1003
1034
|
? (stepDef.timeout_seconds ?? DEFAULT_EXECUTION_TIMEOUT_SECONDS)
|
|
1004
1035
|
: undefined;
|
|
1005
1036
|
const timeoutMs = effectiveTimeoutSeconds !== undefined ? effectiveTimeoutSeconds * 1000 : undefined;
|
|
1037
|
+
const capMs = enforceTimeout && retryConfig !== undefined ? resolveCapMs(retryConfig, timeoutMs) : undefined;
|
|
1038
|
+
const capStart = Date.now(); // wall-clock — the SAME clock the claim horizon is measured against
|
|
1039
|
+
let capExhausted = false;
|
|
1040
|
+
const remainingMs = () => capMs - (Date.now() - capStart);
|
|
1041
|
+
// Programmatic-gate advisory (#119-preserving): the loader refuses `on_timeout: true` without
|
|
1042
|
+
// `idempotent: true` at load (E1) — but a hand-built WorkflowDefinition (a custom embedder, or a
|
|
1043
|
+
// test) can bypass the loader entirely. Surface the same rule here, at the engine surface, as a
|
|
1044
|
+
// pure advisory (never gates, never changes behavior — the willRetry conjunct below already
|
|
1045
|
+
// requires `idempotent === true` independently). Threaded through `traceWarnings` so it reaches
|
|
1046
|
+
// every settle path this function has, INCLUDING the handler-abort return path below (which
|
|
1047
|
+
// otherwise hardcodes `warnings: []`).
|
|
1048
|
+
if (stepDef !== undefined && retryConfig?.on_timeout === true && stepDef.idempotent !== true) {
|
|
1049
|
+
traceWarnings.push(`retry.on_timeout ignored: step '${options.command}' is not declared idempotent — declare ` +
|
|
1050
|
+
`both 'idempotent: true' and 'retry.on_timeout: true'; YAML workflows are refused at load.`);
|
|
1051
|
+
}
|
|
1006
1052
|
// Create a stable rate-limiter registry for all retry attempts of this step.
|
|
1007
1053
|
// Shared state ensures that a pause() triggered on attempt N is still in effect
|
|
1008
1054
|
// when the proactive acquire() runs on attempt N+1. When the caller provides an
|
|
@@ -1015,11 +1061,34 @@ export async function executeStep(store, definition, options) {
|
|
|
1015
1061
|
const allEvidence = [];
|
|
1016
1062
|
let currentWarn;
|
|
1017
1063
|
for (let attemptNum = 1; attemptNum <= maxAttempts; attemptNum++) {
|
|
1064
|
+
// SITE (a) — issue #140, loop-top, BEFORE attemptsUsed is assigned: attempt 1 ALWAYS
|
|
1065
|
+
// proceeds regardless of capMs (the `attemptNum > 1` conjunct) — this guard exists solely for
|
|
1066
|
+
// the clock-anomaly window between `capStart` above and here (a suspend/resume or NTP forward
|
|
1067
|
+
// jump), never to gate the very first attempt.
|
|
1068
|
+
if (capMs !== undefined && attemptNum > 1 && remainingMs() <= 0) {
|
|
1069
|
+
capExhausted = true;
|
|
1070
|
+
break;
|
|
1071
|
+
}
|
|
1018
1072
|
attemptsUsed = attemptNum;
|
|
1019
1073
|
const startedAt = new Date();
|
|
1020
1074
|
let attemptOutput = {};
|
|
1021
1075
|
let attemptError = null;
|
|
1022
1076
|
let resolvedParams;
|
|
1077
|
+
// Per-attempt effective timeout (issue #140): uniform full-clip to whatever cap budget
|
|
1078
|
+
// remains. Clip floor `max(0, remainingMs())` ensures a clock anomaly (see SITE (a) above)
|
|
1079
|
+
// never passes a negative ms to withTimeout. capMs undefined ⇒ effectiveMs === timeoutMs,
|
|
1080
|
+
// byte-identical to pre-#140 behavior (every non-retry-configured, or unopted-uncapped-by-
|
|
1081
|
+
// total_timeout_seconds-being-absent-pre-amendment, auto step). `clippedToMs` records the
|
|
1082
|
+
// per-attempt evidence value ONLY when the cap actually reduced the bound below the step's
|
|
1083
|
+
// own declared/default timeout — never on an uncapped or not-yet-biting attempt.
|
|
1084
|
+
const effectiveMs = timeoutMs !== undefined
|
|
1085
|
+
? capMs !== undefined
|
|
1086
|
+
? Math.min(timeoutMs, Math.max(0, remainingMs()))
|
|
1087
|
+
: timeoutMs
|
|
1088
|
+
: undefined;
|
|
1089
|
+
const clippedToMs = capMs !== undefined && effectiveMs !== undefined && effectiveMs < timeoutMs
|
|
1090
|
+
? effectiveMs
|
|
1091
|
+
: undefined;
|
|
1023
1092
|
try {
|
|
1024
1093
|
const makeCall = (signal) => {
|
|
1025
1094
|
if (stepDef?.execution === 'auto' && stepDef.uses_service !== undefined) {
|
|
@@ -1050,8 +1119,8 @@ export async function executeStep(store, definition, options) {
|
|
|
1050
1119
|
.then((result) => ({ output: result, resolvedParams: undefined }));
|
|
1051
1120
|
}
|
|
1052
1121
|
};
|
|
1053
|
-
const callResult =
|
|
1054
|
-
? await withTimeout((signal) => makeCall(signal),
|
|
1122
|
+
const callResult = effectiveMs !== undefined
|
|
1123
|
+
? await withTimeout((signal) => makeCall(signal), effectiveMs, options.command)
|
|
1055
1124
|
: await makeCall();
|
|
1056
1125
|
// Handle graceful abort from a handler returning { abort: { message } }.
|
|
1057
1126
|
if (callResult.handlerAbort !== undefined) {
|
|
@@ -1114,7 +1183,9 @@ export async function executeStep(store, definition, options) {
|
|
|
1114
1183
|
status: 'ok',
|
|
1115
1184
|
data: {},
|
|
1116
1185
|
evidence: [abortEvidence],
|
|
1117
|
-
|
|
1186
|
+
// Issue #140: was hardcoded `[]` — now threads traceWarnings (e.g. the programmatic
|
|
1187
|
+
// on_timeout/idempotent gate advisory above) so it survives this settle path too.
|
|
1188
|
+
warnings: mergeWarnings(traceWarnings),
|
|
1118
1189
|
errors: [],
|
|
1119
1190
|
context_hint: `Handler step '${options.command}' aborted the run: ${abortMessage}`,
|
|
1120
1191
|
run_phase: (persistedAbortRun ?? abortedRun).run_phase,
|
|
@@ -1166,7 +1237,11 @@ export async function executeStep(store, definition, options) {
|
|
|
1166
1237
|
...(options.stepMeta?.toolCalls !== undefined
|
|
1167
1238
|
? { toolCalls: options.stepMeta.toolCalls }
|
|
1168
1239
|
: {}),
|
|
1169
|
-
|
|
1240
|
+
// issue #140: PER-ATTEMPT value (may be smaller than the outer effectiveTimeoutSeconds once
|
|
1241
|
+
// the cap starts clipping) — byte-identical to the pre-#140 outer value whenever capMs is
|
|
1242
|
+
// undefined or hasn't bitten yet.
|
|
1243
|
+
...(effectiveMs !== undefined ? { effectiveTimeoutSeconds: effectiveMs / 1000 } : {}),
|
|
1244
|
+
...(clippedToMs !== undefined ? { clippedToMs } : {}),
|
|
1170
1245
|
// Gate trace to agent steps only — drop silently for auto/adapter/handler steps.
|
|
1171
1246
|
// When pre-normalized (WAL merge + schema validation ran), pass the pre-normalized
|
|
1172
1247
|
// result to avoid double normalization. Also handle WAL-only case (options.trace may
|
|
@@ -1185,24 +1260,68 @@ export async function executeStep(store, definition, options) {
|
|
|
1185
1260
|
break;
|
|
1186
1261
|
}
|
|
1187
1262
|
dispatchError = attemptError;
|
|
1188
|
-
|
|
1263
|
+
// SITE (b) — issue #140, post-attempt, AFTER dispatchError is set, BEFORE willRetry: the
|
|
1264
|
+
// primary capExhausted setter (site (a) above only catches the loop-top clock-anomaly window;
|
|
1265
|
+
// site (c) below re-checks once more right before an actual sleep). Fires on ANY dispatch
|
|
1266
|
+
// error once the cap is spent, not just STEP_TIMEOUT — the cap bounds the step's total
|
|
1267
|
+
// budget, regardless of which error exhausted it.
|
|
1268
|
+
if (capMs !== undefined && remainingMs() <= 0) {
|
|
1269
|
+
capExhausted = true;
|
|
1270
|
+
}
|
|
1271
|
+
const willRetry = (retryConfig !== undefined && attemptError.retryable && attemptNum < maxAttempts) ||
|
|
1272
|
+
// issue #140 (AMENDED): a STEP_TIMEOUT may ALSO retry in place when the step opted in via
|
|
1273
|
+
// `retry.on_timeout: true` AND attested `idempotent: true` (the concurrency-safety gate).
|
|
1274
|
+
// ALL SIX conjuncts are required; `capMs !== undefined` enforces opted⇒capped
|
|
1275
|
+
// structurally — this disjunct is inert off the enforced auto class even for a hand-built
|
|
1276
|
+
// definition bypassing the loader's E1 gate on a non-auto step, since capMs is undefined
|
|
1277
|
+
// there (shouldEnforceTimeout false ⇒ enforceTimeout false ⇒ capMs undefined) — see R11 in
|
|
1278
|
+
// the design record.
|
|
1279
|
+
(attemptError.code === 'STEP_TIMEOUT' &&
|
|
1280
|
+
capMs !== undefined &&
|
|
1281
|
+
retryConfig?.on_timeout === true &&
|
|
1282
|
+
stepDef.idempotent === true &&
|
|
1283
|
+
!capExhausted &&
|
|
1284
|
+
attemptNum < maxAttempts);
|
|
1189
1285
|
if (willRetry) {
|
|
1190
1286
|
const baseBackoff = computeBackoff(retryConfig, attemptNum);
|
|
1191
1287
|
const retryAfterMs = attemptError instanceof WorkflowError && attemptError.retry_after !== undefined
|
|
1192
1288
|
? attemptError.retry_after * 1000
|
|
1193
1289
|
: 0;
|
|
1194
|
-
|
|
1290
|
+
const waitMs = Math.max(baseBackoff, retryAfterMs);
|
|
1291
|
+
// SITE (c) — issue #140, sleep guard, BEFORE every backoff/retry_after sleep (`>=`: an
|
|
1292
|
+
// exact-fit sleep is doomed too — never sleep into a wall). dispatchError already holds the
|
|
1293
|
+
// ACTUAL last error (e.g. a 429 with retry_after in its details); the post-loop wrap gate
|
|
1294
|
+
// below decides whether/how to wrap it — this site only decides whether to sleep at all.
|
|
1295
|
+
if (capMs !== undefined && sleepWouldExceedCap(Date.now() - capStart, waitMs, capMs)) {
|
|
1296
|
+
capExhausted = true;
|
|
1297
|
+
break;
|
|
1298
|
+
}
|
|
1299
|
+
await delayMs(waitMs);
|
|
1195
1300
|
}
|
|
1196
1301
|
else {
|
|
1197
1302
|
break;
|
|
1198
1303
|
}
|
|
1199
1304
|
}
|
|
1200
|
-
if (dispatchError !== null &&
|
|
1305
|
+
if (dispatchError !== null &&
|
|
1306
|
+
retryConfig !== undefined &&
|
|
1307
|
+
(attemptsUsed === maxAttempts || capExhausted)) {
|
|
1201
1308
|
const lastError = dispatchError;
|
|
1309
|
+
// issue #140: stamp the discriminator on the LAST evidence snapshot regardless of whether the
|
|
1310
|
+
// #134 carve-out below actually wraps dispatchError — the carve-out's recoverable settle path
|
|
1311
|
+
// (Step 5) never wraps, so this evidence stamp is the ONLY durable record of *why* the step
|
|
1312
|
+
// stopped retrying in that case. `exhausted_by: 'total_timeout'` wins the both-true tie (a
|
|
1313
|
+
// step whose LAST attempt both used its final slot and drained the cap is reported as
|
|
1314
|
+
// cap-caused — the more actionable of the two labels for an operator).
|
|
1315
|
+
const lastSnap = allEvidence[allEvidence.length - 1];
|
|
1316
|
+
if (lastSnap !== undefined) {
|
|
1317
|
+
lastSnap.exhausted_by = capExhausted ? 'total_timeout' : 'attempts';
|
|
1318
|
+
}
|
|
1202
1319
|
// #134: do NOT wrap a recoverable-incapability error (a max_attempts:1 not-registered failure
|
|
1203
1320
|
// hits attemptsUsed === maxAttempts). The STEP_RETRY_EXHAUSTED wrap discards the inner code, which
|
|
1204
1321
|
// would rob Step 5 of the discriminator it needs to settle recoverably. Leave dispatchError as the
|
|
1205
|
-
// original not-registered error; all other codes wrap unchanged.
|
|
1322
|
+
// original not-registered error; all other codes wrap unchanged. This carve-out guards BOTH
|
|
1323
|
+
// disjuncts above (attempts-exhaustion and cap-exhaustion alike) — a capped-but-not-registered
|
|
1324
|
+
// step never wraps into STEP_RETRY_EXHAUSTED either.
|
|
1206
1325
|
const isRecoverableIncapability = lastError instanceof WorkflowError &&
|
|
1207
1326
|
(lastError.code === 'ENGINE_HANDLER_NOT_REGISTERED' ||
|
|
1208
1327
|
lastError.code === 'ENGINE_ADAPTER_NOT_REGISTERED');
|
|
@@ -1216,6 +1335,7 @@ export async function executeStep(store, definition, options) {
|
|
|
1216
1335
|
stepName: options.command,
|
|
1217
1336
|
attempts: attemptsUsed,
|
|
1218
1337
|
lastError: lastError.message,
|
|
1338
|
+
exhausted_by: capExhausted ? 'total_timeout' : 'attempts',
|
|
1219
1339
|
...(lastError.retry_after !== undefined ? { retry_after: lastError.retry_after } : {}),
|
|
1220
1340
|
},
|
|
1221
1341
|
});
|
|
@@ -1448,6 +1568,14 @@ export async function executeStep(store, definition, options) {
|
|
|
1448
1568
|
warnings: mergeWarnings(traceWarnings, storeCleanupWarning ?? walCleanupWarning),
|
|
1449
1569
|
errors: [dispatchError.message],
|
|
1450
1570
|
agent_action: effectiveAction,
|
|
1571
|
+
// issue #140 (D3 §2, discriminator OBSERVABLE): additive-optional — lets a caller
|
|
1572
|
+
// discriminate `STEP_RETRY_EXHAUSTED`'s `exhausted_by` (or any other terminal code) without
|
|
1573
|
+
// parsing `errors[0]`'s message text. Mirrors the errorEnvelope()/buildPreExecutionErrorEnvelope
|
|
1574
|
+
// pattern used elsewhere in this file.
|
|
1575
|
+
error_code: dispatchError.code,
|
|
1576
|
+
...(Object.keys(dispatchError.details).length > 0
|
|
1577
|
+
? { error_details: dispatchError.details }
|
|
1578
|
+
: {}),
|
|
1451
1579
|
...(dispatchError.retry_after !== undefined
|
|
1452
1580
|
? { retry_after: dispatchError.retry_after }
|
|
1453
1581
|
: {}),
|