@sensigo/realm 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/adapter-utils.d.ts +6 -0
- package/dist/adapters/adapter-utils.d.ts.map +1 -1
- package/dist/adapters/adapter-utils.js +21 -0
- package/dist/adapters/adapter-utils.js.map +1 -1
- package/dist/adapters/file-adapter.d.ts +1 -1
- package/dist/adapters/file-adapter.d.ts.map +1 -1
- package/dist/adapters/file-adapter.js +10 -2
- package/dist/adapters/file-adapter.js.map +1 -1
- package/dist/adapters/gorgias-adapter.d.ts +1 -1
- package/dist/adapters/gorgias-adapter.d.ts.map +1 -1
- package/dist/adapters/gorgias-adapter.js +28 -13
- package/dist/adapters/gorgias-adapter.js.map +1 -1
- package/dist/adapters/slack-adapter.d.ts.map +1 -1
- package/dist/adapters/slack-adapter.js +11 -0
- package/dist/adapters/slack-adapter.js.map +1 -1
- package/dist/engine/backoff.d.ts +4 -0
- package/dist/engine/backoff.d.ts.map +1 -0
- package/dist/engine/backoff.js +18 -0
- package/dist/engine/backoff.js.map +1 -0
- package/dist/engine/capability.d.ts +52 -0
- package/dist/engine/capability.d.ts.map +1 -0
- package/dist/engine/capability.js +76 -0
- package/dist/engine/capability.js.map +1 -0
- package/dist/engine/claim-liveness.d.ts +86 -0
- package/dist/engine/claim-liveness.d.ts.map +1 -0
- package/dist/engine/claim-liveness.js +108 -0
- package/dist/engine/claim-liveness.js.map +1 -0
- package/dist/engine/execution-loop.d.ts.map +1 -1
- package/dist/engine/execution-loop.js +137 -33
- package/dist/engine/execution-loop.js.map +1 -1
- package/dist/engine/reclaim-step.d.ts +37 -0
- package/dist/engine/reclaim-step.d.ts.map +1 -0
- package/dist/engine/reclaim-step.js +122 -0
- package/dist/engine/reclaim-step.js.map +1 -0
- package/dist/evidence/snapshot.d.ts +5 -0
- package/dist/evidence/snapshot.d.ts.map +1 -1
- package/dist/evidence/snapshot.js +3 -0
- package/dist/evidence/snapshot.js.map +1 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/store/json-file-store.d.ts +2 -0
- package/dist/store/json-file-store.d.ts.map +1 -1
- package/dist/store/json-file-store.js +64 -6
- package/dist/store/json-file-store.js.map +1 -1
- package/dist/store/store-interface.d.ts +9 -0
- package/dist/store/store-interface.d.ts.map +1 -1
- package/dist/types/run-record.d.ts +60 -0
- package/dist/types/run-record.d.ts.map +1 -1
- package/dist/types/workflow-definition.d.ts +10 -0
- package/dist/types/workflow-definition.d.ts.map +1 -1
- package/dist/types/workflow-error.d.ts +1 -1
- package/dist/types/workflow-error.d.ts.map +1 -1
- package/dist/types/workflow-error.js.map +1 -1
- package/dist/workflow/yaml-loader.js +38 -0
- package/dist/workflow/yaml-loader.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { computeBackoff } from './backoff.js';
|
|
2
|
+
/**
|
|
3
|
+
* Default execution timeout (issue A3) enforced on every `execution: 'auto'` step that declares no
|
|
4
|
+
* `timeout_seconds` — bounds a hung adapter/handler call so it fails loudly (`STEP_TIMEOUT`)
|
|
5
|
+
* instead of pinning the runner forever. Generous (1 hour, GitHub-Actions-style): it guards against
|
|
6
|
+
* a genuine hang, not normal-operation latency. Also now the basis of the claim-deadline DETECTION
|
|
7
|
+
* horizon in `computeClaimDeadline` below, so detection tracks the real enforcement bound. See
|
|
8
|
+
* plans/execution-timeout-a3-design.md for the full rationale.
|
|
9
|
+
*/
|
|
10
|
+
export const DEFAULT_EXECUTION_TIMEOUT_SECONDS = 3600; // 1 hour
|
|
11
|
+
/**
|
|
12
|
+
* Whether `step`'s dispatch must be bounded by a timeout (issue A3). Pure: `execution === 'auto'`,
|
|
13
|
+
* period. Deliberately NOT conjoined with `hasFinalizers` — that conjunct is correct for
|
|
14
|
+
* *detection*'s claim horizon below (a claim may legitimately be held through a finalizer drain),
|
|
15
|
+
* but would be WRONG for *enforcement*: it would leave every auto step in a finalizer-bearing
|
|
16
|
+
* workflow completely unbounded — the exact hang this feature exists to prevent. Do NOT re-add a
|
|
17
|
+
* `hasFinalizers`/finalizer conjunct here (this split was the Design Reviewer's blocking-fix).
|
|
18
|
+
*/
|
|
19
|
+
export function shouldEnforceTimeout(step) {
|
|
20
|
+
return step.execution === 'auto';
|
|
21
|
+
}
|
|
22
|
+
// --- Deadline constants (Phase 1: the deadline drives the DETECTION DISPLAY only; it gates no
|
|
23
|
+
// action, so a too-large horizon merely delays a `claim_stale` label — bias LARGE).
|
|
24
|
+
/**
|
|
25
|
+
* @deprecated Superseded by `DEFAULT_EXECUTION_TIMEOUT_SECONDS`; retained for API compatibility.
|
|
26
|
+
* Assumed handler wall-clock bound when a step declares no `timeout_seconds`. Biased large.
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_STEP_TIMEOUT_SECONDS = 300; // 5 min
|
|
29
|
+
/** Added to a step's `timeout_seconds` to absorb scheduling / clock skew before a claim is stale. */
|
|
30
|
+
export const RECLAIM_MARGIN_SECONDS = 60; // 1 min
|
|
31
|
+
/**
|
|
32
|
+
* Minimum staleness horizon. A concrete claim is never `claim_stale` sooner than this, even for a
|
|
33
|
+
* fast handler — so a live-but-slow runner is not mislabeled. Biased large: too-large only delays
|
|
34
|
+
* the label (Phase 1 gates no action); too-small risks flagging live work. (Concrete deadlines are
|
|
35
|
+
* set ONLY for finalizer-free auto steps, so the worst-case finalizer-drain span never applies to a
|
|
36
|
+
* deadline-carrying claim — finalizer-bearing steps get `deadline: null` → `claim_unknown_age`.)
|
|
37
|
+
*/
|
|
38
|
+
export const RECLAIM_FLOOR_SECONDS = 900; // 15 min
|
|
39
|
+
/**
|
|
40
|
+
* Computes the claim deadline for `stepName` at claim time. A CONCRETE deadline is returned ONLY
|
|
41
|
+
* for a reliably time-boundable claim: an `execution: 'auto'` (handler-driven) step in a workflow
|
|
42
|
+
* with NO `execution: 'finalizer'` steps (so no finalizer drain can legitimately extend the claim
|
|
43
|
+
* past the seal). Every other claim returns `null` (→ `claim_unknown_age`): agent steps have no
|
|
44
|
+
* reliable wall-clock bound (the dispatcher returns instantly; `timeout_seconds` is advisory), and
|
|
45
|
+
* any step in a finalizer-bearing workflow may hold its claim through the terminal drain.
|
|
46
|
+
*
|
|
47
|
+
* The horizon is the WORST-CASE wall-clock across every retry attempt (issue #101 follow-up —
|
|
48
|
+
* Design Reviewer finding #4 from the A3 debate): a single-attempt bound understates a retrying
|
|
49
|
+
* step's real claim span (`max_attempts × per-attempt-timeout + the declared backoffs between
|
|
50
|
+
* attempts`), so a legitimately-retrying step could be labeled `claim_stale` — and, if
|
|
51
|
+
* `idempotent`, become eligible for a premature `realm run reclaim --all --force` re-drive —
|
|
52
|
+
* while still on an early attempt. For `n = 1` (no `retry:`, or `max_attempts: 1`) this reduces
|
|
53
|
+
* EXACTLY to the prior single-attempt formula `max(RECLAIM_FLOOR, perAttemptSec + MARGIN)`, so
|
|
54
|
+
* non-retry steps' horizons are byte-unchanged. This uses the DECLARED backoff schedule only — a
|
|
55
|
+
* runtime `retry_after` (rate-limit 429 override, applied in execution-loop.ts) is not knowable at
|
|
56
|
+
* claim time; the horizon stays a best-effort, floored, advisory bound (Phase 1 gates no action on
|
|
57
|
+
* this label) and does not attempt to model `retry_after`.
|
|
58
|
+
*/
|
|
59
|
+
export function computeClaimDeadline(definition, stepName, now) {
|
|
60
|
+
const step = definition.steps[stepName];
|
|
61
|
+
if (step === undefined)
|
|
62
|
+
return null;
|
|
63
|
+
const hasFinalizers = Object.values(definition.steps).some((s) => s.execution === 'finalizer');
|
|
64
|
+
if (step.execution !== 'auto' || hasFinalizers)
|
|
65
|
+
return null;
|
|
66
|
+
const n = step.retry?.max_attempts ?? 1;
|
|
67
|
+
const perAttemptSec = step.timeout_seconds ?? DEFAULT_EXECUTION_TIMEOUT_SECONDS;
|
|
68
|
+
// Backoffs occur BETWEEN attempts: n-1 of them, for attemptNum 1..n-1 (matches the retry loop's
|
|
69
|
+
// schedule in execution-loop.ts). computeBackoff returns ms; the horizon is seconds.
|
|
70
|
+
let backoffSec = 0;
|
|
71
|
+
if (step.retry !== undefined) {
|
|
72
|
+
for (let a = 1; a < n; a++)
|
|
73
|
+
backoffSec += computeBackoff(step.retry, a) / 1000;
|
|
74
|
+
}
|
|
75
|
+
const worstCaseSec = n * perAttemptSec + backoffSec;
|
|
76
|
+
const horizonSeconds = Math.max(RECLAIM_FLOOR_SECONDS, worstCaseSec + RECLAIM_MARGIN_SECONDS);
|
|
77
|
+
return new Date(now.getTime() + horizonSeconds * 1000).toISOString();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Returns a NEW claims record with `stepName` removed (a fresh object — never mutates the input,
|
|
81
|
+
* so it is safe to use on a value spread from the prior record). Used at the settle/seal sites to
|
|
82
|
+
* delete the claim clock in the SAME record mutation that removes the step from
|
|
83
|
+
* `in_progress_steps`. Returns `{}` when the last claim is removed (harmless; detection ignores a
|
|
84
|
+
* run with no in-progress steps). A missed delete self-heals: `claimStep` overwrites `claims[S]`.
|
|
85
|
+
*/
|
|
86
|
+
export function omitClaim(claims, stepName) {
|
|
87
|
+
if (claims === undefined)
|
|
88
|
+
return {};
|
|
89
|
+
const { [stepName]: _removed, ...rest } = claims;
|
|
90
|
+
return rest;
|
|
91
|
+
}
|
|
92
|
+
/** Classifies a single claim record against `now`. Definition-free (reads only the stored deadline). */
|
|
93
|
+
export function classifyClaim(claim, now) {
|
|
94
|
+
if (claim === undefined || claim.deadline === null)
|
|
95
|
+
return 'claim_unknown_age';
|
|
96
|
+
return now.getTime() > new Date(claim.deadline).getTime() ? 'claim_stale' : 'healthy';
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Classifies every in-progress claim on a run. Definition-free — works even when the workflow
|
|
100
|
+
* definition is unresolved (detection reads the stored per-claim deadline, not the definition).
|
|
101
|
+
*/
|
|
102
|
+
export function classifyInProgressClaims(run, now = new Date()) {
|
|
103
|
+
return run.in_progress_steps.map((step) => {
|
|
104
|
+
const claim = run.claims?.[step];
|
|
105
|
+
return { step, state: classifyClaim(claim, now), deadline: claim?.deadline ?? null };
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=claim-liveness.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"claim-liveness.js","sourceRoot":"","sources":["../../src/engine/claim-liveness.ts"],"names":[],"mappings":"AAaA,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,iCAAiC,GAAG,IAAI,CAAC,CAAC,SAAS;AAEhE;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAoB;IACvD,OAAO,IAAI,CAAC,SAAS,KAAK,MAAM,CAAC;AACnC,CAAC;AAED,+FAA+F;AAC/F,oFAAoF;AAEpF;;;GAGG;AACH,MAAM,CAAC,MAAM,4BAA4B,GAAG,GAAG,CAAC,CAAC,QAAQ;AACzD,qGAAqG;AACrG,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,CAAC,CAAC,QAAQ;AAClD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,GAAG,CAAC,CAAC,SAAS;AAEnD;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,oBAAoB,CAClC,UAA8B,EAC9B,QAAgB,EAChB,GAAS;IAET,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,WAAW,CAAC,CAAC;IAC/F,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,aAAa;QAAE,OAAO,IAAI,CAAC;IAC5D,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC;IACxC,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,IAAI,iCAAiC,CAAC;IAChF,gGAAgG;IAChG,qFAAqF;IACrF,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,UAAU,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;IACjF,CAAC;IACD,MAAM,YAAY,GAAG,CAAC,GAAG,aAAa,GAAG,UAAU,CAAC;IACpD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,YAAY,GAAG,sBAAsB,CAAC,CAAC;IAC9F,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,cAAc,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;AACvE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,SAAS,CACvB,MAA+C,EAC/C,QAAgB;IAEhB,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACpC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,MAAM,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC;AAWD,wGAAwG;AACxG,MAAM,UAAU,aAAa,CAAC,KAA8B,EAAE,GAAS;IACrE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;QAAE,OAAO,mBAAmB,CAAC;IAC/E,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC;AACxF,CAAC;AASD;;;GAGG;AACH,MAAM,UAAU,wBAAwB,CACtC,GAAc,EACd,MAAY,IAAI,IAAI,EAAE;IAEtB,OAAO,GAAG,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;QACjC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,IAAI,IAAI,EAAE,CAAC;IACvF,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"execution-loop.d.ts","sourceRoot":"","sources":["../../src/engine/execution-loop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,SAAS,EAIT,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAE5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,kBAAkB,
|
|
1
|
+
{"version":3,"file":"execution-loop.d.ts","sourceRoot":"","sources":["../../src/engine/execution-loop.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,SAAS,EAIT,eAAe,EAChB,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAE5D,OAAO,KAAK,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAClF,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,KAAK,EACV,kBAAkB,EAMnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,KAAK,EAAE,gBAAgB,EAAiB,MAAM,gCAAgC,CAAC;AAStF,OAAO,EAAE,eAAe,EAA0C,MAAM,gBAAgB,CAAC;AAYzF,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAoB9D,MAAM,MAAM,cAAc,GAAG,CAC3B,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC9B,GAAG,EAAE,SAAS,EACd,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;AAEtC,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,cAAc,EAAE,CAAA;KAAE,CAAC;IAC5C;;;;OAIG;IACH,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf;;;;;;OAMG;IACH,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC9B;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,UAAU,EAAE,cAAc,CAAC;IAC3B,uCAAuC;IACvC,QAAQ,CAAC,EAAE,iBAAiB,CAAC;IAC7B;;;;OAIG;IACH,QAAQ,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,cAAc,EAAE,CAAA;KAAE,CAAC;IAC5C,oCAAoC;IACpC,KAAK,CAAC,EAAE,eAAe,EAAE,CAAC;IAC1B,+CAA+C;IAC/C,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAgaD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,UAAU,EAAE,kBAAkB,EAAE,GAAG,EAAE,SAAS,GAAG,UAAU,EAAE,CAwB7F;AAuCD;;;;GAIG;AACH,wBAAgB,8BAA8B,CAC5C,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,MAAM,EAClB,GAAG,EAAE,aAAa,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,gBAAgB,CAkBlB;AAiCD;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,kBAAkB,EAC9B,OAAO,EAAE,kBAAkB,GAC1B,OAAO,CAAC,gBAAgB,CAAC,CA++B3B;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,kBAAkB,EAC9B,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,gBAAgB,CAAC,CAmL3B;AAwbD;;;;GAIG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,kBAAkB,EAC9B,OAAO,EAAE,mBAAmB,GAC3B,OAAO,CAAC,gBAAgB,CAAC,CAiD3B;AAGD,OAAO,EAAE,eAAe,IAAI,eAAe,EAAE,CAAC"}
|
|
@@ -4,13 +4,16 @@ import { captureEvidence } from '../evidence/snapshot.js';
|
|
|
4
4
|
import { validateInputSchema, validateOutputSchema, validateTraceSchema, } from '../validation/input-schema.js';
|
|
5
5
|
import { normalizeTrace } from './trace-normalizer.js';
|
|
6
6
|
import { TERMINAL_PHASES, isTerminalPhase, DRAIN_CEILING_SECONDS } from './lifecycle.js';
|
|
7
|
+
import { omitClaim, shouldEnforceTimeout, DEFAULT_EXECUTION_TIMEOUT_SECONDS, } from './claim-liveness.js';
|
|
8
|
+
import { computeBackoff } from './backoff.js';
|
|
7
9
|
import { checkPreconditions, evaluateAllPreconditions, evaluateGuardConditions, } from './precondition.js';
|
|
8
10
|
import { ExtensionRegistry } from '../extensions/registry.js';
|
|
9
11
|
import { createDefaultRegistry } from '../extensions/default-registry.js';
|
|
10
12
|
import { renderTemplate, resolvePath, UnknownFilterError } from './render-template.js';
|
|
11
13
|
import { generateSchemaSkeleton } from '../utils/schema-skeleton.js';
|
|
12
14
|
import { loadWorkflowContext } from './workflow-context-loader.js';
|
|
13
|
-
import { findEligibleSteps, findEligibleGuardSteps, isWorkflowComplete, buildEvidenceByStep, propagateSkips, } from './eligibility.js';
|
|
15
|
+
import { findEligibleSteps, findEligibleGuardSteps, isWorkflowComplete, buildEvidenceByStep, propagateSkips, deriveRunPhase, } from './eligibility.js';
|
|
16
|
+
import { requirementForStep } from './capability.js';
|
|
14
17
|
import { resolvePreExecutionAgentAction, resolvePostDispatchAgentAction, } from './error-resolution.js';
|
|
15
18
|
function delayMs(ms) {
|
|
16
19
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -83,23 +86,6 @@ function resolveInputMapNode(node, root, keyChain, depth) {
|
|
|
83
86
|
}
|
|
84
87
|
return result;
|
|
85
88
|
}
|
|
86
|
-
/** Computes the delay (ms) before a retry attempt based on the configured backoff strategy. */
|
|
87
|
-
function computeBackoff(config, attemptNum) {
|
|
88
|
-
const backoff = config.backoff ?? 'fixed';
|
|
89
|
-
const base = config.base_delay_ms ?? 0;
|
|
90
|
-
let delay;
|
|
91
|
-
switch (backoff) {
|
|
92
|
-
case 'linear':
|
|
93
|
-
delay = base * attemptNum;
|
|
94
|
-
break;
|
|
95
|
-
case 'exponential':
|
|
96
|
-
delay = base * Math.pow(2, attemptNum - 1);
|
|
97
|
-
break;
|
|
98
|
-
default: // 'fixed'
|
|
99
|
-
delay = base;
|
|
100
|
-
}
|
|
101
|
-
return config.max_delay_ms !== undefined ? Math.min(delay, config.max_delay_ms) : delay;
|
|
102
|
-
}
|
|
103
89
|
/**
|
|
104
90
|
* Resolves and calls the service adapter for an auto step with `uses_service`.
|
|
105
91
|
*
|
|
@@ -130,7 +116,9 @@ async function callAdapter(stepDef, definition, options, pendingRun, rateLimiter
|
|
|
130
116
|
? `. Declare this adapter under 'adapters:' in realm.yaml at your deployment root.`
|
|
131
117
|
: '';
|
|
132
118
|
throw new WorkflowError(`Adapter '${serviceDef.adapter}' for service '${serviceName}' is not registered${extensionHint}`, {
|
|
133
|
-
|
|
119
|
+
// #134 discriminator: minted at the NOT-REGISTERED site ONLY (not the service-not-found or
|
|
120
|
+
// adapter-runtime throws), so Step 5 can settle this RECOVERABLY instead of terminal-burning.
|
|
121
|
+
code: 'ENGINE_ADAPTER_NOT_REGISTERED',
|
|
134
122
|
category: 'ENGINE',
|
|
135
123
|
agentAction: 'stop',
|
|
136
124
|
retryable: false,
|
|
@@ -252,7 +240,9 @@ async function callHandler(stepDef, options, pendingRun, evidenceByStep, signal)
|
|
|
252
240
|
const handler = (options.registry ?? createDefaultRegistry()).getHandler(handlerName);
|
|
253
241
|
if (handler === undefined) {
|
|
254
242
|
throw new WorkflowError(`Handler '${handlerName}' is not registered`, {
|
|
255
|
-
|
|
243
|
+
// #134 discriminator: minted at the NOT-REGISTERED site ONLY (not the ran-and-threw throw
|
|
244
|
+
// below), so Step 5 can settle this RECOVERABLY instead of terminal-burning the step.
|
|
245
|
+
code: 'ENGINE_HANDLER_NOT_REGISTERED',
|
|
256
246
|
category: 'ENGINE',
|
|
257
247
|
agentAction: 'stop',
|
|
258
248
|
retryable: false,
|
|
@@ -684,7 +674,17 @@ export async function executeStep(store, definition, options) {
|
|
|
684
674
|
// Step 4: Dispatch with retry and timeout.
|
|
685
675
|
const retryConfig = stepDef?.retry;
|
|
686
676
|
const maxAttempts = retryConfig?.max_attempts ?? 1;
|
|
687
|
-
|
|
677
|
+
// A3: every `execution: 'auto'` step is bounded — authored timeout_seconds if declared, else the
|
|
678
|
+
// generous DEFAULT_EXECUTION_TIMEOUT_SECONDS default. Resolved ONCE here (before the retry loop
|
|
679
|
+
// below), not per-attempt. Agent/guard steps (shouldEnforceTimeout false) are untouched: agent
|
|
680
|
+
// dispatch stays the instant-return no-op it always was, never wrapped in withTimeout.
|
|
681
|
+
// effectiveTimeoutSeconds is the single source of truth; timeoutMs is derived from it so the
|
|
682
|
+
// two can never diverge. It is also surfaced onto the evidence snapshot below.
|
|
683
|
+
const enforceTimeout = stepDef !== undefined && shouldEnforceTimeout(stepDef);
|
|
684
|
+
const effectiveTimeoutSeconds = enforceTimeout
|
|
685
|
+
? (stepDef.timeout_seconds ?? DEFAULT_EXECUTION_TIMEOUT_SECONDS)
|
|
686
|
+
: undefined;
|
|
687
|
+
const timeoutMs = effectiveTimeoutSeconds !== undefined ? effectiveTimeoutSeconds * 1000 : undefined;
|
|
688
688
|
// Create a stable rate-limiter registry for all retry attempts of this step.
|
|
689
689
|
// Shared state ensures that a pause() triggered on attempt N is still in effect
|
|
690
690
|
// when the proactive acquire() runs on attempt N+1. When the caller provides an
|
|
@@ -754,6 +754,8 @@ export async function executeStep(store, definition, options) {
|
|
|
754
754
|
const withHandlerSkipped = {
|
|
755
755
|
...pendingRun,
|
|
756
756
|
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
757
|
+
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
758
|
+
claims: omitClaim(pendingRun.claims, options.command),
|
|
757
759
|
evidence: [...pendingRun.evidence, abortEvidence],
|
|
758
760
|
skipped_steps: [...pendingRun.skipped_steps, options.command],
|
|
759
761
|
};
|
|
@@ -839,6 +841,7 @@ export async function executeStep(store, definition, options) {
|
|
|
839
841
|
...(options.stepMeta?.toolCalls !== undefined
|
|
840
842
|
? { toolCalls: options.stepMeta.toolCalls }
|
|
841
843
|
: {}),
|
|
844
|
+
...(effectiveTimeoutSeconds !== undefined ? { effectiveTimeoutSeconds } : {}),
|
|
842
845
|
// Gate trace to agent steps only — drop silently for auto/adapter/handler steps.
|
|
843
846
|
// When pre-normalized (WAL merge + schema validation ran), pass the pre-normalized
|
|
844
847
|
// result to avoid double normalization. Also handle WAL-only case (options.trace may
|
|
@@ -871,25 +874,122 @@ export async function executeStep(store, definition, options) {
|
|
|
871
874
|
}
|
|
872
875
|
if (dispatchError !== null && retryConfig !== undefined && attemptsUsed === maxAttempts) {
|
|
873
876
|
const lastError = dispatchError;
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
877
|
+
// #134: do NOT wrap a recoverable-incapability error (a max_attempts:1 not-registered failure
|
|
878
|
+
// hits attemptsUsed === maxAttempts). The STEP_RETRY_EXHAUSTED wrap discards the inner code, which
|
|
879
|
+
// would rob Step 5 of the discriminator it needs to settle recoverably. Leave dispatchError as the
|
|
880
|
+
// original not-registered error; all other codes wrap unchanged.
|
|
881
|
+
const isRecoverableIncapability = lastError instanceof WorkflowError &&
|
|
882
|
+
(lastError.code === 'ENGINE_HANDLER_NOT_REGISTERED' ||
|
|
883
|
+
lastError.code === 'ENGINE_ADAPTER_NOT_REGISTERED');
|
|
884
|
+
if (!isRecoverableIncapability) {
|
|
885
|
+
dispatchError = new WorkflowError(`Step '${options.command}' failed after ${attemptsUsed} attempts`, {
|
|
886
|
+
code: 'STEP_RETRY_EXHAUSTED',
|
|
887
|
+
category: 'ENGINE',
|
|
888
|
+
agentAction: 'report_to_user',
|
|
889
|
+
retryable: false,
|
|
890
|
+
details: {
|
|
891
|
+
stepName: options.command,
|
|
892
|
+
attempts: attemptsUsed,
|
|
893
|
+
lastError: lastError.message,
|
|
894
|
+
...(lastError.retry_after !== undefined ? { retry_after: lastError.retry_after } : {}),
|
|
895
|
+
},
|
|
896
|
+
});
|
|
897
|
+
}
|
|
886
898
|
}
|
|
887
899
|
// Step 5: Handle dispatch failure — move step to failed_steps.
|
|
888
900
|
if (dispatchError !== null) {
|
|
901
|
+
// #134 recoverable-incapability settle: a NOT-REGISTERED handler/adapter means THIS runner cannot
|
|
902
|
+
// execute the step, but a correctly-provisioned runner can. Terminal-burning it into failed_steps
|
|
903
|
+
// would make it permanently un-reclaimable. Instead settle RECOVERABLY: drop it from in_progress and
|
|
904
|
+
// omit its claim (same mutation), do NOT add it to failed_steps, do NOT seal the run, record a
|
|
905
|
+
// capability_blocks marker for diagnostics, and let it fall back to eligible so a capable runner
|
|
906
|
+
// reclaims it. Genuine ran-and-threw / service-not-found / adapter-runtime failures keep their
|
|
907
|
+
// ENGINE_*_FAILED codes and fall through to the terminal path below unchanged.
|
|
908
|
+
const recoverableCode = dispatchError instanceof WorkflowError &&
|
|
909
|
+
(dispatchError.code === 'ENGINE_HANDLER_NOT_REGISTERED' ||
|
|
910
|
+
dispatchError.code === 'ENGINE_ADAPTER_NOT_REGISTERED')
|
|
911
|
+
? dispatchError.code
|
|
912
|
+
: undefined;
|
|
913
|
+
if (recoverableCode !== undefined) {
|
|
914
|
+
const requirement = requirementForStep(options.command, stepDef, definition);
|
|
915
|
+
const blockedDraft = {
|
|
916
|
+
...pendingRun,
|
|
917
|
+
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
918
|
+
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
919
|
+
claims: omitClaim(pendingRun.claims, options.command),
|
|
920
|
+
evidence: [...pendingRun.evidence, ...allEvidence],
|
|
921
|
+
capability_blocks: {
|
|
922
|
+
...pendingRun.capability_blocks,
|
|
923
|
+
[options.command]: {
|
|
924
|
+
requirement: requirement !== undefined
|
|
925
|
+
? { kind: requirement.kind, name: requirement.name }
|
|
926
|
+
: {
|
|
927
|
+
kind: recoverableCode === 'ENGINE_HANDLER_NOT_REGISTERED' ? 'handler' : 'adapter',
|
|
928
|
+
name: 'unknown',
|
|
929
|
+
},
|
|
930
|
+
code: recoverableCode,
|
|
931
|
+
at: new Date().toISOString(),
|
|
932
|
+
},
|
|
933
|
+
},
|
|
934
|
+
};
|
|
935
|
+
// Non-terminal: recompute the phase so the store-fail fallback below is correct too
|
|
936
|
+
// (on the happy path store.update recomputes it identically via deriveRunPhase).
|
|
937
|
+
const blockedRun = { ...blockedDraft, run_phase: deriveRunPhase(blockedDraft) };
|
|
938
|
+
let persistedBlockedRun;
|
|
939
|
+
let blockStoreWarning;
|
|
940
|
+
try {
|
|
941
|
+
persistedBlockedRun = await store.update(blockedRun);
|
|
942
|
+
}
|
|
943
|
+
catch (storeErr) {
|
|
944
|
+
blockStoreWarning = `Failed to persist capability block: ${storeErr instanceof Error ? storeErr.message : String(storeErr)}`;
|
|
945
|
+
}
|
|
946
|
+
let blockWalWarning;
|
|
947
|
+
try {
|
|
948
|
+
// Delete WAL after run state is written — the step's entries are now in evidence.
|
|
949
|
+
await options.traceBufferStore?.delete(options.runId, options.command);
|
|
950
|
+
}
|
|
951
|
+
catch (walErr) {
|
|
952
|
+
blockWalWarning = `Failed to clean up trace buffer after capability block: ${walErr instanceof Error ? walErr.message : String(walErr)}`;
|
|
953
|
+
}
|
|
954
|
+
// Non-terminal 'stop' → 'report_to_user' via the existing mapping: a human must provision the
|
|
955
|
+
// runner (or re-run on a capable one); no further progress is possible on THIS runner.
|
|
956
|
+
const blockedAction = resolvePostDispatchAgentAction(dispatchError, false);
|
|
957
|
+
let blockedNextActions = [];
|
|
958
|
+
if (blockedAction !== 'stop' && blockStoreWarning === undefined) {
|
|
959
|
+
try {
|
|
960
|
+
blockedNextActions = buildNextActions(definition, persistedBlockedRun ?? blockedRun);
|
|
961
|
+
}
|
|
962
|
+
catch {
|
|
963
|
+
// buildNextActions can throw for unresolvable template references; fall back to [].
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
const reqLabel = requirement !== undefined
|
|
967
|
+
? `${requirement.kind} '${requirement.name}'`
|
|
968
|
+
: recoverableCode === 'ENGINE_HANDLER_NOT_REGISTERED'
|
|
969
|
+
? 'handler'
|
|
970
|
+
: 'adapter';
|
|
971
|
+
return {
|
|
972
|
+
command: options.command,
|
|
973
|
+
run_id: options.runId,
|
|
974
|
+
run_version: (persistedBlockedRun ?? blockedRun).version,
|
|
975
|
+
status: 'error',
|
|
976
|
+
data: {},
|
|
977
|
+
evidence: allEvidence,
|
|
978
|
+
warnings: mergeWarnings(traceWarnings, blockStoreWarning ?? blockWalWarning),
|
|
979
|
+
errors: [dispatchError.message],
|
|
980
|
+
agent_action: blockedAction,
|
|
981
|
+
error_code: recoverableCode,
|
|
982
|
+
context_hint: `Step '${options.command}' is blocked: its ${reqLabel} is not registered in this runner. The run is NOT terminated — the step remains eligible, so a runner that provides this ${requirement?.kind ?? 'capability'} can execute it. Provision this runner (or re-run on a capable one), then follow next_actions.`,
|
|
983
|
+
run_phase: (persistedBlockedRun ?? blockedRun).run_phase,
|
|
984
|
+
next_actions: blockedNextActions,
|
|
985
|
+
};
|
|
986
|
+
}
|
|
889
987
|
// Pure in-memory derivations — no I/O, no try required.
|
|
890
988
|
const afterFail = {
|
|
891
989
|
...pendingRun,
|
|
892
990
|
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
991
|
+
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
992
|
+
claims: omitClaim(pendingRun.claims, options.command),
|
|
893
993
|
failed_steps: [...pendingRun.failed_steps, options.command],
|
|
894
994
|
};
|
|
895
995
|
// Propagate skips: mark steps whose trigger_rule can never be satisfied after this failure.
|
|
@@ -1117,6 +1217,8 @@ export async function executeStep(store, definition, options) {
|
|
|
1117
1217
|
const afterComplete = {
|
|
1118
1218
|
...pendingRun,
|
|
1119
1219
|
in_progress_steps: pendingRun.in_progress_steps.filter((s) => s !== options.command),
|
|
1220
|
+
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
1221
|
+
claims: omitClaim(pendingRun.claims, options.command),
|
|
1120
1222
|
completed_steps: [...pendingRun.completed_steps, options.command],
|
|
1121
1223
|
evidence: [...pendingRun.evidence, ...allEvidence],
|
|
1122
1224
|
};
|
|
@@ -1262,6 +1364,8 @@ export async function submitHumanResponse(store, definition, options) {
|
|
|
1262
1364
|
const afterGate = {
|
|
1263
1365
|
...rest,
|
|
1264
1366
|
in_progress_steps: rest.in_progress_steps.filter((s) => s !== gateStepName),
|
|
1367
|
+
// Delete the claim clock in the SAME mutation that removes the step (issue #101).
|
|
1368
|
+
claims: omitClaim(rest.claims, gateStepName),
|
|
1265
1369
|
completed_steps: [...rest.completed_steps, gateStepName],
|
|
1266
1370
|
evidence: [...rest.evidence, gateSnapshot],
|
|
1267
1371
|
};
|