@dzhechkov/harness-core 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dz-manifest.json +339 -75
- package/README.md +47 -4
- package/dist/codex-hooks-assets.d.ts.map +1 -1
- package/dist/codex-hooks-assets.js +39 -2
- package/dist/codex-hooks-assets.js.map +1 -1
- package/dist/codex-hooks-verify.d.ts +23 -2
- package/dist/codex-hooks-verify.d.ts.map +1 -1
- package/dist/codex-hooks-verify.js +29 -0
- package/dist/codex-hooks-verify.js.map +1 -1
- package/dist/codex-hooks.d.ts +90 -7
- package/dist/codex-hooks.d.ts.map +1 -1
- package/dist/codex-hooks.js +171 -21
- package/dist/codex-hooks.js.map +1 -1
- package/dist/feature-adr-routing.d.ts +22 -0
- package/dist/feature-adr-routing.d.ts.map +1 -1
- package/dist/feature-adr-routing.js +45 -0
- package/dist/feature-adr-routing.js.map +1 -1
- package/dist/index.d.ts +11 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -2
- package/dist/index.js.map +1 -1
- package/dist/loop-blobs.generated.d.ts +1 -1
- package/dist/loop-blobs.generated.d.ts.map +1 -1
- package/dist/loop-blobs.generated.js +12 -3
- package/dist/loop-blobs.generated.js.map +1 -1
- package/dist/loop-plan.d.ts +70 -0
- package/dist/loop-plan.d.ts.map +1 -1
- package/dist/loop-plan.js +103 -0
- package/dist/loop-plan.js.map +1 -1
- package/dist/loop-render.d.ts.map +1 -1
- package/dist/loop-render.js +38 -130
- package/dist/loop-render.js.map +1 -1
- package/dist/loop-run-semantics.d.ts +130 -0
- package/dist/loop-run-semantics.d.ts.map +1 -0
- package/dist/loop-run-semantics.js +257 -0
- package/dist/loop-run-semantics.js.map +1 -0
- package/dist/loop-trace.d.ts +106 -5
- package/dist/loop-trace.d.ts.map +1 -1
- package/dist/loop-trace.js +151 -18
- package/dist/loop-trace.js.map +1 -1
- package/dist/managed-hooks.d.ts +10 -0
- package/dist/managed-hooks.d.ts.map +1 -1
- package/dist/managed-hooks.js +17 -5
- package/dist/managed-hooks.js.map +1 -1
- package/dist/named-lock.d.ts +57 -0
- package/dist/named-lock.d.ts.map +1 -0
- package/dist/named-lock.js +247 -0
- package/dist/named-lock.js.map +1 -0
- package/dist/operations.d.ts +81 -5
- package/dist/operations.d.ts.map +1 -1
- package/dist/operations.js +356 -38
- package/dist/operations.js.map +1 -1
- package/dist/parity.d.ts +70 -2
- package/dist/parity.d.ts.map +1 -1
- package/dist/parity.js +133 -2
- package/dist/parity.js.map +1 -1
- package/dist/qe-bridge.d.ts +291 -0
- package/dist/qe-bridge.d.ts.map +1 -0
- package/dist/qe-bridge.js +538 -0
- package/dist/qe-bridge.js.map +1 -0
- package/dist/score.d.ts.map +1 -1
- package/dist/score.js +43 -9
- package/dist/score.js.map +1 -1
- package/dist/trace-corroborate.d.ts +48 -0
- package/dist/trace-corroborate.d.ts.map +1 -0
- package/dist/trace-corroborate.js +172 -0
- package/dist/trace-corroborate.js.map +1 -0
- package/dist/workflow-run-dispatch.d.ts +230 -0
- package/dist/workflow-run-dispatch.d.ts.map +1 -0
- package/dist/workflow-run-dispatch.js +363 -0
- package/dist/workflow-run-dispatch.js.map +1 -0
- package/dist/workflow-run.d.ts +513 -0
- package/dist/workflow-run.d.ts.map +1 -0
- package/dist/workflow-run.js +1377 -0
- package/dist/workflow-run.js.map +1 -0
- package/package.json +2 -2
- package/sbom.json +740 -80
- package/src/codex-hooks-assets.ts +39 -2
- package/src/codex-hooks-verify.ts +55 -2
- package/src/codex-hooks.ts +172 -20
- package/src/feature-adr-routing.ts +55 -0
- package/src/index.ts +46 -1
- package/src/loop-blobs.generated.ts +12 -3
- package/src/loop-plan.ts +185 -0
- package/src/loop-render.ts +38 -128
- package/src/loop-run-semantics.ts +278 -0
- package/src/loop-trace.ts +207 -16
- package/src/managed-hooks.ts +26 -5
- package/src/named-lock.ts +277 -0
- package/src/operations.ts +441 -40
- package/src/parity.ts +177 -2
- package/src/qe-bridge.ts +737 -0
- package/src/score.ts +50 -9
- package/src/trace-corroborate.ts +205 -0
- package/src/workflow-run-dispatch.ts +459 -0
- package/src/workflow-run.ts +1773 -0
|
@@ -0,0 +1,1377 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `workflow-run` — the PURE scheduler half of `dz workflow run` (feature dz-workflow-run).
|
|
3
|
+
*
|
|
4
|
+
* `dz workflow run` INTERPRETS a `loop-plan/1` plan; it never executes the rendered Claude-host
|
|
5
|
+
* script (ADR-001). That is the whole design in one sentence, and everything in this file follows
|
|
6
|
+
* from it: the plan is read through a PROJECTION (`toRunProjection` — the AM-3 doctrine, now with a
|
|
7
|
+
* second enactor), the enactment DECISIONS come from the one shared module the rendered script also
|
|
8
|
+
* carries as a blob (`loop-run-semantics`), and the trace is emitted through the SAME validated
|
|
9
|
+
* grammar the Claude host uses (`loop-trace`), so a divergent event cannot even be buffered.
|
|
10
|
+
*
|
|
11
|
+
* PURITY (NFR-3): everything here is deterministic under injected seams — a `Dispatcher` per family,
|
|
12
|
+
* a `RunStore` for every byte that touches a disk, an injected clock and an injected runId. No
|
|
13
|
+
* `spawn`, no `fs`, no `Date`, no randomness. That is what buys the determinism test, the
|
|
14
|
+
* replay-derived discrimination leg, the landed-barrier mutant and all 22 taxonomy producers
|
|
15
|
+
* WITHOUT spawning a single child.
|
|
16
|
+
*
|
|
17
|
+
* The three refusal planes, in the order a run meets them:
|
|
18
|
+
* 1. PREFLIGHT — decidable before anything is spawned (no trace plane, unroutable model, a write
|
|
19
|
+
* that escapes the root, same-family QE, an operator override that cannot fit a boundary).
|
|
20
|
+
* 2. RESUME — decidable from state + checkpoints + artifact probes, never from a step list.
|
|
21
|
+
* 3. RUNTIME — dispatch failures, gate verdicts, deliverables that did not land, budget.
|
|
22
|
+
* Every refusal names itself from ONE closed list (`WF_RUN_REASONS`), and the list is the count
|
|
23
|
+
* authority a reachability suite walks.
|
|
24
|
+
*/
|
|
25
|
+
import { toRunProjection, planDigest } from './loop-plan.js';
|
|
26
|
+
import { computeExecAxisInputs, computeExecFingerprint } from './loop-render.js';
|
|
27
|
+
import { parseTrace, traceClose, traceDrain, traceInit, traceLedgerLine, traceOnDispatch, traceOnSettle, traceShellQuote, TRACE_KEY_RE, } from './loop-trace.js';
|
|
28
|
+
import { classifyFailure, errSnap, gateVerdict, joinRegion, stepContractLines } from './loop-run-semantics.js';
|
|
29
|
+
import { checkpointInputHash, decideCheckpointResume, parseCheckpointRead, serializeCheckpoint } from './feature-adr-checkpoints.js';
|
|
30
|
+
import { modelFamily } from './qe-bridge.js';
|
|
31
|
+
import { CODEX_EXEC_XHIGH_TIMEOUT_MS, defangGateEchoes } from './workflow-run-dispatch.js';
|
|
32
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
33
|
+
// Schemas / constants
|
|
34
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
35
|
+
export const WF_RUN_STATE_SCHEMA = 'wf-run-state/1';
|
|
36
|
+
export const WF_BUDGET_ROW_SCHEMA = 'wf-budget-1';
|
|
37
|
+
export const WF_PAUSE_ENVELOPE_SCHEMA = 'wf-pause-envelope/1';
|
|
38
|
+
export const WF_RUN_RESULT_SCHEMA = 'wf-run-result/1';
|
|
39
|
+
export const WF_RUN_OWNER_HOST = 'dz-workflow-run';
|
|
40
|
+
/** ADR-004 W9 — a DECLARED GUESS, deliberately ONE exported constant so calibration is a one-line
|
|
41
|
+
* change with a name, not a number sprinkled through the scheduler. */
|
|
42
|
+
export const WALLCLOCK_CEILING_MULTIPLIER = 1.5;
|
|
43
|
+
/** 75 = sysexits EX_TEMPFAIL ("try again later"). NOT 3: that collides with workflow-lint's
|
|
44
|
+
* inconclusive and reads ignorable, while a pause strands resumable progress (AM-11). */
|
|
45
|
+
export const WF_EXIT = { completed: 0, failed: 1, usage: 2, pause: 75 };
|
|
46
|
+
/**
|
|
47
|
+
* The CLOSED reason set (AM-1 + AM-15 + AM-19) as DATA — the single count authority. The
|
|
48
|
+
* reachability suite WALKS this array: a member no in-suite scenario can produce fails the suite, so
|
|
49
|
+
* the list can never quietly grow a decorative member or lose a real one. Every count written in
|
|
50
|
+
* prose anywhere is a DESCRIPTION of this array, never a second authority.
|
|
51
|
+
*/
|
|
52
|
+
export const WF_RUN_REASONS = [
|
|
53
|
+
'plan-invalid', 'trace-emit-required', 'plan-model-unroutable', 'artifact-path-escapes-root',
|
|
54
|
+
'probe-failed', 'dispatch-timeout', 'dispatch-dead', 'gate-verdict-unparseable',
|
|
55
|
+
'deliverable-not-landed', 'same-family-qe-refused', 'prompt-over-ceiling',
|
|
56
|
+
'stale-input-refused', 'resume-model-unavailable', 'foreign-run-refused', 'run-exists',
|
|
57
|
+
'run-locked', 'budget-exhausted', 'budget-extension-exhausted', 'budget-invariant-violated',
|
|
58
|
+
'resume-already-completed', 'wall-extension-exhausted', 'reservation-unsatisfiable',
|
|
59
|
+
// AM-19: a PARSED `GATE: FAIL` with its routing exhausted is not an unparseable verdict. The
|
|
60
|
+
// model answered clearly; the plan declared nowhere for the answer to go. Two producers, two
|
|
61
|
+
// members — collapsing them would tell an operator the model produced garbage when it did not.
|
|
62
|
+
'gate-failed',
|
|
63
|
+
// AM-20 (Step-8 HIGH-7): a plan-declared pause is a first-class RESULT of a run, and AM-16
|
|
64
|
+
// requires the envelope's `reason` to be a member of THIS list. It was emitting the free string
|
|
65
|
+
// `typed-pause`, which is exactly the "closed set with an escape hatch" shape the taxonomy exists
|
|
66
|
+
// to forbid — a wrapper switching on the list would have fallen through.
|
|
67
|
+
'plan-pause',
|
|
68
|
+
];
|
|
69
|
+
/**
|
|
70
|
+
* The members whose PRODUCER lives in the impure half (`dz workflow run`'s CLI): a run DIRECTORY
|
|
71
|
+
* that already holds a run, and a LIVE owner marker. Neither is decidable without a filesystem, so
|
|
72
|
+
* neither can be produced by the core suite.
|
|
73
|
+
*
|
|
74
|
+
* Exported as DATA so the reachability proof spans both packages WITHOUT either side restating the
|
|
75
|
+
* other's list: the core suite asserts `produced-in-core ∪ this === WF_RUN_REASONS`, and the CLI
|
|
76
|
+
* suite asserts it produces every member of exactly this array. Together that is a real 23/23 walk;
|
|
77
|
+
* a member that fell out of both halves would fail the core union check, and a member added here
|
|
78
|
+
* without a CLI producer would fail the CLI check.
|
|
79
|
+
*/
|
|
80
|
+
export const WF_RUN_REASONS_CLI_PRODUCED = ['run-exists', 'run-locked'];
|
|
81
|
+
/** Small, dependency-free 64-bit FNV — the same shape the checkpoint plane uses, kept local so this
|
|
82
|
+
* module has no reason to reach for a crypto import in a file that must stay trivially pure. */
|
|
83
|
+
function fnv64(text) {
|
|
84
|
+
let h1 = 0x811c9dc5;
|
|
85
|
+
let h2 = 0x01000193;
|
|
86
|
+
for (let i = 0; i < text.length; i++) {
|
|
87
|
+
const c = text.charCodeAt(i);
|
|
88
|
+
h1 = Math.imul(h1 ^ c, 0x01000193) >>> 0;
|
|
89
|
+
h2 = Math.imul(h2 ^ (c + i), 0x85ebca6b) >>> 0;
|
|
90
|
+
}
|
|
91
|
+
return (h1 >>> 0).toString(16).padStart(8, '0') + (h2 >>> 0).toString(16).padStart(8, '0');
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* The run-args identity hash, over the inputs MINUS an ENUMERATED exclusion list (AM-13 / W12).
|
|
95
|
+
*
|
|
96
|
+
* The exclusions are the whole point and they are returned as DATA so a test can pin them rather
|
|
97
|
+
* than restate them: the plan-declared `resumeArg` keys (supplying one is what a resume IS),
|
|
98
|
+
* `budgetExtra` and `wallClockExtra` (extending a ceiling is a resume-plane input, not a different
|
|
99
|
+
* run). Everything else — including a resume arg the plan never declared — is IDENTITY, so changing
|
|
100
|
+
* it makes the resume stale rather than silently continuing a different run.
|
|
101
|
+
*/
|
|
102
|
+
export function computeRunArgsHash(inputs, proj) {
|
|
103
|
+
const excluded = [...proj.resumeArgKeys, 'budgetExtra', 'wallClockExtra'].sort();
|
|
104
|
+
const declared = new Set(proj.resumeArgKeys);
|
|
105
|
+
const identityArgs = Object.keys(inputs.resumeArgs)
|
|
106
|
+
.filter((k) => !declared.has(k))
|
|
107
|
+
.sort()
|
|
108
|
+
.map((k) => [k, inputs.resumeArgs[k] ?? null]);
|
|
109
|
+
const tuple = [
|
|
110
|
+
'wf-run-args/1',
|
|
111
|
+
inputs.runId,
|
|
112
|
+
inputs.coderFamily,
|
|
113
|
+
inputs.allowSameFamilyQe,
|
|
114
|
+
inputs.defaultFamily,
|
|
115
|
+
inputs.budgetOverride,
|
|
116
|
+
inputs.maxWallClockMsOverride,
|
|
117
|
+
inputs.stageTimeoutMsOverride,
|
|
118
|
+
inputs.cwdRoot,
|
|
119
|
+
identityArgs,
|
|
120
|
+
];
|
|
121
|
+
return { hash: fnv64(JSON.stringify(tuple)), excluded };
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Worst-case reservation PER BOUNDARY (AM-4: a region is reserved as a whole and never interrupted,
|
|
125
|
+
* which is what makes "pause BEFORE the region" expressible at all).
|
|
126
|
+
*
|
|
127
|
+
* • stage — the step's declared `maxAgents`;
|
|
128
|
+
* • gate — the gate's own allowance PLUS the declared redo allowance, reserved AT the gate
|
|
129
|
+
* boundary (a plan-declared redo must be affordable where it is spent, not somewhere
|
|
130
|
+
* upstream, or a short budget pauses in the middle of a redo loop);
|
|
131
|
+
* • region — every activated member × the whole chain. This is the number that can legitimately
|
|
132
|
+
* exceed the plan's total: the render's ceiling counts a member step ONCE, the runtime
|
|
133
|
+
* dispatches it per item. Naming the gap here is what makes the extension cap
|
|
134
|
+
* satisfiable instead of a surprise mid-run.
|
|
135
|
+
*/
|
|
136
|
+
export function computeBoundaryReservations(proj, timeoutMsFor) {
|
|
137
|
+
const gateAllowanceOf = (b) => {
|
|
138
|
+
const g = b.stage?.gate;
|
|
139
|
+
if (b.stage === undefined || g === null || g === undefined)
|
|
140
|
+
return 0;
|
|
141
|
+
if (g.maxRedos <= 0 || g.failRoute === null || g.failRoute.startsWith('terminal:'))
|
|
142
|
+
return 0;
|
|
143
|
+
const route = proj.boundaries.find((x) => x.boundaryId === g.failRoute)?.stage;
|
|
144
|
+
return g.maxRedos * ((route === undefined ? 1 : worstCaseInvocations(route)) + worstCaseInvocations(b.stage));
|
|
145
|
+
};
|
|
146
|
+
const gateWallOf = (b) => {
|
|
147
|
+
const g = b.stage?.gate;
|
|
148
|
+
if (b.stage === undefined || g === null || g === undefined)
|
|
149
|
+
return 0;
|
|
150
|
+
if (g.maxRedos <= 0 || g.failRoute === null || g.failRoute.startsWith('terminal:'))
|
|
151
|
+
return 0;
|
|
152
|
+
const route = proj.boundaries.find((x) => x.boundaryId === g.failRoute)?.stage;
|
|
153
|
+
const routeWall = route === undefined ? 0 : worstCaseInvocations(route) * timeoutMsFor(route);
|
|
154
|
+
return g.maxRedos * (routeWall + worstCaseInvocations(b.stage) * timeoutMsFor(b.stage));
|
|
155
|
+
};
|
|
156
|
+
const out = [];
|
|
157
|
+
for (const b of proj.boundaries) {
|
|
158
|
+
if (b.kind === 'pause')
|
|
159
|
+
continue; // a pause dispatches nothing — it reserves nothing
|
|
160
|
+
if (b.kind === 'region') {
|
|
161
|
+
const r = b.region;
|
|
162
|
+
if (r === undefined)
|
|
163
|
+
continue;
|
|
164
|
+
const members = activatedMembers(r.registry, r.dedup, r.maxFanout).length;
|
|
165
|
+
const perItem = r.chain.reduce((n, s) => n + worstCaseInvocations(s), 0);
|
|
166
|
+
const perItemWall = r.chain.reduce((n, s) => n + worstCaseInvocations(s) * timeoutMsFor(s), 0);
|
|
167
|
+
out.push({
|
|
168
|
+
boundaryId: b.boundaryId,
|
|
169
|
+
kind: 'region',
|
|
170
|
+
steps: r.chain.map((s) => s.stepId),
|
|
171
|
+
invocations: members * perItem,
|
|
172
|
+
// a `barrier`/`pipeline` region runs at most maxFanout branches at once, so its worst-case
|
|
173
|
+
// WALL is the serialized work divided by the concurrency bound (never below one full item)
|
|
174
|
+
wallMs: r.maxFanout > 0 ? Math.ceil((members * perItemWall) / r.maxFanout) : members * perItemWall,
|
|
175
|
+
});
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
const s = b.stage;
|
|
179
|
+
if (s === undefined)
|
|
180
|
+
continue;
|
|
181
|
+
out.push({
|
|
182
|
+
boundaryId: b.boundaryId,
|
|
183
|
+
kind: b.kind === 'gate' ? 'gate' : 'stage',
|
|
184
|
+
steps: [s.stepId],
|
|
185
|
+
invocations: worstCaseInvocations(s) + gateAllowanceOf(b),
|
|
186
|
+
wallMs: worstCaseInvocations(s) * timeoutMsFor(s) + gateWallOf(b),
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
return out;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* The members a fanout actually ACTIVATES.
|
|
193
|
+
*
|
|
194
|
+
* `maxFanout` caps ACTIVATION, not merely concurrency — the rendered script emits
|
|
195
|
+
* `registry.slice(0, maxFanout)` ("bounded fanout (INV-2): never args-derived, never uncapped").
|
|
196
|
+
* Getting this wrong is a STRUCTURAL-EQUIVALENCE break, not a performance detail: a runner that
|
|
197
|
+
* treated the bound as a concurrency limit would dispatch all six entries of a six-item registry
|
|
198
|
+
* where the Claude host dispatched three, and the two hosts' traces would disagree on dispatch
|
|
199
|
+
* multiplicity for the same plan. (MEASURED: that is exactly what the F5 equivalence leg caught
|
|
200
|
+
* against the committed `pkg-audit-1` run — 6 lanes here, 3 there.)
|
|
201
|
+
*/
|
|
202
|
+
function activatedMembers(registry, dedup, maxFanout) {
|
|
203
|
+
const base = dedup ? [...new Set(registry)] : [...registry];
|
|
204
|
+
return maxFanout > 0 ? base.slice(0, maxFanout) : base;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* The worst-case invocations ONE occurrence of a step can consume: it cannot attempt more times
|
|
208
|
+
* than its retry profile allows, and the budget guard caps it at its declared allowance. Reserving
|
|
209
|
+
* the raw `maxAgents` would over-reserve by the whole retry headroom a step never asked for — and
|
|
210
|
+
* an over-reservation is not "safe", it is a run that pauses before work it could have afforded.
|
|
211
|
+
*/
|
|
212
|
+
function worstCaseInvocations(s) {
|
|
213
|
+
return Math.max(1, Math.min(s.maxAgents, s.retryMaxAttempts));
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* The extension cap (AM-14, correcting AM-5's doubling): `max(2 × originalTotal, Σ reservations)`.
|
|
217
|
+
*
|
|
218
|
+
* The doubling bound ALONE left a valid plan permanently unresumable — counterexample from the
|
|
219
|
+
* amendment: T=10 with a last boundary needing 25 can never be finished, because the achievable cap
|
|
220
|
+
* is 20 and the prefix already spent some of it. Σ reservations is PREFIX-CLOSED (actual spend per
|
|
221
|
+
* construct never exceeds its own reservation), so default caps are satisfiable BY CONSTRUCTION.
|
|
222
|
+
* The named cost: for attempt-heavy plans the cap is larger than 2×T, so the doubling guarantee is
|
|
223
|
+
* weakened exactly there — the extension records are what keep that audited.
|
|
224
|
+
*/
|
|
225
|
+
export function computeAchievableMax(originalTotal, reservations) {
|
|
226
|
+
const sigma = reservations.reduce((n, r) => n + r.invocations, 0);
|
|
227
|
+
return Math.max(2 * originalTotal, sigma);
|
|
228
|
+
}
|
|
229
|
+
/** ADR-004 W9: Σ worst-case wall × the declared multiplier. The multiplier is a GUESS with a name. */
|
|
230
|
+
export function computeWallClockCeilingMs(reservations) {
|
|
231
|
+
const sigma = reservations.reduce((n, r) => n + r.wallMs, 0);
|
|
232
|
+
return Math.ceil(sigma * WALLCLOCK_CEILING_MULTIPLIER);
|
|
233
|
+
}
|
|
234
|
+
/** The per-stage timeout the run uses: the operator's override, else the measured codex xhigh
|
|
235
|
+
* ceiling (the far end of the distribution — a shorter default would turn slow models into
|
|
236
|
+
* `dispatch-timeout` noise). */
|
|
237
|
+
export function stageTimeoutMs(inputs) {
|
|
238
|
+
const o = inputs.stageTimeoutMsOverride;
|
|
239
|
+
return typeof o === 'number' && Number.isFinite(o) && o > 0 ? Math.floor(o) : CODEX_EXEC_XHIGH_TIMEOUT_MS;
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* A declared artifact path — READ or WRITE — must resolve INSIDE the run root.
|
|
243
|
+
*
|
|
244
|
+
* Step-8 CRITICAL-4 closed two holes at once:
|
|
245
|
+
* - only `writes` were checked, while AM-9 covers reads and writes. A read is a path the runner
|
|
246
|
+
* hands a model and asks it to open; `../../.ssh/id_rsa` is not less dangerous for being a read.
|
|
247
|
+
* - containment was decided by `realpath(fullPath)`, which is NULL for a not-yet-created leaf —
|
|
248
|
+
* the normal case for a write. A nonexistent leaf under a SYMLINKED ANCESTOR therefore passed:
|
|
249
|
+
* nothing resolved, so nothing was compared.
|
|
250
|
+
*
|
|
251
|
+
* The fix is the nearest-existing-ancestor walk the CLI already uses for `--out`: every EXISTING
|
|
252
|
+
* component on the way down is resolved and required to stay under the real root, so a symlink
|
|
253
|
+
* anywhere on the path is caught whether or not the leaf exists yet.
|
|
254
|
+
*
|
|
255
|
+
* (The control-character class is spelled with escapes now. It was written with literal control
|
|
256
|
+
* BYTES, which made the guard invisible to `grep` and one careless editor save away from silently
|
|
257
|
+
* becoming `/[ -]/`.)
|
|
258
|
+
*/
|
|
259
|
+
function escapesRoot(rel, root, deps) {
|
|
260
|
+
if (rel === '')
|
|
261
|
+
return 'an empty path';
|
|
262
|
+
// eslint-disable-next-line no-control-regex
|
|
263
|
+
if (/[\x00-\x1f\x7f]/.test(rel))
|
|
264
|
+
return 'a control character';
|
|
265
|
+
if (rel.startsWith('/'))
|
|
266
|
+
return 'an absolute path';
|
|
267
|
+
const segments = rel.split('/').filter((seg) => seg !== '' && seg !== '.');
|
|
268
|
+
if (segments.some((seg) => seg === '..'))
|
|
269
|
+
return 'a ".." segment';
|
|
270
|
+
if (segments.length === 0)
|
|
271
|
+
return 'an empty path';
|
|
272
|
+
const base = root.replace(/\/+$/, '');
|
|
273
|
+
const realRoot = deps.realpath(base) ?? base;
|
|
274
|
+
const under = (candidate) => candidate === realRoot || candidate.startsWith(realRoot + '/');
|
|
275
|
+
let walked = base;
|
|
276
|
+
for (const seg of segments) {
|
|
277
|
+
walked = walked + '/' + seg;
|
|
278
|
+
if (!deps.exists(walked))
|
|
279
|
+
continue; // nothing to resolve yet — deeper components cannot exist either
|
|
280
|
+
const real = deps.realpath(walked);
|
|
281
|
+
if (real === null)
|
|
282
|
+
return 'a path component that cannot be resolved';
|
|
283
|
+
if (!under(real))
|
|
284
|
+
return 'a component whose real location is outside the run root (a symlinked ancestor)';
|
|
285
|
+
}
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
/** Every dispatching step of a projection, top-level and fanout members alike. */
|
|
289
|
+
function allSpecs(proj) {
|
|
290
|
+
const out = [];
|
|
291
|
+
for (const b of proj.boundaries) {
|
|
292
|
+
if (b.stage !== undefined)
|
|
293
|
+
out.push(b.stage);
|
|
294
|
+
if (b.region !== undefined)
|
|
295
|
+
out.push(...b.region.chain);
|
|
296
|
+
}
|
|
297
|
+
return out;
|
|
298
|
+
}
|
|
299
|
+
/**
|
|
300
|
+
* Refusals in a FIXED order, each one named:
|
|
301
|
+
* trace-emit-required → plan-model-unroutable → artifact-path-escapes-root →
|
|
302
|
+
* same-family-qe-refused (unless waived) → reservation-unsatisfiable.
|
|
303
|
+
*
|
|
304
|
+
* The order is not cosmetic. `trace-emit-required` comes first because a run with no trace plane is
|
|
305
|
+
* unverifiable — refusing it later would mean deciding routing for a run nobody could ever check.
|
|
306
|
+
* `reservation-unsatisfiable` comes last because it is the only refusal that depends on every
|
|
307
|
+
* preceding resolution, and it fires ONLY for an EXPLICIT operator override: the defaults are
|
|
308
|
+
* satisfiable by construction (AM-14), so a default-budget run that cannot fit a boundary pauses and
|
|
309
|
+
* resumes rather than refusing to start.
|
|
310
|
+
*
|
|
311
|
+
* `plan-invalid` never reaches here — the CLI parses and validates first and exits 2.
|
|
312
|
+
*/
|
|
313
|
+
export function preflight(inputs, deps) {
|
|
314
|
+
const proj = toRunProjection(inputs.plan);
|
|
315
|
+
if (!proj.traceEmit) {
|
|
316
|
+
return {
|
|
317
|
+
ok: false,
|
|
318
|
+
reason: 'trace-emit-required',
|
|
319
|
+
detail: 'the plan does not set `trace.emit: true` — a dz run with no trace plane produces nothing the reader, the invariants or the discrimination criterion can check, so it is refused rather than run blind (AM-6)',
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
const specs = allSpecs(proj);
|
|
323
|
+
const families = {};
|
|
324
|
+
for (const s of specs) {
|
|
325
|
+
const f = modelFamily(s.model) ?? inputs.defaultFamily;
|
|
326
|
+
if (f === null) {
|
|
327
|
+
return {
|
|
328
|
+
ok: false,
|
|
329
|
+
reason: 'plan-model-unroutable',
|
|
330
|
+
detail: `step ${s.stepId} declares model ${JSON.stringify(s.model)}, which maps to no model family, and no --default-family was given — guessing a family here would silently decide who is allowed to review this run's code`,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
families[s.stepId] = f;
|
|
334
|
+
}
|
|
335
|
+
for (const s of specs) {
|
|
336
|
+
// READS AND WRITES (AM-9). A read is a path the runner hands a model and asks it to open —
|
|
337
|
+
// containment is not a property of the direction of the I/O.
|
|
338
|
+
for (const [kind, list] of [['read', s.reads], ['write', s.writes]]) {
|
|
339
|
+
for (const rel of list) {
|
|
340
|
+
const why = escapesRoot(rel, inputs.cwdRoot, deps);
|
|
341
|
+
if (why !== null) {
|
|
342
|
+
return {
|
|
343
|
+
ok: false,
|
|
344
|
+
reason: 'artifact-path-escapes-root',
|
|
345
|
+
detail: `step ${s.stepId} declares ${kind} ${JSON.stringify(rel)}, which is ${why} — a declared artifact path is one the runner PROBES and a model is asked to open or create; it may not leave the run root`,
|
|
346
|
+
};
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
const qeWaivers = [];
|
|
352
|
+
for (const s of specs) {
|
|
353
|
+
if (!s.qeRole)
|
|
354
|
+
continue;
|
|
355
|
+
if (families[s.stepId] !== inputs.coderFamily)
|
|
356
|
+
continue;
|
|
357
|
+
if (!inputs.allowSameFamilyQe) {
|
|
358
|
+
return {
|
|
359
|
+
ok: false,
|
|
360
|
+
reason: 'same-family-qe-refused',
|
|
361
|
+
detail: `step ${s.stepId} is marked \`x-role: qe\` and resolves to family ${families[s.stepId]}, which is the family that WROTE the code (--coder-family ${inputs.coderFamily}). Independent cross-model review catches what self-review misses; pass --allow-same-family-qe to proceed under a recorded re-QE debt`,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
qeWaivers.push({ step: s.stepId });
|
|
365
|
+
}
|
|
366
|
+
const reservations = computeBoundaryReservations(proj, () => stageTimeoutMs(inputs));
|
|
367
|
+
const budgetTotal = inputs.budgetOverride ?? proj.budgetTotal;
|
|
368
|
+
const wallCeilingMs = inputs.maxWallClockMsOverride ?? computeWallClockCeilingMs(reservations);
|
|
369
|
+
if (inputs.budgetOverride !== null) {
|
|
370
|
+
const worst = reservations.reduce((a, r) => (a === null || r.invocations > a.invocations ? r : a), null);
|
|
371
|
+
if (worst !== null && worst.invocations > budgetTotal) {
|
|
372
|
+
return {
|
|
373
|
+
ok: false,
|
|
374
|
+
reason: 'reservation-unsatisfiable',
|
|
375
|
+
detail: `--budget ${budgetTotal} is below the worst-case reservation of boundary ${worst.boundaryId} (${worst.invocations} invocations over step(s) ${worst.steps.join(', ')}) — that boundary could never run, so the run would pause forever instead of finishing`,
|
|
376
|
+
};
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (inputs.maxWallClockMsOverride !== null) {
|
|
380
|
+
const worst = reservations.reduce((a, r) => (a === null || r.wallMs > a.wallMs ? r : a), null);
|
|
381
|
+
if (worst !== null && worst.wallMs > wallCeilingMs) {
|
|
382
|
+
return {
|
|
383
|
+
ok: false,
|
|
384
|
+
reason: 'reservation-unsatisfiable',
|
|
385
|
+
detail: `--max-wall-clock ${wallCeilingMs}ms is below the worst-case wall reservation of boundary ${worst.boundaryId} (${worst.wallMs}ms over step(s) ${worst.steps.join(', ')}) — that boundary could never run`,
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
const { hash: argsHash } = computeRunArgsHash(inputs, proj);
|
|
390
|
+
return {
|
|
391
|
+
ok: true,
|
|
392
|
+
projection: proj,
|
|
393
|
+
planDigest: planDigest(inputs.plan),
|
|
394
|
+
execFp: computeExecFingerprint(computeExecAxisInputs(inputs.plan)),
|
|
395
|
+
argsHash,
|
|
396
|
+
families,
|
|
397
|
+
reservations,
|
|
398
|
+
budgetTotal,
|
|
399
|
+
wallCeilingMs,
|
|
400
|
+
achievableMax: computeAchievableMax(proj.budgetTotal, reservations),
|
|
401
|
+
achievableWallMs: Math.max(2 * computeWallClockCeilingMs(reservations), reservations.reduce((n, r) => n + r.wallMs, 0)),
|
|
402
|
+
qeWaivers,
|
|
403
|
+
};
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* The runner's checkpoint key: `checkpointInputHash` (the `fa-ckpt-2` line shape the timeline reader
|
|
407
|
+
* already merges), SALTED with the exec fingerprint exactly like the rendered script's
|
|
408
|
+
* `__ckptInputHash`. Salting with the fingerprint is what makes a semantics change invalidate every
|
|
409
|
+
* prior checkpoint instead of resuming into a plan that no longer means the same thing.
|
|
410
|
+
*/
|
|
411
|
+
export function runnerCheckpointHash(boundaryId, execFp, promptText, depResults) {
|
|
412
|
+
return checkpointInputHash(boundaryId, [execFp, promptText, ...depResults]);
|
|
413
|
+
}
|
|
414
|
+
/**
|
|
415
|
+
* Refusals in a FIXED order: foreign-run-refused → resume-already-completed → stale-input-refused →
|
|
416
|
+
* resume-model-unavailable. (`run-locked` is the CALLER's — it belongs to the lock, not to the
|
|
417
|
+
* decision.)
|
|
418
|
+
*
|
|
419
|
+
* Two properties this function exists to guarantee:
|
|
420
|
+
* • A STALE-INPUT mismatch never resumes, in ANY mode — there is no override, because the hash
|
|
421
|
+
* proves run-INPUT identity and an input change means the remaining work is not the work the
|
|
422
|
+
* checkpoints describe.
|
|
423
|
+
* • The cursor is built from CHECKPOINT LINES plus ARTIFACT PROBES and from nothing else.
|
|
424
|
+
* `completedSteps` in run-state is diagnostic: a crashed writer can leave it optimistic, and a
|
|
425
|
+
* resume that believed it would skip work that never happened.
|
|
426
|
+
*/
|
|
427
|
+
/**
|
|
428
|
+
* The OWNERSHIP + IDENTITY half of the resume decision — everything decidable WITHOUT dispatching
|
|
429
|
+
* anything (Step-8 re-QE NEW-B1).
|
|
430
|
+
*
|
|
431
|
+
* Round 1 hoisted the model probe above the whole decision so `resume-model-unavailable` could be
|
|
432
|
+
* produced at all. That was right, and it had a cost nobody priced: a foreign, completed or
|
|
433
|
+
* stale-input run PROBED first and therefore wrote a `wf-budget-1` probe row before being refused.
|
|
434
|
+
* A refused run must spend NOTHING — a budget row is a record of work, and there was none.
|
|
435
|
+
*
|
|
436
|
+
* So the decision is in two phases now. This one needs no dispatcher, runs first, and refuses on
|
|
437
|
+
* ownership or identity. The model-availability half stays in `decideRunResume`, after the probe.
|
|
438
|
+
* Returns null when there is nothing to refuse.
|
|
439
|
+
*/
|
|
440
|
+
export function decideResumeIdentity(opts) {
|
|
441
|
+
const st = opts.runState;
|
|
442
|
+
if (st === null) {
|
|
443
|
+
if (opts.hasTrace) {
|
|
444
|
+
return {
|
|
445
|
+
reason: 'foreign-run-refused',
|
|
446
|
+
detail: 'the run directory holds a trace but no `wf-run-state/1` — it was written by another host (a Claude-host generated loop writes traces here too). Cross-host merge of one runId is refused, never reconciled',
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
return { reason: 'foreign-run-refused', detail: 'no run state to resume' };
|
|
450
|
+
}
|
|
451
|
+
if (st.schema !== WF_RUN_STATE_SCHEMA || st.owner?.host !== WF_RUN_OWNER_HOST) {
|
|
452
|
+
return {
|
|
453
|
+
reason: 'foreign-run-refused',
|
|
454
|
+
detail: `run state is owned by ${JSON.stringify(st.owner?.host ?? null)} under schema ${JSON.stringify(st.schema ?? null)}, not ${WF_RUN_OWNER_HOST}/${WF_RUN_STATE_SCHEMA}`,
|
|
455
|
+
};
|
|
456
|
+
}
|
|
457
|
+
if (st.status === 'completed') {
|
|
458
|
+
return { reason: 'resume-already-completed', detail: `run ${st.runId} already completed — a completed run is not resumable` };
|
|
459
|
+
}
|
|
460
|
+
const axes = [
|
|
461
|
+
['planDigest', st.planDigest, opts.identity.planDigest],
|
|
462
|
+
['execFp', st.execFp, opts.identity.execFp],
|
|
463
|
+
['argsHash', st.argsHash, opts.identity.argsHash],
|
|
464
|
+
];
|
|
465
|
+
for (const [name, was, now] of axes) {
|
|
466
|
+
if (was !== now) {
|
|
467
|
+
return {
|
|
468
|
+
reason: 'stale-input-refused',
|
|
469
|
+
detail: `${name} changed since the run was written (${String(was).slice(0, 16)}… → ${String(now).slice(0, 16)}…) — the checkpoints describe a different run, and there is no override for that`,
|
|
470
|
+
};
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
return null;
|
|
474
|
+
}
|
|
475
|
+
/**
|
|
476
|
+
* Accumulate the resume args of the WHOLE pause chain (Step-8 re-QE NEW-B2).
|
|
477
|
+
*
|
|
478
|
+
* A pause is satisfied by an arg supplied in SOME leg, not necessarily the current one. Reading only
|
|
479
|
+
* `inputs.resumeArgs` meant leg 3 (supplying `go2`) forgot that leg 2 had supplied `go1`, so the run
|
|
480
|
+
* fell back to the already-satisfied first pause and could never advance past two pauses at all
|
|
481
|
+
* (MEASURED: P1 → P2 → P1 → P1). The satisfied set is run STATE, so it lives in run-state.
|
|
482
|
+
*
|
|
483
|
+
* FIRST WINS for a key that has already been consumed: the boundary it satisfied has already run,
|
|
484
|
+
* and its result is in the checkpoints and the trace. Re-supplying the SAME value is a harmless
|
|
485
|
+
* repeat; re-supplying a DIFFERENT one is refused rather than silently re-deciding history.
|
|
486
|
+
*/
|
|
487
|
+
export function accumulateResumeArgs(persisted, supplied) {
|
|
488
|
+
const args = { ...(persisted ?? {}) };
|
|
489
|
+
for (const [k, v] of Object.entries(supplied)) {
|
|
490
|
+
const prior = args[k];
|
|
491
|
+
if (prior !== undefined && prior !== v)
|
|
492
|
+
return { args, conflict: { key: k, was: prior, now: v } };
|
|
493
|
+
args[k] = v;
|
|
494
|
+
}
|
|
495
|
+
return { args, conflict: null };
|
|
496
|
+
}
|
|
497
|
+
export function decideRunResume(opts) {
|
|
498
|
+
const empty = new Set();
|
|
499
|
+
// the ownership/identity half — SHARED with the pre-probe phase, so the two can never disagree
|
|
500
|
+
const refusal = decideResumeIdentity({ runState: opts.runState, hasTrace: opts.hasTrace, identity: opts.identity });
|
|
501
|
+
if (refusal !== null)
|
|
502
|
+
return { ok: false, reason: refusal.reason, detail: refusal.detail, cursor: empty };
|
|
503
|
+
const st = opts.runState;
|
|
504
|
+
for (const [stepId, persisted] of Object.entries(st.resolvedModels ?? {})) {
|
|
505
|
+
if (!(stepId in opts.reprobedModels))
|
|
506
|
+
continue;
|
|
507
|
+
if (opts.reprobedModels[stepId] !== persisted) {
|
|
508
|
+
return {
|
|
509
|
+
ok: false,
|
|
510
|
+
reason: 'resume-model-unavailable',
|
|
511
|
+
detail: `step ${stepId} ran on model ${persisted}, which no longer answers a probe (re-probe said ${JSON.stringify(opts.reprobedModels[stepId])}) — finishing a run on a different model would silently mix two models' work`,
|
|
512
|
+
cursor: empty,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
const parsed = parseCheckpointRead(opts.checkpointsText ?? '');
|
|
517
|
+
const cursor = new Set();
|
|
518
|
+
for (const [boundaryId, entry] of Object.entries(parsed.entries)) {
|
|
519
|
+
// ARTIFACT PROBES, not just hashes (the `STAGE_ARTIFACTS` semantics ADR-003 Confirmation-5
|
|
520
|
+
// names): a checkpoint says "this stage ran", and the declared write says "and here is what it
|
|
521
|
+
// produced". If the write is gone, the stage did not leave what the run depends on, so the
|
|
522
|
+
// checkpoint alone must not skip it — a resume that trusts the hash and not the disk silently
|
|
523
|
+
// continues without the deliverable.
|
|
524
|
+
const writes = opts.declaredWritesFor?.(boundaryId) ?? [];
|
|
525
|
+
const listing = new Set(writes.filter((rel) => opts.probeArtifact(rel)));
|
|
526
|
+
const d = decideCheckpointResume({
|
|
527
|
+
mode: 'auto',
|
|
528
|
+
entry,
|
|
529
|
+
inputHash: opts.expectedHashFor(boundaryId),
|
|
530
|
+
artifactRel: writes.length === 0 ? null : writes,
|
|
531
|
+
listing,
|
|
532
|
+
});
|
|
533
|
+
if (d.resume)
|
|
534
|
+
cursor.add(boundaryId);
|
|
535
|
+
}
|
|
536
|
+
return { ok: true, cursor };
|
|
537
|
+
}
|
|
538
|
+
/**
|
|
539
|
+
* The results map a RESUME must recompute checkpoint hashes against.
|
|
540
|
+
*
|
|
541
|
+
* Step-8 BLOCKER-2: the hashes were recomputed with `{}`, while the PERSISTED hashes were built from
|
|
542
|
+
* the live `ctx.results`. Any boundary with a dep therefore hashed differently on resume and re-ran
|
|
543
|
+
* — work already paid for, done twice. The reconstruction has two sources, because the run had two:
|
|
544
|
+
* checkpointed stage results, and the resume ARGS that satisfied a pause boundary (a pause is never
|
|
545
|
+
* checkpointed; its "result" is the value the operator supplied).
|
|
546
|
+
*/
|
|
547
|
+
function resumeResults(pre, effectiveArgs, checkpointsText) {
|
|
548
|
+
const out = checkpointResults(checkpointsText);
|
|
549
|
+
for (const b of pre.projection.boundaries) {
|
|
550
|
+
if (b.kind !== 'pause')
|
|
551
|
+
continue;
|
|
552
|
+
const key = b.pause?.resumeArg ?? '';
|
|
553
|
+
if (key !== '' && Object.prototype.hasOwnProperty.call(effectiveArgs, key))
|
|
554
|
+
out[b.boundaryId] = effectiveArgs[key];
|
|
555
|
+
}
|
|
556
|
+
return out;
|
|
557
|
+
}
|
|
558
|
+
/** The writes a boundary DECLARES — a stage's own, or every chain step's for a region. */
|
|
559
|
+
function declaredWritesOf(pre, boundaryId) {
|
|
560
|
+
const b = pre.projection.boundaries.find((x) => x.boundaryId === boundaryId);
|
|
561
|
+
if (b === undefined)
|
|
562
|
+
return [];
|
|
563
|
+
if (b.stage !== undefined)
|
|
564
|
+
return b.stage.deliverable === 'file' ? [...b.stage.writes] : [];
|
|
565
|
+
if (b.region !== undefined)
|
|
566
|
+
return b.region.chain.filter((c) => c.deliverable === 'file').flatMap((c) => c.writes);
|
|
567
|
+
return [];
|
|
568
|
+
}
|
|
569
|
+
/** The checkpointed RESULT for a resumed boundary (so downstream steps see what it produced). */
|
|
570
|
+
function checkpointResults(checkpointsText) {
|
|
571
|
+
const parsed = parseCheckpointRead(checkpointsText ?? '');
|
|
572
|
+
const out = {};
|
|
573
|
+
for (const [stage, entry] of Object.entries(parsed.entries))
|
|
574
|
+
out[stage] = entry.result;
|
|
575
|
+
return out;
|
|
576
|
+
}
|
|
577
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
578
|
+
// Small shared readers / envelopes
|
|
579
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
580
|
+
/**
|
|
581
|
+
* Torn-tail-tolerant JSONL reader (ADR-004 Confirmation-4b). A malformed LAST line is the signature
|
|
582
|
+
* of a process that died mid-append — it is skipped with a NAMED warning. A malformed INTERIOR line
|
|
583
|
+
* is not: something rewrote history, and silently continuing would hide it.
|
|
584
|
+
*/
|
|
585
|
+
export function readJsonlTolerant(text) {
|
|
586
|
+
const lines = String(text ?? '').split('\n');
|
|
587
|
+
// trailing empties are the normal shape of an append-per-line file, not a torn tail
|
|
588
|
+
while (lines.length > 0 && (lines[lines.length - 1] ?? '').trim() === '')
|
|
589
|
+
lines.pop();
|
|
590
|
+
const rows = [];
|
|
591
|
+
const warnings = [];
|
|
592
|
+
let tornTail = false;
|
|
593
|
+
for (let i = 0; i < lines.length; i++) {
|
|
594
|
+
const t = (lines[i] ?? '').trim();
|
|
595
|
+
if (t === '')
|
|
596
|
+
continue;
|
|
597
|
+
try {
|
|
598
|
+
rows.push(JSON.parse(t));
|
|
599
|
+
}
|
|
600
|
+
catch {
|
|
601
|
+
if (i === lines.length - 1) {
|
|
602
|
+
tornTail = true;
|
|
603
|
+
warnings.push(`torn tail: the LAST line is unparseable (${t.length} chars) — skipped as an interrupted append; every earlier row was read`);
|
|
604
|
+
}
|
|
605
|
+
else {
|
|
606
|
+
warnings.push(`unparseable INTERIOR line ${i + 1} — NOT skipped silently: an interior tear means the file was rewritten, not merely interrupted`);
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
return { rows, tornTail, warnings };
|
|
611
|
+
}
|
|
612
|
+
/** The machine-readable pause envelope (AM-16): a wrapper distinguishes pause from failure using
|
|
613
|
+
* ONLY stdout + the exit code, never prose. */
|
|
614
|
+
export function buildPauseEnvelope(runId, pauseState, reason, planPath, resumeArg, where) {
|
|
615
|
+
// Step-8 HIGH-7: both of these were hard-coded to the DEFAULT run home, so a run under
|
|
616
|
+
// `--run-dir features/<slug>` was handed a state path that does not exist and a resume command
|
|
617
|
+
// that would start a different run. An envelope whose own pointer is wrong is worse than none —
|
|
618
|
+
// it is a machine-readable wrong answer.
|
|
619
|
+
//
|
|
620
|
+
// re-QE R3-B: every INTERPOLATED value is shell-quoted with `traceShellQuote` (the repo's existing
|
|
621
|
+
// single-quote-escape helper, shared with the trace flush), so a run dir / plan path / runId /
|
|
622
|
+
// arg key containing a space or a metachar is safe for an operator to PASTE. A resume command that
|
|
623
|
+
// word-splits or injects when pasted is a command the pause envelope should never emit.
|
|
624
|
+
const q = traceShellQuote;
|
|
625
|
+
const runDirFlag = where?.runDirArg === undefined || where.runDirArg === null || where.runDirArg === '' ? '' : ` --run-dir ${q(where.runDirArg)}`;
|
|
626
|
+
const resumeCmd = `dz workflow run ${q(planPath)} --resume ${q(runId)}${runDirFlag}`
|
|
627
|
+
+ (resumeArg === null || resumeArg === '' ? '' : ` --arg ${q(resumeArg + '=<value>')}`);
|
|
628
|
+
return {
|
|
629
|
+
schema: WF_PAUSE_ENVELOPE_SCHEMA,
|
|
630
|
+
runId,
|
|
631
|
+
exitCode: 75,
|
|
632
|
+
pauseState,
|
|
633
|
+
reason,
|
|
634
|
+
runStatePath: where?.runStatePath ?? `.dz/loop-trace/${runId}/run-state.json`,
|
|
635
|
+
resumeCmd,
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
/** The item-key domain is the SHARED one (`TRACE_KEY_RE`): a registry value that the trace plane
|
|
639
|
+
* would refuse must never reach a dispatch, or a trace-on run refuses what a trace-off run completes. */
|
|
640
|
+
function safeItemKey(k) {
|
|
641
|
+
return TRACE_KEY_RE.test(k);
|
|
642
|
+
}
|
|
643
|
+
/** Map a dispatch outcome onto the CLOSED failure enum, through the SAME classifier the rendered
|
|
644
|
+
* script uses. A timeout is the one case the adapter knows better than any message heuristic. */
|
|
645
|
+
function failureClassOf(res) {
|
|
646
|
+
if (res.failure?.reason === 'dispatch-timeout')
|
|
647
|
+
return 'timeout';
|
|
648
|
+
if (res.outcome === 'null')
|
|
649
|
+
return classifyFailure('null', []);
|
|
650
|
+
return classifyFailure('error', errSnap(new Error(res.failure?.detail ?? res.text ?? '')));
|
|
651
|
+
}
|
|
652
|
+
class BudgetInvariantError extends Error {
|
|
653
|
+
boundaryId;
|
|
654
|
+
detail;
|
|
655
|
+
constructor(boundaryId, detail) {
|
|
656
|
+
super(detail);
|
|
657
|
+
this.boundaryId = boundaryId;
|
|
658
|
+
this.detail = detail;
|
|
659
|
+
this.name = 'BudgetInvariantError';
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* THE interpreter loop.
|
|
664
|
+
*
|
|
665
|
+
* Per boundary, in plan order: RESERVE (and pause BEFORE the boundary if the worst case does not
|
|
666
|
+
* fit) → dispatch through the family `Dispatcher` with immediate retries over the closed classes →
|
|
667
|
+
* settle through the shared trace emitter → verify a file deliverable actually landed → checkpoint →
|
|
668
|
+
* append the budget row. A per-spawn guard fires only if the reservation arithmetic was WRONG, and
|
|
669
|
+
* that is a FAILURE (`budget-invariant-violated`), never a pause: a pause promises the remaining
|
|
670
|
+
* work fits after an extension, and a broken invariant promises nothing.
|
|
671
|
+
*/
|
|
672
|
+
export async function runWorkflow(inputs, pre, deps) {
|
|
673
|
+
const store = deps.store;
|
|
674
|
+
const startedAt = deps.now();
|
|
675
|
+
// ── PROBE FIRST (Step-8 BLOCKER-1) ──
|
|
676
|
+
//
|
|
677
|
+
// The probe has to happen BEFORE the resume decision, not after it. Probing afterwards meant
|
|
678
|
+
// `decideRunResume` was handed an empty `reprobedModels`, skipped every persisted step, and
|
|
679
|
+
// `resume-model-unavailable` could not be produced by the real runner at all — a resumed run
|
|
680
|
+
// silently switched model ids and finished green. A probe answer is the ONLY thing that makes the
|
|
681
|
+
// persisted id checkable, so it is the first thing the runner establishes.
|
|
682
|
+
//
|
|
683
|
+
// A null answer is NOT decided here: on a resume it means `resume-model-unavailable` (the id this
|
|
684
|
+
// run committed to is gone), on a fresh run it means `probe-failed`. Deciding it after the resume
|
|
685
|
+
// check keeps each refusal the precise one.
|
|
686
|
+
// ── PHASE 0: ownership + identity, BEFORE anything is spent (re-QE NEW-B1) ──
|
|
687
|
+
//
|
|
688
|
+
// A refused run must write ZERO budget rows. The probe below is real work with a real record, so
|
|
689
|
+
// everything decidable without it is decided first.
|
|
690
|
+
const priorStateEarly = inputs.resume !== null ? store.readRunState() : null;
|
|
691
|
+
if (inputs.resume !== null) {
|
|
692
|
+
const refusal = decideResumeIdentity({
|
|
693
|
+
runState: priorStateEarly,
|
|
694
|
+
hasTrace: store.hasTrace(),
|
|
695
|
+
identity: { planDigest: pre.planDigest, execFp: pre.execFp, argsHash: pre.argsHash },
|
|
696
|
+
});
|
|
697
|
+
if (refusal !== null)
|
|
698
|
+
return failOutcome(inputs.runId, refusal.reason, refusal.detail, null);
|
|
699
|
+
}
|
|
700
|
+
// The pause chain's args, accumulated across every leg (re-QE NEW-B2).
|
|
701
|
+
const accumulated = accumulateResumeArgs(priorStateEarly?.resumeArgs, inputs.resumeArgs);
|
|
702
|
+
if (accumulated.conflict !== null) {
|
|
703
|
+
const c = accumulated.conflict;
|
|
704
|
+
return failOutcome(inputs.runId, 'stale-input-refused', `--arg ${c.key} was already supplied as ${JSON.stringify(c.was)} on an earlier leg and is now ${JSON.stringify(c.now)} — the pause it satisfied has already run, and its result is in the checkpoints and the trace. Re-deciding it now would make this run's history disagree with itself`, null);
|
|
705
|
+
}
|
|
706
|
+
const effectiveArgs = accumulated.args;
|
|
707
|
+
const usedFamilies = [...new Set(Object.values(pre.families))];
|
|
708
|
+
const probedIds = {};
|
|
709
|
+
let probeAgentCalls = 0;
|
|
710
|
+
const probeDetail = {};
|
|
711
|
+
for (const family of usedFamilies) {
|
|
712
|
+
const candidates = [...new Set(allSpecs(pre.projection).filter((s) => pre.families[s.stepId] === family).map((s) => s.model).filter((m) => typeof m === 'string'))];
|
|
713
|
+
const probe = await deps.dispatchers[family].probe(candidates);
|
|
714
|
+
probeAgentCalls++;
|
|
715
|
+
store.appendBudgetRow({
|
|
716
|
+
schema: WF_BUDGET_ROW_SCHEMA,
|
|
717
|
+
kind: 'probe',
|
|
718
|
+
runId: inputs.runId,
|
|
719
|
+
dispatchSeq: null,
|
|
720
|
+
stepId: null,
|
|
721
|
+
itemKey: null,
|
|
722
|
+
attempt: null,
|
|
723
|
+
family,
|
|
724
|
+
model: probe.id,
|
|
725
|
+
wallMs: probe.wallMs,
|
|
726
|
+
tokensIn: null,
|
|
727
|
+
tokensOut: null,
|
|
728
|
+
tokensSource: null,
|
|
729
|
+
outcome: null,
|
|
730
|
+
timeoutMs: null,
|
|
731
|
+
});
|
|
732
|
+
probedIds[family] = probe.id;
|
|
733
|
+
probeDetail[family] = `candidates: ${candidates.join(', ') || 'defaults'} — ${probe.detail}`;
|
|
734
|
+
}
|
|
735
|
+
/** stepId → the id that answers NOW (null when nothing did). */
|
|
736
|
+
const reprobedModels = {};
|
|
737
|
+
for (const spec of allSpecs(pre.projection)) {
|
|
738
|
+
reprobedModels[spec.stepId] = probedIds[pre.families[spec.stepId]] ?? null;
|
|
739
|
+
}
|
|
740
|
+
// ── resume decision (authority: checkpoints + artifact probes; NEVER completedSteps) ──
|
|
741
|
+
let cursor = new Set();
|
|
742
|
+
const priorState = priorStateEarly;
|
|
743
|
+
if (inputs.resume !== null) {
|
|
744
|
+
const checkpointsText = store.readCheckpointsText();
|
|
745
|
+
const priorResults = resumeResults(pre, effectiveArgs, checkpointsText);
|
|
746
|
+
const decision = decideRunResume({
|
|
747
|
+
runState: priorState,
|
|
748
|
+
hasTrace: store.hasTrace(),
|
|
749
|
+
identity: { planDigest: pre.planDigest, execFp: pre.execFp, argsHash: pre.argsHash },
|
|
750
|
+
checkpointsText,
|
|
751
|
+
// the SAME inputs the persisted hashes were built from — see `resumeResults`
|
|
752
|
+
expectedHashFor: (id) => expectedHashFor(pre, id, priorResults),
|
|
753
|
+
probeArtifact: (rel) => store.probeArtifact(rel),
|
|
754
|
+
declaredWritesFor: (id) => declaredWritesOf(pre, id),
|
|
755
|
+
reprobedModels,
|
|
756
|
+
});
|
|
757
|
+
if (!decision.ok) {
|
|
758
|
+
const reason = decision.reason ?? 'foreign-run-refused';
|
|
759
|
+
return failOutcome(inputs.runId, reason, decision.detail ?? '', null);
|
|
760
|
+
}
|
|
761
|
+
cursor = decision.cursor;
|
|
762
|
+
}
|
|
763
|
+
// a family with NO answering model cannot run anything — decided after the resume check so a
|
|
764
|
+
// resumed run reports the precise `resume-model-unavailable` instead of a generic probe failure
|
|
765
|
+
for (const family of usedFamilies) {
|
|
766
|
+
if (probedIds[family] == null) {
|
|
767
|
+
return failOutcome(inputs.runId, 'probe-failed', `no ${family} model answered a probe (${probeDetail[family] ?? ''})`, null);
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
const state = {
|
|
771
|
+
schema: WF_RUN_STATE_SCHEMA,
|
|
772
|
+
owner: {
|
|
773
|
+
host: WF_RUN_OWNER_HOST,
|
|
774
|
+
runnerVersion: inputs.runnerVersion,
|
|
775
|
+
// the CURRENT holder of the claim, not the previous leg's (a resumed run is owned by the
|
|
776
|
+
// process running it) — and never the placeholder 0
|
|
777
|
+
pid: deps.ownerPid ?? priorState?.owner.pid ?? 0,
|
|
778
|
+
startedMarker: deps.ownerStartedMarker ?? priorState?.owner.startedMarker ?? startedAt,
|
|
779
|
+
},
|
|
780
|
+
status: 'running',
|
|
781
|
+
runId: inputs.runId,
|
|
782
|
+
planDigest: pre.planDigest,
|
|
783
|
+
execFp: pre.execFp,
|
|
784
|
+
argsHash: pre.argsHash,
|
|
785
|
+
completedSteps: [...cursor],
|
|
786
|
+
resolvedModels: { ...(priorState?.resolvedModels ?? {}) },
|
|
787
|
+
// CUMULATIVE (Step-8 HIGH-5): each leg adds its delta to the total the PREVIOUS leg reached,
|
|
788
|
+
// never to the plan default. Resetting to `default + this leg's extra` while keeping the old
|
|
789
|
+
// extension rows made the ledger and the ceiling disagree — two resumes of +1 each left a
|
|
790
|
+
// total of default+1 with two rows claiming otherwise, so extensions were laundered away (and,
|
|
791
|
+
// with a bigger first extra, could be re-granted for free).
|
|
792
|
+
budget: {
|
|
793
|
+
total: (priorState?.budget.total ?? pre.budgetTotal) + (inputs.budgetExtra ?? 0),
|
|
794
|
+
spent: priorState?.budget.spent ?? 0,
|
|
795
|
+
extensions: [...(priorState?.budget.extensions ?? [])],
|
|
796
|
+
},
|
|
797
|
+
wallClock: {
|
|
798
|
+
ceilingMs: (priorState?.wallClock.ceilingMs ?? pre.wallCeilingMs) + (inputs.wallClockExtraMs ?? 0),
|
|
799
|
+
spentMs: priorState?.wallClock.spentMs ?? 0,
|
|
800
|
+
extensions: [...(priorState?.wallClock.extensions ?? [])],
|
|
801
|
+
},
|
|
802
|
+
waivers: pre.qeWaivers.map((w) => ({ kind: 'same-family-qe', step: w.step, recordedDebt: `.fa-state/reqe-due.json` })),
|
|
803
|
+
resumeArgs: effectiveArgs,
|
|
804
|
+
coderFamily: inputs.coderFamily,
|
|
805
|
+
startedAt: priorState?.startedAt ?? startedAt,
|
|
806
|
+
updatedAt: startedAt,
|
|
807
|
+
...(deps.dispatcherOverride === true ? { dispatcherOverride: true } : {}),
|
|
808
|
+
};
|
|
809
|
+
// ── extension caps (AM-13/AM-14): recorded, and refused beyond the achievable maximum ──
|
|
810
|
+
if (inputs.budgetExtra !== null && inputs.budgetExtra > 0) {
|
|
811
|
+
const newTotal = state.budget.total;
|
|
812
|
+
if (newTotal > pre.achievableMax) {
|
|
813
|
+
return failOutcome(inputs.runId, 'budget-extension-exhausted', `--budget-extra ${inputs.budgetExtra} takes the total to ${newTotal}, past the achievable maximum ${pre.achievableMax} (= max(2 × ${pre.projection.budgetTotal}, Σ reservations ${pre.reservations.reduce((n, r) => n + r.invocations, 0)})) — the cap is finite and plan-derived on purpose`, null);
|
|
814
|
+
}
|
|
815
|
+
state.budget.extensions.push({ ts: startedAt, extra: inputs.budgetExtra, newTotal });
|
|
816
|
+
}
|
|
817
|
+
if (inputs.wallClockExtraMs !== null && inputs.wallClockExtraMs > 0) {
|
|
818
|
+
const newCeilingMs = state.wallClock.ceilingMs;
|
|
819
|
+
// AM-14's reservation-derived form, not `2 × whatever this invocation's ceiling happens to be`
|
|
820
|
+
// (Step-8 HIGH-5): the cap must be a property of the PLAN, or `--max-wall-clock` would inflate
|
|
821
|
+
// its own cap and `--wall-clock-extra` could then walk it anywhere.
|
|
822
|
+
const wallCap = pre.achievableWallMs;
|
|
823
|
+
if (newCeilingMs > wallCap) {
|
|
824
|
+
return failOutcome(inputs.runId, 'wall-extension-exhausted', `--wall-clock-extra ${inputs.wallClockExtraMs}ms takes the ceiling to ${newCeilingMs}ms, past the cap ${wallCap}ms`, null);
|
|
825
|
+
}
|
|
826
|
+
state.wallClock.extensions.push({ ts: startedAt, extraMs: inputs.wallClockExtraMs, newCeilingMs });
|
|
827
|
+
}
|
|
828
|
+
const opened = openTrace(inputs, pre, store, inputs.resume !== null);
|
|
829
|
+
const ctx = {
|
|
830
|
+
inputs,
|
|
831
|
+
pre,
|
|
832
|
+
deps,
|
|
833
|
+
trace: opened.trace,
|
|
834
|
+
state,
|
|
835
|
+
results: resumeResults(pre, effectiveArgs, store.readCheckpointsText()),
|
|
836
|
+
settleSeq: opened.settleSeq,
|
|
837
|
+
invocationN: 0,
|
|
838
|
+
dispatchCount: 0,
|
|
839
|
+
agentCalls: probeAgentCalls,
|
|
840
|
+
activeReservation: null,
|
|
841
|
+
};
|
|
842
|
+
// a waived same-family QE step owes a machine debt, not a doc instruction (the FR-2.9 precedent)
|
|
843
|
+
for (const w of pre.qeWaivers) {
|
|
844
|
+
// The FR-2.9 `reqe-due-1` shape VERBATIM (`reqe.ts` parseReqeDebt), because the debt is only
|
|
845
|
+
// real if `dz reqe` can read it — a record in a shape the reader rejects is a promise, not a
|
|
846
|
+
// debt. `qeFamily` equals `coderFamily` here: that identity IS the waiver.
|
|
847
|
+
store.writeReqeDebt({
|
|
848
|
+
schema: 'reqe-due-1',
|
|
849
|
+
slug: deps.slug ?? inputs.runId,
|
|
850
|
+
coderFamily: inputs.coderFamily,
|
|
851
|
+
qeFamily: inputs.coderFamily,
|
|
852
|
+
qeGrade: null,
|
|
853
|
+
reason: `dz workflow run ${inputs.runId}: step ${w.step} (x-role: qe) resolved to the CODER family ${inputs.coderFamily} and ran under --allow-same-family-qe — the cross-family guard was consciously suspended, so an independent review is OWED`,
|
|
854
|
+
emittedAt: startedAt,
|
|
855
|
+
runStamp: `${inputs.runId}:${pre.execFp.slice(0, 16)}`,
|
|
856
|
+
});
|
|
857
|
+
}
|
|
858
|
+
// the PROBED ids (established above, before the resume decision) become this run's resolution
|
|
859
|
+
for (const s of allSpecs(pre.projection)) {
|
|
860
|
+
const id = probedIds[pre.families[s.stepId]];
|
|
861
|
+
if (id != null)
|
|
862
|
+
state.resolvedModels[s.stepId] = id;
|
|
863
|
+
}
|
|
864
|
+
// ── the boundary walk ──
|
|
865
|
+
const reservationOf = new Map(pre.reservations.map((r) => [r.boundaryId, r]));
|
|
866
|
+
try {
|
|
867
|
+
for (const b of pre.projection.boundaries) {
|
|
868
|
+
if (cursor.has(b.boundaryId))
|
|
869
|
+
continue; // resumed — its result already lives in ctx.results
|
|
870
|
+
const res = reservationOf.get(b.boundaryId) ?? null;
|
|
871
|
+
const remainingBudget = state.budget.total - state.budget.spent;
|
|
872
|
+
const remainingWall = state.wallClock.ceilingMs - state.wallClock.spentMs;
|
|
873
|
+
if (res !== null && res.invocations > remainingBudget) {
|
|
874
|
+
return pauseOutcome(ctx, b, 'AWAITING_BUDGET', 'budget-exhausted', `boundary ${b.boundaryId} reserves ${res.invocations} invocation(s) over step(s) ${res.steps.join(', ')}, and only ${remainingBudget} of ${state.budget.total} remain — pausing BEFORE the boundary so it is never interrupted mid-region (AM-4)`);
|
|
875
|
+
}
|
|
876
|
+
if (res !== null && res.wallMs > remainingWall) {
|
|
877
|
+
return pauseOutcome(ctx, b, 'AWAITING_WALL_CLOCK', 'budget-exhausted', `boundary ${b.boundaryId} reserves ${res.wallMs}ms of wall clock and only ${remainingWall}ms of ${state.wallClock.ceilingMs}ms remain — pausing BEFORE the boundary`);
|
|
878
|
+
}
|
|
879
|
+
if (b.kind === 'pause') {
|
|
880
|
+
const key = b.pause?.resumeArg ?? '';
|
|
881
|
+
// satisfied by ANY leg, not just this one (re-QE NEW-B2)
|
|
882
|
+
const supplied = key !== '' && Object.prototype.hasOwnProperty.call(effectiveArgs, key);
|
|
883
|
+
if (!supplied) {
|
|
884
|
+
return pauseOutcome(ctx, b, b.pause?.state ?? 'AWAITING_INPUT', 'plan-pause', `the plan declares pause state ${JSON.stringify(b.pause?.state ?? '')} here; re-invoke with --arg ${key}=<value> to continue`);
|
|
885
|
+
}
|
|
886
|
+
ctx.results[b.boundaryId] = effectiveArgs[key];
|
|
887
|
+
continue;
|
|
888
|
+
}
|
|
889
|
+
ctx.activeReservation = res === null ? null : { boundaryId: b.boundaryId, remaining: deps.corruptReservations === true ? 0 : res.invocations };
|
|
890
|
+
if (b.kind === 'region') {
|
|
891
|
+
await runRegion(ctx, b);
|
|
892
|
+
ctx.activeReservation = null;
|
|
893
|
+
continue;
|
|
894
|
+
}
|
|
895
|
+
const terminal = await runStage(ctx, b);
|
|
896
|
+
ctx.activeReservation = null;
|
|
897
|
+
if (terminal !== null) {
|
|
898
|
+
// A gate `terminal:` route ends the run BY PLAN DESIGN. Parity with the rendered script:
|
|
899
|
+
// the top-level terminal return skips the epilogue, so `run.closed` is NOT written and the
|
|
900
|
+
// trace parses as incomplete. See the FLAG in the change manifest: `RunOutcome` has no
|
|
901
|
+
// terminal member, so this reports as `completed` with the route named in the ledger row.
|
|
902
|
+
return terminalOutcome(ctx, terminal, startedAt);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
catch (e) {
|
|
907
|
+
if (e instanceof BudgetInvariantError) {
|
|
908
|
+
return failOutcome(inputs.runId, 'budget-invariant-violated', e.detail, ctx);
|
|
909
|
+
}
|
|
910
|
+
if (e instanceof RunFailure) {
|
|
911
|
+
return failOutcome(inputs.runId, e.reason, e.detail, ctx);
|
|
912
|
+
}
|
|
913
|
+
throw e;
|
|
914
|
+
}
|
|
915
|
+
// ── completed epilogue: the ONLY path that closes the trace ──
|
|
916
|
+
traceClose(ctx.trace);
|
|
917
|
+
flush(ctx);
|
|
918
|
+
ctx.state.status = 'completed';
|
|
919
|
+
ctx.state.completedSteps = pre.projection.boundaries.map((b) => b.boundaryId);
|
|
920
|
+
ctx.state.updatedAt = deps.now();
|
|
921
|
+
stampTraceBinding(ctx.state, store);
|
|
922
|
+
deps.lock(() => store.writeRunState(ctx.state));
|
|
923
|
+
appendLedger(ctx, 'completed');
|
|
924
|
+
return {
|
|
925
|
+
kind: 'completed',
|
|
926
|
+
exitCode: 0,
|
|
927
|
+
result: {
|
|
928
|
+
schema: WF_RUN_RESULT_SCHEMA,
|
|
929
|
+
runId: inputs.runId,
|
|
930
|
+
status: 'completed',
|
|
931
|
+
exitCode: 0,
|
|
932
|
+
...(deps.dispatcherOverride === true ? { dispatcherOverride: true } : {}),
|
|
933
|
+
},
|
|
934
|
+
};
|
|
935
|
+
}
|
|
936
|
+
/**
|
|
937
|
+
* Open the run's trace state. A FRESH run buffers its own `run.opened` frame; a RESUMED leg picks
|
|
938
|
+
* the seq counter up where the previous leg left it and emits NO second `run.opened` — one run has
|
|
939
|
+
* ONE opened frame and ONE monotonically allocated seq space, whatever the process boundaries.
|
|
940
|
+
* The dispatched/settled counters are recovered too, so the `run.closed` counts describe the whole
|
|
941
|
+
* run rather than its last leg.
|
|
942
|
+
*/
|
|
943
|
+
function openTrace(inputs, pre, store, resuming) {
|
|
944
|
+
// 'dz-process': this runner IS the dz process, and it appends trace.jsonl itself. The value is a
|
|
945
|
+
// fact about which code path is running, not a claim about trustworthiness — see ADR-001.
|
|
946
|
+
const st = traceInit(inputs.runId, pre.planDigest, pre.execFp, 'dz-process');
|
|
947
|
+
if (!resuming)
|
|
948
|
+
return { trace: st, settleSeq: {} };
|
|
949
|
+
const priorText = store.readTraceText();
|
|
950
|
+
if (priorText === null || priorText.trim() === '')
|
|
951
|
+
return { trace: st, settleSeq: {} };
|
|
952
|
+
const prior = parseTrace(priorText);
|
|
953
|
+
const maxSeq = prior.events.reduce((m, e) => Math.max(m, e.seq), 0);
|
|
954
|
+
if (maxSeq <= 0)
|
|
955
|
+
return { trace: st, settleSeq: {} };
|
|
956
|
+
st.buffer.length = 0;
|
|
957
|
+
st.seq = maxSeq;
|
|
958
|
+
st.dispatched = prior.events.filter((e) => e.event === 'dispatched').length;
|
|
959
|
+
st.settled = prior.events.filter((e) => e.event === 'settled').length;
|
|
960
|
+
// Recover the previous leg's settle seqs so a resumed step's `causedBy` still points at the
|
|
961
|
+
// upstream settle that actually happened. Without this a resumed leg emits `causedBy: []` for
|
|
962
|
+
// every dep it did not re-run — technically honest, but it drops the causal edge a reader draws.
|
|
963
|
+
const stepOf = new Map();
|
|
964
|
+
for (const e of prior.events) {
|
|
965
|
+
if (e.event === 'dispatched')
|
|
966
|
+
stepOf.set(e.invocationId, { stepId: e.stepId, itemKey: e.itemKey });
|
|
967
|
+
}
|
|
968
|
+
const settleSeq = {};
|
|
969
|
+
for (const e of prior.events) {
|
|
970
|
+
if (e.event !== 'settled')
|
|
971
|
+
continue;
|
|
972
|
+
const who = stepOf.get(e.invocationId);
|
|
973
|
+
if (who !== undefined)
|
|
974
|
+
settleSeq[settleKey(who.stepId, who.itemKey)] = e.seq;
|
|
975
|
+
}
|
|
976
|
+
return { trace: st, settleSeq };
|
|
977
|
+
}
|
|
978
|
+
/** A named, non-retryable runtime refusal raised from deep inside the walk. */
|
|
979
|
+
class RunFailure extends Error {
|
|
980
|
+
reason;
|
|
981
|
+
detail;
|
|
982
|
+
constructor(reason, detail) {
|
|
983
|
+
super(detail);
|
|
984
|
+
this.reason = reason;
|
|
985
|
+
this.detail = detail;
|
|
986
|
+
this.name = 'RunFailure';
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
function expectedHashFor(pre, boundaryId, results) {
|
|
990
|
+
const b = pre.projection.boundaries.find((x) => x.boundaryId === boundaryId);
|
|
991
|
+
const spec = b?.stage ?? null;
|
|
992
|
+
const prompt = spec === null ? '' : assemblePrompt(spec, null, null);
|
|
993
|
+
const deps = (b?.deps ?? []).map((d) => results[d] ?? null);
|
|
994
|
+
return runnerCheckpointHash(boundaryId, pre.execFp, prompt, deps);
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* THE prompt assembly, both enactors' version: the USER prompt seed, then the SHARED contract lines
|
|
998
|
+
* (`stepContractLines` — the same function `loop-render` splices), then the per-item binding, then
|
|
999
|
+
* the upstream value.
|
|
1000
|
+
*
|
|
1001
|
+
* The upstream value is spliced through `defangGateEchoes` (ADR-002 Confirmation-3). This is the
|
|
1002
|
+
* INGRESS half of the gate defence and it is separate from the LAST-anchored parser for a reason:
|
|
1003
|
+
* the parser stops one reply from smuggling a verdict past its own terminal line, while this stops
|
|
1004
|
+
* an upstream gate's LEGITIMATE verdict from becoming a downstream step's terminal line the moment
|
|
1005
|
+
* that step quotes its input back. "Summarize the previous review" would otherwise be a working
|
|
1006
|
+
* exploit against a plan that did nothing wrong. Neutralization, not deletion — the downstream
|
|
1007
|
+
* model still reads every upstream word.
|
|
1008
|
+
*/
|
|
1009
|
+
function assemblePrompt(spec, itemKey, upstream) {
|
|
1010
|
+
const parts = [spec.prompt ?? `TODO: prompt for step ${spec.stepId} (phase ${spec.phase})`]; // no-stubs: the SAME default prompt sentinel the render emits (loop-render.ts:186) — an authoring cue both enactors must show identically, not unfinished code
|
|
1011
|
+
parts.push(...stepContractLines({
|
|
1012
|
+
reads: spec.reads,
|
|
1013
|
+
writes: spec.writes,
|
|
1014
|
+
deliverable: spec.deliverable,
|
|
1015
|
+
tools: spec.tools,
|
|
1016
|
+
gate: spec.gate === null ? null : { kind: spec.gate.kind },
|
|
1017
|
+
}));
|
|
1018
|
+
if (itemKey !== null)
|
|
1019
|
+
parts.push('this branch handles registry item: ' + itemKey);
|
|
1020
|
+
if (upstream !== null && upstream !== undefined) {
|
|
1021
|
+
// a STRING travels as text (that is what the model produced and what it must read back); any
|
|
1022
|
+
// other shape is serialized. Both go through the defang — a JSON string can carry a newline too.
|
|
1023
|
+
const raw = typeof upstream === 'string' ? upstream : JSON.stringify(upstream);
|
|
1024
|
+
parts.push('upstream value: ' + defangGateEchoes(raw));
|
|
1025
|
+
}
|
|
1026
|
+
return parts.join('\n');
|
|
1027
|
+
}
|
|
1028
|
+
function flush(ctx) {
|
|
1029
|
+
const lines = traceDrain(ctx.trace);
|
|
1030
|
+
if (lines.length > 0)
|
|
1031
|
+
ctx.deps.store.appendTraceLines(lines);
|
|
1032
|
+
}
|
|
1033
|
+
function spendBudget(ctx, stepId) {
|
|
1034
|
+
if (ctx.activeReservation !== null) {
|
|
1035
|
+
if (ctx.activeReservation.remaining <= 0) {
|
|
1036
|
+
throw new BudgetInvariantError(ctx.activeReservation.boundaryId, `boundary ${ctx.activeReservation.boundaryId} tried to dispatch ${stepId} beyond its own reservation — the reservation arithmetic is WRONG. This is a FAILURE, never a pause: a pause promises the remaining work fits after an extension, and a broken invariant promises nothing`);
|
|
1037
|
+
}
|
|
1038
|
+
ctx.activeReservation.remaining--;
|
|
1039
|
+
}
|
|
1040
|
+
ctx.state.budget.spent++;
|
|
1041
|
+
ctx.agentCalls++;
|
|
1042
|
+
}
|
|
1043
|
+
/** Dispatch ONE invocation with the closed-class immediate retry policy. Returns the settled result. */
|
|
1044
|
+
async function dispatchOnce(ctx, spec, itemKey, upstream, causedBy) {
|
|
1045
|
+
const family = ctx.pre.families[spec.stepId];
|
|
1046
|
+
const dispatcher = ctx.deps.dispatchers[family];
|
|
1047
|
+
const model = ctx.state.resolvedModels[spec.stepId] ?? null;
|
|
1048
|
+
const timeoutMs = stageTimeoutMs(ctx.inputs);
|
|
1049
|
+
const prompt = assemblePrompt(spec, itemKey, upstream);
|
|
1050
|
+
let lastRes = null;
|
|
1051
|
+
for (let attempt = 1; attempt <= spec.retryMaxAttempts; attempt++) {
|
|
1052
|
+
// CONTAINMENT re-check FIRST — before ANY invocation accounting (R4: refuse before you
|
|
1053
|
+
// account, the NEW-B1 discipline). Round-3 R3-A put this after spendBudget + traceOnDispatch,
|
|
1054
|
+
// so a refused step still burned a budget unit and emitted a phantom `dispatched` event with no
|
|
1055
|
+
// settle. Preflight validated these paths against a tree the models this run has since written
|
|
1056
|
+
// to; a symlink planted since is caught HERE, for READS and writes alike, before the adapter is
|
|
1057
|
+
// handed the target cwd and before the run charges itself for a dispatch that never happens.
|
|
1058
|
+
for (const [kind, rel] of [...spec.reads.map((r) => ['read', r]), ...spec.writes.map((w) => ['write', w])]) {
|
|
1059
|
+
if (!ctx.deps.store.pathContainmentOk(rel)) {
|
|
1060
|
+
return {
|
|
1061
|
+
ok: false,
|
|
1062
|
+
value: null,
|
|
1063
|
+
res: null,
|
|
1064
|
+
reason: 'artifact-path-escapes-root',
|
|
1065
|
+
detail: `step ${spec.stepId} declared ${kind} ${JSON.stringify(rel)} no longer resolves inside the run root at dispatch time — a symlink was planted after preflight, and a declared artifact path may not leave the root whichever direction the I/O goes`,
|
|
1066
|
+
};
|
|
1067
|
+
}
|
|
1068
|
+
}
|
|
1069
|
+
spendBudget(ctx, spec.stepId);
|
|
1070
|
+
const invocationId = spec.stepId + (itemKey === null ? '' : ':' + itemKey) + '#' + String(++ctx.invocationN);
|
|
1071
|
+
const dispatchSeq = traceOnDispatch(ctx.trace, {
|
|
1072
|
+
invocationId,
|
|
1073
|
+
stepId: spec.stepId,
|
|
1074
|
+
itemKey,
|
|
1075
|
+
attempt,
|
|
1076
|
+
phase: spec.phase,
|
|
1077
|
+
model,
|
|
1078
|
+
causedBy: causedBy.filter((n) => typeof n === 'number' && n > 0),
|
|
1079
|
+
});
|
|
1080
|
+
ctx.dispatchCount++;
|
|
1081
|
+
const baseline = spec.deliverable === 'file' && spec.writes.length > 0 ? ctx.deps.store.snapshotWrites(spec.writes) : null;
|
|
1082
|
+
const t0 = ctx.deps.monotonicMs();
|
|
1083
|
+
const res = await dispatcher.dispatch({
|
|
1084
|
+
stepId: spec.stepId,
|
|
1085
|
+
itemKey,
|
|
1086
|
+
attempt,
|
|
1087
|
+
prompt,
|
|
1088
|
+
family,
|
|
1089
|
+
resolvedModelId: model ?? '',
|
|
1090
|
+
deliverable: spec.deliverable,
|
|
1091
|
+
expectedReads: spec.reads,
|
|
1092
|
+
expectedWrites: spec.writes,
|
|
1093
|
+
timeoutMs,
|
|
1094
|
+
cwd: ctx.inputs.cwdRoot,
|
|
1095
|
+
});
|
|
1096
|
+
const wallMs = res.wallMs > 0 ? res.wallMs : Math.max(0, ctx.deps.monotonicMs() - t0);
|
|
1097
|
+
ctx.state.wallClock.spentMs += wallMs;
|
|
1098
|
+
lastRes = res;
|
|
1099
|
+
// LANDED BARRIER (scheduler-owned, dispatcher-independent): a settled dispatch is not a
|
|
1100
|
+
// delivered file. Declared writes must EXIST and be NEWLY CHANGED against the pre-dispatch
|
|
1101
|
+
// snapshot — an untouched leftover from a previous run is not this step's deliverable.
|
|
1102
|
+
let outcome = res.outcome;
|
|
1103
|
+
let landedFailure = null;
|
|
1104
|
+
if (outcome === 'ok' && baseline !== null && ctx.deps.disableLandedBarrier !== true) {
|
|
1105
|
+
const notLanded = [];
|
|
1106
|
+
for (const rel of spec.writes) {
|
|
1107
|
+
if (!ctx.deps.store.probeArtifact(rel)) {
|
|
1108
|
+
notLanded.push(`${rel} (absent)`);
|
|
1109
|
+
continue;
|
|
1110
|
+
}
|
|
1111
|
+
const after = ctx.deps.store.snapshotWrites([rel])[rel] ?? null;
|
|
1112
|
+
if (after !== null && after === baseline[rel])
|
|
1113
|
+
notLanded.push(`${rel} (unchanged since before the dispatch)`);
|
|
1114
|
+
}
|
|
1115
|
+
if (notLanded.length > 0) {
|
|
1116
|
+
outcome = 'error';
|
|
1117
|
+
landedFailure = `step ${spec.stepId} settled ok but its declared write(s) did not land: ${notLanded.join(', ')}`;
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
const settleSeq = traceOnSettle(ctx.trace, { invocationId, outcome });
|
|
1121
|
+
ctx.settleSeq[settleKey(spec.stepId, itemKey)] = settleSeq;
|
|
1122
|
+
ctx.deps.store.appendBudgetRow({
|
|
1123
|
+
schema: WF_BUDGET_ROW_SCHEMA,
|
|
1124
|
+
kind: 'stage',
|
|
1125
|
+
runId: ctx.inputs.runId,
|
|
1126
|
+
dispatchSeq,
|
|
1127
|
+
stepId: spec.stepId,
|
|
1128
|
+
itemKey,
|
|
1129
|
+
attempt,
|
|
1130
|
+
family,
|
|
1131
|
+
model: res.modelUsed ?? model,
|
|
1132
|
+
wallMs,
|
|
1133
|
+
tokensIn: res.tokensIn,
|
|
1134
|
+
tokensOut: res.tokensOut,
|
|
1135
|
+
tokensSource: res.tokensSource,
|
|
1136
|
+
outcome,
|
|
1137
|
+
timeoutMs,
|
|
1138
|
+
});
|
|
1139
|
+
flush(ctx);
|
|
1140
|
+
if (outcome === 'ok')
|
|
1141
|
+
return { ok: true, value: res.text, res, reason: null, detail: '' };
|
|
1142
|
+
if (landedFailure !== null) {
|
|
1143
|
+
return { ok: false, value: null, res, reason: 'deliverable-not-landed', detail: landedFailure };
|
|
1144
|
+
}
|
|
1145
|
+
const cls = failureClassOf(res);
|
|
1146
|
+
const retryable = attempt < spec.retryMaxAttempts && cls !== null && spec.retryOn.includes(cls);
|
|
1147
|
+
if (!retryable) {
|
|
1148
|
+
return {
|
|
1149
|
+
ok: false,
|
|
1150
|
+
value: null,
|
|
1151
|
+
res,
|
|
1152
|
+
reason: res.failure?.reason ?? 'dispatch-dead',
|
|
1153
|
+
detail: res.failure?.detail ?? `step ${spec.stepId} settled ${outcome} (class ${String(cls)}) and this step's retryOn does not cover it`,
|
|
1154
|
+
};
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
return { ok: false, value: null, res: lastRes, reason: lastRes?.failure?.reason ?? 'dispatch-dead', detail: 'retries exhausted' };
|
|
1158
|
+
}
|
|
1159
|
+
function settleKey(stepId, itemKey) {
|
|
1160
|
+
return stepId + '' + (itemKey ?? '');
|
|
1161
|
+
}
|
|
1162
|
+
function causedByOf(ctx, deps) {
|
|
1163
|
+
return deps.map((d) => ctx.settleSeq[settleKey(d, null)] ?? -1).filter((n) => n > 0);
|
|
1164
|
+
}
|
|
1165
|
+
/** A top-level agent/gate boundary. Returns a terminal-route label, or null to continue. */
|
|
1166
|
+
async function runStage(ctx, b) {
|
|
1167
|
+
const spec = b.stage;
|
|
1168
|
+
if (spec === undefined)
|
|
1169
|
+
return null;
|
|
1170
|
+
const upstream = b.deps.length === 1 ? ctx.results[b.deps[0]] ?? null : null;
|
|
1171
|
+
const causedBy = causedByOf(ctx, b.deps);
|
|
1172
|
+
let redosLeft = spec.gate?.maxRedos ?? 0;
|
|
1173
|
+
for (;;) {
|
|
1174
|
+
const r = await dispatchOnce(ctx, spec, null, upstream, causedBy);
|
|
1175
|
+
if (!r.ok)
|
|
1176
|
+
throw new RunFailure(r.reason ?? 'dispatch-dead', r.detail);
|
|
1177
|
+
if (spec.kind !== 'gate' || spec.gate === null) {
|
|
1178
|
+
ctx.results[b.boundaryId] = r.value;
|
|
1179
|
+
checkpoint(ctx, b, r.value);
|
|
1180
|
+
return null;
|
|
1181
|
+
}
|
|
1182
|
+
// GATE — the verdict is PARSED by the shared grammar and never synthesized.
|
|
1183
|
+
const verdict = gateVerdict(r.value);
|
|
1184
|
+
if (verdict === 'pass') {
|
|
1185
|
+
ctx.results[b.boundaryId] = r.value;
|
|
1186
|
+
checkpoint(ctx, b, r.value);
|
|
1187
|
+
return null;
|
|
1188
|
+
}
|
|
1189
|
+
const route = spec.gate.failRoute;
|
|
1190
|
+
if (route !== null && route.startsWith('terminal:'))
|
|
1191
|
+
return route;
|
|
1192
|
+
if (redosLeft <= 0) {
|
|
1193
|
+
// Parity with the rendered script (`loop-render.ts` gate routing): a non-pass verdict with
|
|
1194
|
+
// its declared routing exhausted is a LOUD run failure — never a silent pass, never a retry.
|
|
1195
|
+
// TWO members, because there are two different facts to report (AM-19).
|
|
1196
|
+
throw verdict === 'invalid'
|
|
1197
|
+
? new RunFailure('gate-verdict-unparseable', `gate ${spec.stepId} produced no single anchored terminal "GATE: PASS|FAIL" line — an unparseable verdict is a loud failure, never a synthesized pass, and no redos remain`)
|
|
1198
|
+
: new RunFailure('gate-failed', `gate ${spec.stepId} returned a PARSED "GATE: FAIL" verdict with no redos left and no terminal route declared (failRoute ${JSON.stringify(route)}) — the grammar was satisfied and the model answered clearly; the run fails because the plan declares nowhere for a failing gate to go`);
|
|
1199
|
+
}
|
|
1200
|
+
redosLeft--;
|
|
1201
|
+
if (route !== null) {
|
|
1202
|
+
const routeBoundary = ctx.pre.projection.boundaries.find((x) => x.boundaryId === route);
|
|
1203
|
+
const routeSpec = routeBoundary?.stage;
|
|
1204
|
+
if (routeSpec !== undefined) {
|
|
1205
|
+
const rr = await dispatchOnce(ctx, routeSpec, null, ctx.results[route] ?? null, causedByOf(ctx, routeBoundary?.deps ?? []));
|
|
1206
|
+
if (!rr.ok)
|
|
1207
|
+
throw new RunFailure(rr.reason ?? 'dispatch-dead', rr.detail);
|
|
1208
|
+
ctx.results[route] = rr.value;
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
}
|
|
1212
|
+
}
|
|
1213
|
+
/** A parallel region: every registry member activated, at most `maxFanout` in flight, joined by the
|
|
1214
|
+
* SHARED `joinRegion` decision. */
|
|
1215
|
+
async function runRegion(ctx, b) {
|
|
1216
|
+
const r = b.region;
|
|
1217
|
+
if (r === undefined)
|
|
1218
|
+
return;
|
|
1219
|
+
const members = activatedMembers(r.registry, r.dedup, r.maxFanout);
|
|
1220
|
+
for (const k of members) {
|
|
1221
|
+
if (!safeItemKey(k)) {
|
|
1222
|
+
throw new RunFailure('plan-invalid', `fanout ${r.fanout} registry item ${JSON.stringify(k)} is outside the shared ItemKey domain ${String(TRACE_KEY_RE)} — the trace plane would refuse the event, so a trace-on run must refuse the dispatch rather than complete where a trace-off run would`);
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
const causedBy = causedByOf(ctx, b.deps);
|
|
1226
|
+
const bound = r.maxFanout > 0 ? r.maxFanout : 1;
|
|
1227
|
+
const results = new Array(members.length).fill(null);
|
|
1228
|
+
/** The FIRST branch failure, kept so a join refusal reports what actually killed the region
|
|
1229
|
+
* rather than inventing a reason of its own. */
|
|
1230
|
+
const branchFailures = [];
|
|
1231
|
+
let next = 0;
|
|
1232
|
+
const worker = async () => {
|
|
1233
|
+
for (;;) {
|
|
1234
|
+
const i = next++;
|
|
1235
|
+
if (i >= members.length)
|
|
1236
|
+
return;
|
|
1237
|
+
const item = members[i];
|
|
1238
|
+
let value = ctx.results[b.boundaryId] ?? null;
|
|
1239
|
+
let failed = false;
|
|
1240
|
+
for (const spec of r.chain) {
|
|
1241
|
+
const rr = await dispatchOnce(ctx, spec, item, value, causedBy);
|
|
1242
|
+
if (!rr.ok) {
|
|
1243
|
+
// a failing branch is a NULL branch — the JOIN decides what that means, not the loop
|
|
1244
|
+
failed = true;
|
|
1245
|
+
branchFailures.push({ reason: rr.reason ?? 'dispatch-dead', detail: rr.detail });
|
|
1246
|
+
break;
|
|
1247
|
+
}
|
|
1248
|
+
value = rr.value;
|
|
1249
|
+
}
|
|
1250
|
+
results[i] = failed ? null : value;
|
|
1251
|
+
}
|
|
1252
|
+
};
|
|
1253
|
+
await Promise.all(Array.from({ length: Math.min(bound, Math.max(members.length, 1)) }, () => worker()));
|
|
1254
|
+
// `joinRegion` THROWS a named message when the policy is not met — parity with the rendered
|
|
1255
|
+
// script, which routes that throw through its single terminal exit. Here it becomes a typed run
|
|
1256
|
+
// failure carrying the FIRST branch's reason: the join did not invent the failure, it refused to
|
|
1257
|
+
// paper over one.
|
|
1258
|
+
let join;
|
|
1259
|
+
try {
|
|
1260
|
+
join = joinRegion(results, { policy: r.joinPolicy, onInvalid: r.onInvalid, region: r.fanout });
|
|
1261
|
+
}
|
|
1262
|
+
catch (e) {
|
|
1263
|
+
const failure = branchFailures[0];
|
|
1264
|
+
throw new RunFailure(failure?.reason ?? 'dispatch-dead', `${e instanceof Error ? e.message : String(e)} (onInvalid: ${r.onInvalid})` + (failure === undefined ? '' : ` — first failing branch: ${failure.detail}`));
|
|
1265
|
+
}
|
|
1266
|
+
ctx.results[b.boundaryId] = join.values;
|
|
1267
|
+
if (r.join !== '')
|
|
1268
|
+
ctx.results[r.join] = join.values;
|
|
1269
|
+
checkpoint(ctx, b, join.values);
|
|
1270
|
+
}
|
|
1271
|
+
/** K6: the runner checkpoints EVERY top-level agent/gate/region boundary UNCONDITIONALLY — its
|
|
1272
|
+
* resume cursor is built from these lines, so making them optional would make resume optional.
|
|
1273
|
+
* `plan.checkpointing` stays what it always was: the CLAUDE-host opt-in. */
|
|
1274
|
+
function checkpoint(ctx, b, value) {
|
|
1275
|
+
const line = serializeCheckpoint(b.boundaryId, expectedHashFor(ctx.pre, b.boundaryId, ctx.results), value);
|
|
1276
|
+
if (line !== null)
|
|
1277
|
+
ctx.deps.store.appendCheckpointLine(line);
|
|
1278
|
+
}
|
|
1279
|
+
/**
|
|
1280
|
+
* THE ONE PLACE the content binding is refreshed. A helper rather than three inline copies, because
|
|
1281
|
+
* three call sites mean three chances to forget — and a forgotten one degrades silently to
|
|
1282
|
+
* `unknown` (fail-closed, but a real dz run would then read as un-attested).
|
|
1283
|
+
* Called AFTER the final trace flush, so it describes the bytes a reader will actually see.
|
|
1284
|
+
*/
|
|
1285
|
+
function stampTraceBinding(state, store) {
|
|
1286
|
+
const m = store.measureTrace();
|
|
1287
|
+
if (m === null)
|
|
1288
|
+
return; // no trace ⇒ no binding to make; the reader will say `unknown`
|
|
1289
|
+
state.traceSha256 = m.sha256;
|
|
1290
|
+
state.traceLines = m.lines;
|
|
1291
|
+
}
|
|
1292
|
+
function appendLedger(ctx, outcome) {
|
|
1293
|
+
const line = traceLedgerLine({
|
|
1294
|
+
slug: ctx.deps.slug ?? ctx.inputs.runId,
|
|
1295
|
+
runId: ctx.inputs.runId,
|
|
1296
|
+
planDigest: ctx.pre.planDigest,
|
|
1297
|
+
agents: ctx.agentCalls,
|
|
1298
|
+
outcome,
|
|
1299
|
+
date: null,
|
|
1300
|
+
});
|
|
1301
|
+
if (line !== null)
|
|
1302
|
+
ctx.deps.store.appendLedgerLine(line);
|
|
1303
|
+
}
|
|
1304
|
+
/** PAUSE — flush WITHOUT `run.closed` (parity with the render's top-level terminal return, which
|
|
1305
|
+
* skips the epilogue). The trace legitimately parses as incomplete, and window-truncated invariants
|
|
1306
|
+
* report inconclusive rather than pass. */
|
|
1307
|
+
function pauseOutcome(ctx, b, pauseState, reason, detail) {
|
|
1308
|
+
flush(ctx);
|
|
1309
|
+
const remaining = ctx.pre.projection.boundaries.map((x) => x.boundaryId).slice(ctx.pre.projection.boundaries.findIndex((x) => x.boundaryId === b.boundaryId));
|
|
1310
|
+
ctx.state.status = 'paused';
|
|
1311
|
+
ctx.state.pause = {
|
|
1312
|
+
state: pauseState,
|
|
1313
|
+
resumeArg: b.pause?.resumeArg ?? '',
|
|
1314
|
+
payloadSchema: b.pause?.payloadSchema ?? null,
|
|
1315
|
+
remainingSteps: remaining,
|
|
1316
|
+
reservationNote: detail,
|
|
1317
|
+
};
|
|
1318
|
+
ctx.state.updatedAt = ctx.deps.now();
|
|
1319
|
+
stampTraceBinding(ctx.state, ctx.deps.store);
|
|
1320
|
+
ctx.deps.lock(() => ctx.deps.store.writeRunState(ctx.state));
|
|
1321
|
+
appendLedger(ctx, 'paused');
|
|
1322
|
+
return {
|
|
1323
|
+
kind: 'paused',
|
|
1324
|
+
exitCode: 75,
|
|
1325
|
+
envelope: buildPauseEnvelope(ctx.inputs.runId, pauseState, reason, ctx.deps.planPath ?? '<plan.json>', b.pause?.resumeArg ?? null, {
|
|
1326
|
+
runStatePath: ctx.deps.runStatePath,
|
|
1327
|
+
runDirArg: ctx.deps.runDirArg ?? null,
|
|
1328
|
+
}),
|
|
1329
|
+
};
|
|
1330
|
+
}
|
|
1331
|
+
function terminalOutcome(ctx, route, _startedAt) {
|
|
1332
|
+
flush(ctx); // NO traceClose: parity with the render's terminal return skipping the epilogue
|
|
1333
|
+
ctx.state.status = 'completed';
|
|
1334
|
+
ctx.state.updatedAt = ctx.deps.now();
|
|
1335
|
+
stampTraceBinding(ctx.state, ctx.deps.store);
|
|
1336
|
+
ctx.deps.lock(() => ctx.deps.store.writeRunState(ctx.state));
|
|
1337
|
+
appendLedger(ctx, route);
|
|
1338
|
+
return {
|
|
1339
|
+
kind: 'completed',
|
|
1340
|
+
exitCode: 0,
|
|
1341
|
+
result: {
|
|
1342
|
+
schema: WF_RUN_RESULT_SCHEMA,
|
|
1343
|
+
runId: ctx.inputs.runId,
|
|
1344
|
+
status: 'completed',
|
|
1345
|
+
exitCode: 0,
|
|
1346
|
+
terminalRoute: route, // MEDIUM-13: a wrapper can now tell this from an ordinary completion
|
|
1347
|
+
...(ctx.deps.dispatcherOverride === true ? { dispatcherOverride: true } : {}),
|
|
1348
|
+
},
|
|
1349
|
+
};
|
|
1350
|
+
}
|
|
1351
|
+
/** FAIL — flush WITHOUT `run.closed`, name the reason, and emit NO pause envelope (AM-16: that is
|
|
1352
|
+
* exactly what lets a wrapper tell a pause from a failure using stdout and the exit code alone). */
|
|
1353
|
+
function failOutcome(runId, reason, detail, ctx) {
|
|
1354
|
+
if (ctx !== null) {
|
|
1355
|
+
flush(ctx);
|
|
1356
|
+
ctx.state.status = 'failed';
|
|
1357
|
+
ctx.state.failure = { reason, detail };
|
|
1358
|
+
ctx.state.updatedAt = ctx.deps.now();
|
|
1359
|
+
ctx.deps.lock(() => ctx.deps.store.writeRunState(ctx.state));
|
|
1360
|
+
appendLedger(ctx, 'failed');
|
|
1361
|
+
}
|
|
1362
|
+
return {
|
|
1363
|
+
kind: 'failed',
|
|
1364
|
+
exitCode: 1,
|
|
1365
|
+
reason,
|
|
1366
|
+
detail,
|
|
1367
|
+
result: {
|
|
1368
|
+
schema: WF_RUN_RESULT_SCHEMA,
|
|
1369
|
+
runId,
|
|
1370
|
+
status: 'failed',
|
|
1371
|
+
reason,
|
|
1372
|
+
exitCode: 1,
|
|
1373
|
+
...(ctx?.deps.dispatcherOverride === true ? { dispatcherOverride: true } : {}),
|
|
1374
|
+
},
|
|
1375
|
+
};
|
|
1376
|
+
}
|
|
1377
|
+
//# sourceMappingURL=workflow-run.js.map
|