faberun 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +131 -0
- package/bin/faberun.mjs +25 -0
- package/integrations/claude-code/statusline-bench.sh +42 -0
- package/integrations/claude-code/statusline.sh +80 -0
- package/package.json +33 -0
- package/skills/faberun/SKILL.md +24 -0
- package/skills/faberun/references/contract.md +380 -0
- package/skills/faberun/references/engineering.md +29 -0
- package/skills/faberun/references/handoffs.md +26 -0
- package/skills/faberun/references/operations.md +184 -0
- package/skills/faberun/references/rules.md +35 -0
- package/skills/faberun/references/workflow.md +23 -0
- package/skills/init-agentkit/SKILL.md +108 -0
- package/skills/init-agentkit/scripts/install-agentkit.sh +127 -0
- package/skills/init-agentkit/templates/.claude/commands/create-adr.md +44 -0
- package/skills/init-agentkit/templates/.github/workflows/quality.yml +43 -0
- package/skills/init-agentkit/templates/.sentrux/baseline.json +9 -0
- package/skills/init-agentkit/templates/.sentrux/rules.toml +21 -0
- package/skills/init-agentkit/templates/AGENTS.md +110 -0
- package/skills/init-agentkit/templates/docs/ABSTRACTIONS.md +30 -0
- package/skills/init-agentkit/templates/docs/ARCHITECTURE.md +31 -0
- package/skills/init-agentkit/templates/docs/GETTING-STARTED.md +44 -0
- package/skills/init-agentkit/templates/docs/VISION.md +33 -0
- package/skills/init-agentkit/templates/docs/adr/0001-record-architecture-decisions.md +36 -0
- package/skills/init-agentkit/templates/docs/adr/0002-root-managed-ai-guidance.md +37 -0
- package/skills/init-agentkit/templates/docs/adr/0003-sentrux-structural-quality-gates.md +49 -0
- package/skills/init-agentkit/templates/docs/adr/README.md +52 -0
- package/skills/init-agentkit/templates/docs/sentrux.md +66 -0
- package/skills/init-agentkit/templates/githooks/commit-msg +22 -0
- package/skills/init-agentkit/templates/githooks/pre-commit +32 -0
- package/src/campaign/brief.mjs +394 -0
- package/src/campaign/chain.mjs +555 -0
- package/src/campaign/handoff.mjs +516 -0
- package/src/campaign/index.mjs +300 -0
- package/src/campaign/journal.mjs +347 -0
- package/src/campaign/layout.mjs +51 -0
- package/src/campaign/metrics-evals.mjs +25 -0
- package/src/campaign/metrics.mjs +517 -0
- package/src/campaign/projection.mjs +250 -0
- package/src/campaign/record.mjs +102 -0
- package/src/campaign/unpark.mjs +56 -0
- package/src/cli/brand.mjs +205 -0
- package/src/cli/campaign.mjs +730 -0
- package/src/cli/contract.mjs +67 -0
- package/src/cli/init.mjs +170 -0
- package/src/cli/launch.mjs +239 -0
- package/src/cli/seat.mjs +139 -0
- package/src/cli/setup.mjs +294 -0
- package/src/cli/skills.mjs +105 -0
- package/src/cli/update.mjs +216 -0
- package/src/cli.mjs +525 -0
- package/src/contract/articles.mjs +12 -0
- package/src/contract/assert.mjs +162 -0
- package/src/contract/definition-of-done.mjs +97 -0
- package/src/contract/final-verification.mjs +96 -0
- package/src/contract/index.mjs +641 -0
- package/src/contract/judge-envelope.mjs +25 -0
- package/src/contract/review-modes.mjs +151 -0
- package/src/contract/runtime.mjs +204 -0
- package/src/contract/schema-version.mjs +25 -0
- package/src/contract/scope-findings.mjs +77 -0
- package/src/contract/snapshot.mjs +639 -0
- package/src/contract/task-packet.mjs +495 -0
- package/src/contract/untrusted.mjs +75 -0
- package/src/contract/verification.mjs +185 -0
- package/src/contract/worker-result.mjs +138 -0
- package/src/engine/assignment.mjs +63 -0
- package/src/engine/backoff.mjs +492 -0
- package/src/engine/bulk-read.mjs +361 -0
- package/src/engine/cancel.mjs +177 -0
- package/src/engine/detach.mjs +101 -0
- package/src/engine/dispatch.mjs +752 -0
- package/src/engine/failover.mjs +192 -0
- package/src/engine/gate.mjs +183 -0
- package/src/engine/judge-gate.mjs +517 -0
- package/src/engine/lifecycle.mjs +772 -0
- package/src/engine/live-preflight.mjs +299 -0
- package/src/engine/mutation.mjs +146 -0
- package/src/engine/notify-queue.mjs +327 -0
- package/src/engine/process-identity.mjs +72 -0
- package/src/engine/process.mjs +774 -0
- package/src/engine/prompts.mjs +289 -0
- package/src/engine/recover.mjs +300 -0
- package/src/engine/result-file.mjs +222 -0
- package/src/engine/resume.mjs +635 -0
- package/src/engine/retry.mjs +334 -0
- package/src/engine/review.mjs +228 -0
- package/src/engine/run-command.mjs +287 -0
- package/src/engine/run-identity.mjs +411 -0
- package/src/engine/runtime-discovery.mjs +235 -0
- package/src/engine/scheduler.mjs +526 -0
- package/src/engine/scope.mjs +378 -0
- package/src/engine/settle.mjs +207 -0
- package/src/engine/state.mjs +148 -0
- package/src/engine/supervise.mjs +713 -0
- package/src/engine/verify.mjs +167 -0
- package/src/harnesses/agy/index.mjs +62 -0
- package/src/harnesses/catalogue.mjs +509 -0
- package/src/harnesses/claude/index.mjs +90 -0
- package/src/harnesses/codex/index.mjs +87 -0
- package/src/harnesses/dsh/closed-packet.patch.yml +42 -0
- package/src/harnesses/dsh/index.mjs +210 -0
- package/src/harnesses/dsh/runner.mjs +259 -0
- package/src/harnesses/exec-jsonl/index.mjs +788 -0
- package/src/harnesses/index.mjs +508 -0
- package/src/harnesses/protocol.mjs +531 -0
- package/src/harnesses/replay/bin.mjs +386 -0
- package/src/harnesses/replay/index.mjs +238 -0
- package/src/harnesses/zcode/index.mjs +276 -0
- package/src/host/config.mjs +87 -0
- package/src/host/home.mjs +149 -0
- package/src/host/package.mjs +23 -0
- package/src/host/preflight.mjs +520 -0
- package/src/host/tool-policy-decisions.mjs +341 -0
- package/src/host/tool-policy-hook.mjs +270 -0
- package/src/notify/index.mjs +359 -0
- package/src/notify/os-macos.mjs +81 -0
- package/src/repo/declared-paths.mjs +220 -0
- package/src/repo/integrate.mjs +546 -0
- package/src/repo/scope-closure.mjs +665 -0
- package/src/repo/signal-block.mjs +16 -0
- package/src/repo/signal.mjs +222 -0
- package/src/repo/source-identity.mjs +295 -0
- package/src/repo/workspace.mjs +557 -0
- package/src/repo/worktree.mjs +352 -0
- package/src/report/final.mjs +200 -0
- package/src/report/metrics-report.mjs +99 -0
- package/src/report/next.mjs +383 -0
- package/src/report/render.mjs +716 -0
- package/src/run/disk-gc.mjs +251 -0
- package/src/run/lock.mjs +329 -0
- package/src/run/node-store.mjs +62 -0
- package/src/run/operations.mjs +286 -0
- package/src/run/store.mjs +187 -0
- package/src/run/usage.mjs +337 -0
- package/src/seat/harnesses.mjs +83 -0
- package/src/seat/index.mjs +239 -0
- package/src/seat/tmux.mjs +208 -0
- package/src/util.mjs +0 -0
- package/src/web/api.mjs +371 -0
- package/src/web/boundary.mjs +88 -0
- package/src/web/index.html +299 -0
- package/src/web/server.mjs +552 -0
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The write boundary: what a node was allowed to touch, what it actually
|
|
3
|
+
* touched, and what to do when those differ.
|
|
4
|
+
*
|
|
5
|
+
* Scope is advisory by design -- an unexpected write is recorded as a finding
|
|
6
|
+
* and shown to the judge, not treated as a crime -- with one exception:
|
|
7
|
+
* `resolveUnknownEffect` decides whether an invocation whose effect is unproven
|
|
8
|
+
* may be replayed at all, and a dirty scope there is a refusal.
|
|
9
|
+
*/
|
|
10
|
+
import { SETTLED } from "./prompts.mjs";
|
|
11
|
+
import { appendTransitionEvent, recordExecutionOverride, transition, writeNode } from "./state.mjs";
|
|
12
|
+
import { attemptWorkspace } from "../repo/worktree.mjs";
|
|
13
|
+
|
|
14
|
+
import { errorCode, errorMessage, excerpt } from "../util.mjs";
|
|
15
|
+
import { executeControllerVerification } from "./verify.mjs";
|
|
16
|
+
import { providerReceiptsFromInvocationTail, settleInvocation } from "../run/operations.mjs";
|
|
17
|
+
import { readJson } from "../run/store.mjs";
|
|
18
|
+
import { scopeFindingFromScope } from "../contract/scope-findings.mjs";
|
|
19
|
+
import { captureWorkspaceScope, compareWorkspaceSnapshot, validateWorkspaceScopeBoundary } from "../repo/workspace.mjs";
|
|
20
|
+
|
|
21
|
+
/** @typedef {import("../contract/index.mjs").BoundedScope} BoundedScope */
|
|
22
|
+
/** @typedef {import("./lifecycle.mjs").Invocation} Invocation */
|
|
23
|
+
/** @typedef {import("./lifecycle.mjs").Job} Job */
|
|
24
|
+
/** @typedef {import("../cli.mjs").LockHandle} LockHandle */
|
|
25
|
+
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
26
|
+
/** @typedef {import("../run/usage.mjs").RecoveryOutcome} RecoveryOutcome */
|
|
27
|
+
/** @typedef {import("../repo/workspace.mjs").ScopeComparison} ScopeComparison */
|
|
28
|
+
/** @typedef {import("../contract/index.mjs").TaskPacket} TaskPacket */
|
|
29
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
30
|
+
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
31
|
+
/** @typedef {import("../repo/workspace.mjs").WorkspaceScopeBoundary} WorkspaceScopeBoundary */
|
|
32
|
+
/** @typedef {import("../repo/workspace.mjs").WorkspaceSnapshot} WorkspaceSnapshot */
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* @param {NodeSnapshot} state
|
|
36
|
+
* @returns {string|null}
|
|
37
|
+
*/
|
|
38
|
+
export function sourceWorkerRuntime(state) {
|
|
39
|
+
return [...(state.invocations ?? [])].reverse().find((invocation) => invocation.phase === "worker")?.runtimeId
|
|
40
|
+
?? state.runtime?.id
|
|
41
|
+
?? null;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* @param {ScopeComparison} scope
|
|
45
|
+
* @param {import("../repo/workspace.mjs").WorkspaceScopeBoundary} boundary
|
|
46
|
+
* @returns {BoundedScope}
|
|
47
|
+
*/
|
|
48
|
+
function boundedScope(scope, boundary) {
|
|
49
|
+
return {
|
|
50
|
+
boundary,
|
|
51
|
+
changedPaths: scope.changedPaths.slice(0, 64),
|
|
52
|
+
unexpectedPaths: scope.unexpectedPaths.slice(0, 64),
|
|
53
|
+
changedPathCount: scope.changedPaths.length,
|
|
54
|
+
unexpectedPathCount: scope.unexpectedPaths.length,
|
|
55
|
+
truncated: scope.changedPaths.length > 64 || scope.unexpectedPaths.length > 64,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* @param {import("../repo/workspace.mjs").WorkspaceScopeBoundary} boundary
|
|
60
|
+
* @returns {BoundedScope}
|
|
61
|
+
*/
|
|
62
|
+
export function emptyScope(boundary) {
|
|
63
|
+
if (!boundary) throw Object.assign(new Error("worker scope boundary is missing"), { code: "scope_boundary_missing" });
|
|
64
|
+
return {
|
|
65
|
+
boundary,
|
|
66
|
+
changedPaths: [],
|
|
67
|
+
unexpectedPaths: [],
|
|
68
|
+
changedPathCount: 0,
|
|
69
|
+
unexpectedPathCount: 0,
|
|
70
|
+
truncated: false,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* @param {ValidatedContract} contract
|
|
75
|
+
* @returns {Map<string, import("../repo/workspace.mjs").WorkspaceScopeBoundary>}
|
|
76
|
+
*/
|
|
77
|
+
export function captureNodeScopeBoundaries(contract) {
|
|
78
|
+
return new Map(contract.nodes.map((node) => [node.id, captureWorkspaceScope(contract.cwd, workerScope(node.taskPacket))]));
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* @param {ValidatedContract} contract
|
|
82
|
+
* @param {ValidatedNode} node
|
|
83
|
+
* @param {NodeSnapshot|undefined} state
|
|
84
|
+
* @returns {import("../repo/workspace.mjs").WorkspaceScopeBoundary}
|
|
85
|
+
*/
|
|
86
|
+
export function persistedScopeBoundary(contract, node, state, workspace = contract.cwd) {
|
|
87
|
+
const boundary = state?.scope?.boundary;
|
|
88
|
+
if (!boundary) throw Object.assign(new Error(`node ${node.id} has no persisted worker scope boundary`), { code: "scope_boundary_missing" });
|
|
89
|
+
return validateWorkspaceScopeBoundary(workspace, boundary, workerScope(node.taskPacket));
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* @param {import("../contract/index.mjs").TaskPacket} taskPacket
|
|
93
|
+
* @returns {{files: string[], roots: string[]}}
|
|
94
|
+
*/
|
|
95
|
+
export function workerScope(taskPacket) {
|
|
96
|
+
return {
|
|
97
|
+
files: taskPacket.writeFiles ?? [],
|
|
98
|
+
roots: taskPacket.writeRoots ?? [],
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* @param {ValidatedContract} contract
|
|
103
|
+
* @param {string} runDir
|
|
104
|
+
* @param {Job} job
|
|
105
|
+
* @param {LockHandle} lock
|
|
106
|
+
* @param {{deferViolation?: boolean}} [options]
|
|
107
|
+
* @returns {boolean}
|
|
108
|
+
*/
|
|
109
|
+
export function checkWorkerScope(contract, runDir, job, lock, options = {}) {
|
|
110
|
+
if (job.scopeChecked) return !job.scopeViolation;
|
|
111
|
+
job.scopeChecked = true;
|
|
112
|
+
const state = job.state;
|
|
113
|
+
try {
|
|
114
|
+
const baseline = /** @type {WorkspaceSnapshot|undefined} */ (job.scopeBaseline ?? (job.invocation.snapshotPath ? readJson(job.invocation.snapshotPath) : null));
|
|
115
|
+
if (!baseline) throw Object.assign(new Error("worker scope snapshot is missing"), { code: "scope_snapshot_missing" });
|
|
116
|
+
const boundary = persistedScopeBoundary(contract, job.node, state, job.cwd);
|
|
117
|
+
const scope = compareWorkspaceSnapshot(baseline, job.cwd, { ...workerScope(job.node.taskPacket), boundary });
|
|
118
|
+
const bounded = boundedScope(scope, boundary);
|
|
119
|
+
state.scope = bounded;
|
|
120
|
+
if (!scope.unexpectedPaths.length) return true;
|
|
121
|
+
job.scopeViolation = true;
|
|
122
|
+
// A completed attempt whose controller verification passes never fails
|
|
123
|
+
// on scope alone (TECH-SPEC lean, rule 1): the caller defers the verdict
|
|
124
|
+
// until verification has run and records an advisory finding instead.
|
|
125
|
+
if (options.deferViolation) return true;
|
|
126
|
+
const shown = bounded.unexpectedPaths.slice(0, 8).join(", ");
|
|
127
|
+
const message = `unexpected paths changed (${scope.unexpectedPaths.length}): ${shown}`;
|
|
128
|
+
if (!SETTLED.has(state.status)) {
|
|
129
|
+
transition(runDir, state, "failed", { phase: "worker", error: { code: "unexpected_write", message: excerpt(message) } }, lock);
|
|
130
|
+
appendTransitionEvent(runDir, state, "failed", "failed", {
|
|
131
|
+
unexpectedPaths: bounded.unexpectedPaths,
|
|
132
|
+
unexpectedPathCount: bounded.unexpectedPathCount,
|
|
133
|
+
}, lock);
|
|
134
|
+
}
|
|
135
|
+
return false;
|
|
136
|
+
} catch (error) {
|
|
137
|
+
job.scopeViolation = true;
|
|
138
|
+
if (!SETTLED.has(state.status)) {
|
|
139
|
+
transition(runDir, state, "failed", { phase: "worker", error: { code: /** @type {string} */ (errorCode(error) ?? "scope_snapshot_invalid"), message: excerpt(errorMessage(error)) } }, lock);
|
|
140
|
+
}
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* A result-only continuation is not implementation work. Its baseline is
|
|
146
|
+
* captured immediately before that single turn, so every workspace change is
|
|
147
|
+
* outside its authority (the run-owned result file is ignored by snapshots).
|
|
148
|
+
*
|
|
149
|
+
* @param {ValidatedContract} contract
|
|
150
|
+
* @param {string} runDir
|
|
151
|
+
* @param {Job} job
|
|
152
|
+
* @param {LockHandle} lock
|
|
153
|
+
* @param {string} [label]
|
|
154
|
+
* @returns {boolean}
|
|
155
|
+
*/
|
|
156
|
+
export function checkResultMaterializationScope(contract, runDir, job, lock, label = "result materialization") {
|
|
157
|
+
if (job.scopeChecked) return !job.scopeViolation;
|
|
158
|
+
job.scopeChecked = true;
|
|
159
|
+
const state = job.state;
|
|
160
|
+
try {
|
|
161
|
+
const baseline = /** @type {WorkspaceSnapshot|undefined} */ (job.recoveryBaseline ?? (job.invocation.snapshotPath ? readJson(job.invocation.snapshotPath) : null));
|
|
162
|
+
if (!baseline) throw Object.assign(new Error(`${label} scope snapshot is missing`), { code: "scope_snapshot_missing" });
|
|
163
|
+
const scope = compareWorkspaceSnapshot(baseline, job.cwd);
|
|
164
|
+
if (!scope.changedPaths.length) return true;
|
|
165
|
+
job.scopeViolation = true;
|
|
166
|
+
const shown = scope.changedPaths.slice(0, 8).join(", ");
|
|
167
|
+
transition(runDir, state, "failed", {
|
|
168
|
+
phase: "worker",
|
|
169
|
+
error: { code: "unexpected_write", message: excerpt(`${label} changed workspace paths (${scope.changedPaths.length}): ${shown}`) },
|
|
170
|
+
}, lock);
|
|
171
|
+
return false;
|
|
172
|
+
} catch (error) {
|
|
173
|
+
job.scopeViolation = true;
|
|
174
|
+
transition(runDir, state, "failed", {
|
|
175
|
+
phase: "worker",
|
|
176
|
+
error: { code: /** @type {string} */ (errorCode(error) ?? "scope_snapshot_invalid"), message: excerpt(errorMessage(error)) },
|
|
177
|
+
}, lock);
|
|
178
|
+
return false;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Materialization can reuse prior controller evidence only when that evidence
|
|
183
|
+
* survived the same worker attempt. A fresh worker clears verification; the
|
|
184
|
+
* retained, completed record is therefore the bounded same-attempt proof.
|
|
185
|
+
* Judge evidence cannot currently carry that identity, so gated nodes fail
|
|
186
|
+
* closed and run their normal judge phase.
|
|
187
|
+
*
|
|
188
|
+
* @param {NodeSnapshot} state
|
|
189
|
+
* @param {ValidatedNode} node
|
|
190
|
+
* @returns {boolean}
|
|
191
|
+
*/
|
|
192
|
+
export function canReuseResultEvidence(state, node) {
|
|
193
|
+
if (state.verification?.completed !== true || state.verification.passed !== true) return false;
|
|
194
|
+
return !node.gate.enabled;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Compare the current workspace against the persisted worker baseline without
|
|
198
|
+
* transitioning the node. Shared by the unexpected-write failure path and the
|
|
199
|
+
* unknown_effect replay gate.
|
|
200
|
+
*
|
|
201
|
+
* @param {ValidatedContract} contract
|
|
202
|
+
* @param {ValidatedNode} node
|
|
203
|
+
* @param {NodeSnapshot} state
|
|
204
|
+
* @param {Invocation|undefined} invocation
|
|
205
|
+
* @param {{strict?: boolean}} [options]
|
|
206
|
+
* @returns {{ok: true}|{ok: false, code: string, detail: string, unexpectedPaths?: string[], unexpectedPathCount?: number, changedPaths?: string[], changedPathCount?: number}}
|
|
207
|
+
*/
|
|
208
|
+
function evaluatePersistedWorkerScope(contract, node, state, invocation, options = {}) {
|
|
209
|
+
const strict = options.strict === true;
|
|
210
|
+
try {
|
|
211
|
+
const baseline = invocation?.snapshotPath
|
|
212
|
+
? /** @type {WorkspaceSnapshot} */ (readJson(invocation.snapshotPath))
|
|
213
|
+
: null;
|
|
214
|
+
if (!baseline) return { ok: false, code: "scope_snapshot_missing", detail: "worker scope snapshot is missing" };
|
|
215
|
+
const workspace = attemptWorkspace(state) ?? invocation?.workspace ?? contract.cwd;
|
|
216
|
+
const boundary = persistedScopeBoundary(contract, node, state, workspace);
|
|
217
|
+
const scope = compareWorkspaceSnapshot(baseline, workspace, { ...workerScope(node.taskPacket), boundary });
|
|
218
|
+
const bounded = boundedScope(scope, boundary);
|
|
219
|
+
state.scope = bounded;
|
|
220
|
+
if (!scope.unexpectedPaths.length && (!strict || scope.changedPaths.length === 0)) return { ok: true };
|
|
221
|
+
if (scope.unexpectedPaths.length) {
|
|
222
|
+
const shown = bounded.unexpectedPaths.slice(0, 8).join(", ");
|
|
223
|
+
return {
|
|
224
|
+
ok: false,
|
|
225
|
+
code: "unexpected_write",
|
|
226
|
+
detail: `unexpected paths changed (${scope.unexpectedPaths.length}): ${shown}`,
|
|
227
|
+
unexpectedPaths: bounded.unexpectedPaths,
|
|
228
|
+
unexpectedPathCount: bounded.unexpectedPathCount,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
ok: false,
|
|
233
|
+
code: "declared_paths_changed",
|
|
234
|
+
detail: `declared workspace paths changed across the ambiguous window (${scope.changedPaths.length}): ${bounded.changedPaths.slice(0, 8).join(", ")}`,
|
|
235
|
+
changedPaths: bounded.changedPaths,
|
|
236
|
+
changedPathCount: bounded.changedPathCount,
|
|
237
|
+
};
|
|
238
|
+
} catch (error) {
|
|
239
|
+
return { ok: false, code: /** @type {string} */ (errorCode(error) ?? "scope_snapshot_invalid"), detail: errorMessage(error) };
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* Gate a worker restart behind proof that the replay cannot duplicate effects.
|
|
244
|
+
* Declared workspace changes are not proof of absence: any change across the
|
|
245
|
+
* ambiguous window (declared or unexpected) or a missing scope baseline is
|
|
246
|
+
* reconciled as terminal attention instead of silently replaying the attempt.
|
|
247
|
+
* Returns true when the restart must not proceed (the node was blocked).
|
|
248
|
+
*
|
|
249
|
+
* @param {ValidatedContract} contract
|
|
250
|
+
* @param {string} runDir
|
|
251
|
+
* @param {ValidatedNode} node
|
|
252
|
+
* @param {NodeSnapshot} state
|
|
253
|
+
* @param {Invocation|undefined} invocation
|
|
254
|
+
* @param {RecoveryOutcome} recovery
|
|
255
|
+
* @param {Record<string, unknown>|undefined} persistedRecovery
|
|
256
|
+
* @param {LockHandle} lock
|
|
257
|
+
* @returns {boolean}
|
|
258
|
+
*/
|
|
259
|
+
export function reconcileAmbiguousWorkerRestart(contract, runDir, node, state, invocation, recovery, persistedRecovery, lock) {
|
|
260
|
+
const evaluation = evaluatePersistedWorkerScope(contract, node, state, invocation, { strict: true });
|
|
261
|
+
if (evaluation.ok) return false;
|
|
262
|
+
const invocationId = recovery.invocationId ?? invocation?.id;
|
|
263
|
+
const reason = evaluation.code === "declared_paths_changed"
|
|
264
|
+
? `declared workspace changes across the ambiguous window are not proof that replay cannot duplicate effects for node ${state.id}: ${evaluation.detail}`
|
|
265
|
+
: evaluation.code === "unexpected_write"
|
|
266
|
+
? `workspace moved outside the declared write scope across the ambiguous window: ${evaluation.detail}`
|
|
267
|
+
: `the ambiguous worker window for node ${state.id} cannot prove replay safety: ${evaluation.detail}`;
|
|
268
|
+
if (invocationId) {
|
|
269
|
+
settleInvocation(runDir, invocationId, {
|
|
270
|
+
status: "reconciled",
|
|
271
|
+
usage: invocation?.usage ?? recovery.usage ?? null,
|
|
272
|
+
costUsd: typeof invocation?.costUsd === "number" ? invocation.costUsd : recovery.costUsd ?? null,
|
|
273
|
+
receipts: providerReceiptsFromInvocationTail(contract, invocation),
|
|
274
|
+
unknownEffect: true,
|
|
275
|
+
classification: "unknown_effect",
|
|
276
|
+
reason,
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
if (!persistedRecovery) recordExecutionOverride(runDir, state, {
|
|
280
|
+
kind: "recovery",
|
|
281
|
+
decision: "reconciled",
|
|
282
|
+
invocationId,
|
|
283
|
+
phase: recovery.phase,
|
|
284
|
+
reason,
|
|
285
|
+
}, lock);
|
|
286
|
+
transition(runDir, state, "blocked", {
|
|
287
|
+
phase: recovery.phase,
|
|
288
|
+
error: { code: "unknown_effect_reconciled", message: excerpt(reason) },
|
|
289
|
+
}, lock);
|
|
290
|
+
return true;
|
|
291
|
+
}
|
|
292
|
+
/**
|
|
293
|
+
* @param {ValidatedContract} contract
|
|
294
|
+
* @param {string} runDir
|
|
295
|
+
* @param {NodeSnapshot} state
|
|
296
|
+
* @param {ValidatedNode} node
|
|
297
|
+
* @param {Invocation|undefined} invocation
|
|
298
|
+
* @param {LockHandle} lock
|
|
299
|
+
* @param {{materialization?: boolean}} [options]
|
|
300
|
+
* @returns {boolean}
|
|
301
|
+
*/
|
|
302
|
+
export function checkPersistedWorkerScope(contract, runDir, state, node, invocation, lock, options = {}) {
|
|
303
|
+
const materialization = options.materialization === true;
|
|
304
|
+
const evaluation = evaluatePersistedWorkerScope(contract, node, state, invocation, { strict: materialization });
|
|
305
|
+
if (evaluation.ok) return true;
|
|
306
|
+
// A recovered materialization turn mirrors the live check: it may only write
|
|
307
|
+
// the canonical result file, so declared-path changes are as terminal as
|
|
308
|
+
// unexpected ones.
|
|
309
|
+
const failure = materialization && (evaluation.code === "unexpected_write" || evaluation.code === "declared_paths_changed")
|
|
310
|
+
? {
|
|
311
|
+
code: "unexpected_write",
|
|
312
|
+
message: excerpt(`result materialization changed workspace paths (${evaluation.changedPathCount ?? evaluation.unexpectedPathCount}): ${(evaluation.changedPaths ?? evaluation.unexpectedPaths ?? []).slice(0, 8).join(", ")}`),
|
|
313
|
+
}
|
|
314
|
+
: { code: evaluation.code, message: excerpt(evaluation.detail) };
|
|
315
|
+
transition(runDir, state, "failed", {
|
|
316
|
+
phase: "worker",
|
|
317
|
+
error: failure,
|
|
318
|
+
}, lock);
|
|
319
|
+
if (evaluation.unexpectedPaths) {
|
|
320
|
+
appendTransitionEvent(runDir, state, "failed", "failed", {
|
|
321
|
+
unexpectedPaths: evaluation.unexpectedPaths,
|
|
322
|
+
unexpectedPathCount: evaluation.unexpectedPathCount,
|
|
323
|
+
}, lock);
|
|
324
|
+
}
|
|
325
|
+
return false;
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Record a deferred scope violation as an advisory finding on a node whose
|
|
329
|
+
* controller verification passed: the node proceeds into the gate exactly as
|
|
330
|
+
* a clean node would (TECH-SPEC lean, rule 1).
|
|
331
|
+
*
|
|
332
|
+
* @param {string} runDir
|
|
333
|
+
* @param {NodeSnapshot} state
|
|
334
|
+
* @param {LockHandle} lock
|
|
335
|
+
*/
|
|
336
|
+
export function recordScopeFinding(runDir, state, lock) {
|
|
337
|
+
if (!state.scope?.unexpectedPaths?.length) return;
|
|
338
|
+
state.scopeFindings = scopeFindingFromScope(state.scope);
|
|
339
|
+
writeNode(runDir, state, lock);
|
|
340
|
+
appendTransitionEvent(runDir, state, state.status, state.status, {
|
|
341
|
+
type: "scope.finding",
|
|
342
|
+
unexpectedPaths: state.scopeFindings.unexpectedPaths,
|
|
343
|
+
unexpectedPathCount: state.scope.unexpectedPathCount,
|
|
344
|
+
}, lock);
|
|
345
|
+
}
|
|
346
|
+
/**
|
|
347
|
+
* Resolve an unknown_effect window (intent without settlement) per the node's
|
|
348
|
+
* replayPolicy. Adoption proof was already applied by recoverOrphan when it
|
|
349
|
+
* applied; what remains is the scoped safe-replay or a durable reconcile.
|
|
350
|
+
*
|
|
351
|
+
* @param {ValidatedContract} contract
|
|
352
|
+
* @param {string} runDir
|
|
353
|
+
* @param {ValidatedNode} node
|
|
354
|
+
* @param {NodeSnapshot} state
|
|
355
|
+
* @param {Invocation|undefined} workerInvocation
|
|
356
|
+
* @param {LockHandle} lock
|
|
357
|
+
* @returns {Promise<{action: "replay"}|{action: "reconcile", reason: string}>}
|
|
358
|
+
*/
|
|
359
|
+
export async function resolveUnknownEffect(contract, runDir, node, state, workerInvocation, lock) {
|
|
360
|
+
const policy = node.replayPolicy ?? "safe";
|
|
361
|
+
if (policy !== "safe") {
|
|
362
|
+
return { action: "reconcile", reason: `node ${state.id} declares replayPolicy ${policy}; the interrupted attempt with unknown effects requires manual reconciliation` };
|
|
363
|
+
}
|
|
364
|
+
const evaluation = evaluatePersistedWorkerScope(contract, node, state, workerInvocation, { strict: true });
|
|
365
|
+
if (!evaluation.ok) {
|
|
366
|
+
return {
|
|
367
|
+
action: "reconcile",
|
|
368
|
+
reason: evaluation.code === "declared_paths_changed"
|
|
369
|
+
? `declared workspace changes across the ambiguous window are not proof that replay cannot duplicate effects for node ${state.id}: ${evaluation.detail}`
|
|
370
|
+
: `workspace moved outside the declared write scope across the ambiguous window: ${evaluation.detail}`,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
await executeControllerVerification(contract, runDir, node, state, lock);
|
|
374
|
+
if (!state.verification?.passed) {
|
|
375
|
+
return { action: "reconcile", reason: `deterministic verification failed while resolving the ambiguous window for node ${state.id}; partial effects cannot be proven absent` };
|
|
376
|
+
}
|
|
377
|
+
return { action: "replay" };
|
|
378
|
+
}
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How a node ends: accepted and integrated, rejected, or parked for attention.
|
|
3
|
+
*
|
|
4
|
+
* These are the settlement primitives, and they are their own module because
|
|
5
|
+
* both the control loop and the review policy call them -- `settleDone`'s
|
|
6
|
+
* `onVerificationFailure` callback rejects the candidate, and `review.mjs`
|
|
7
|
+
* settles a node whose judge passed. Leaving them in `engine/lifecycle.mjs`
|
|
8
|
+
* made those two import each other, which was the last runtime import cycle in
|
|
9
|
+
* `src/` and the one the allowlist used to name.
|
|
10
|
+
*
|
|
11
|
+
* `settleDone` is the only place a node becomes `done`, and it does it through
|
|
12
|
+
* the integration transaction: the journal and the conditional ref update
|
|
13
|
+
* belong to `repo/integrate.mjs`, and the callbacks here own node state alone.
|
|
14
|
+
*/
|
|
15
|
+
import { candidateOnlyFailures, resetPhaseRouting, verificationFailureVerdict } from "./judge-gate.mjs";
|
|
16
|
+
import { retryPrompt } from "./prompts.mjs";
|
|
17
|
+
import { renderPreviousAttemptSection } from "./retry.mjs";
|
|
18
|
+
import { startWorker } from "./dispatch.mjs";
|
|
19
|
+
import { ensureTerminalEvent, transition, writeNode } from "./state.mjs";
|
|
20
|
+
import { verificationFailureWithScope } from "../contract/scope-findings.mjs";
|
|
21
|
+
import { alreadyNotified, notifyQueueFor } from "./notify-queue.mjs";
|
|
22
|
+
|
|
23
|
+
import {
|
|
24
|
+
attemptWorkspace,
|
|
25
|
+
attemptWorktreePath,
|
|
26
|
+
removeWorktree,
|
|
27
|
+
sealAttempt,
|
|
28
|
+
} from "../repo/worktree.mjs";
|
|
29
|
+
import { basename } from "node:path";
|
|
30
|
+
import { boundedUtf8, errorCode, errorMessage } from "../util.mjs";
|
|
31
|
+
import { campaignIdOf } from "../campaign/record.mjs";
|
|
32
|
+
import { integrateAttempt } from "../repo/integrate.mjs";
|
|
33
|
+
import { verifyCandidateWorkspace } from "./verify.mjs";
|
|
34
|
+
|
|
35
|
+
/** @typedef {import("../repo/integrate.mjs").IntegrationResult} IntegrationResult */
|
|
36
|
+
/** @typedef {import("./lifecycle.mjs").Job} Job */
|
|
37
|
+
/** @typedef {import("./prompts.mjs").JudgeVerdict} JudgeVerdict */
|
|
38
|
+
/** @typedef {import("../cli.mjs").LockHandle} LockHandle */
|
|
39
|
+
/** @typedef {import("../contract/index.mjs").NodeSnapshot} NodeSnapshot */
|
|
40
|
+
/** @typedef {import("../contract/index.mjs").ValidatedContract} ValidatedContract */
|
|
41
|
+
/** @typedef {import("../contract/index.mjs").ValidatedNode} ValidatedNode */
|
|
42
|
+
|
|
43
|
+
/** Settle one worker-generation rejection: bounded revision when one remains, otherwise terminal exhausted/failed. @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} verdict @param {{code: string, label: string, phase?: "worker"|"judge", message?: string, forceFresh?: boolean}} options */
|
|
44
|
+
export function applyRejection(contract, node, state, runDir, running, lock, states, campaignPath, verdict, options) {
|
|
45
|
+
const { code, label, phase = "worker", message = verdict.summary, forceFresh = true } = options;
|
|
46
|
+
state.gate = verdict;
|
|
47
|
+
if (node.gate.enabled && state.revisions < (node.gate.maxRevisions ?? 1)) {
|
|
48
|
+
resetPhaseRouting(state);
|
|
49
|
+
state.revisions += 1;
|
|
50
|
+
// The fresh-session decision travels with the node, not just this call:
|
|
51
|
+
// the `running === null` branch below hands the retry to the scheduler,
|
|
52
|
+
// whose own `startWorker` call carries no policy and would otherwise
|
|
53
|
+
// rediscover the prior compatible continuation from the persisted ledger.
|
|
54
|
+
// `startWorker` consumes and clears it, so it governs one dispatch only.
|
|
55
|
+
state.sessionPolicy = forceFresh ? { forceFresh: true } : null;
|
|
56
|
+
process.stdout.write(`[${label}] ${node.id} retry · ${verdict.summary}\n`);
|
|
57
|
+
// The retry is a fresh session when the decision says so, so the evidence
|
|
58
|
+
// it needs has to travel in the prompt, not the transcript. Rendering the
|
|
59
|
+
// bounded `## Previous attempt` section here means both the immediate
|
|
60
|
+
// dispatch below and a later scheduler dispatch (the `running` is null
|
|
61
|
+
// path) carry it.
|
|
62
|
+
state.previousAttempt = renderPreviousAttemptSection(state) ?? state.previousAttempt;
|
|
63
|
+
if (running) {
|
|
64
|
+
// Dispatching here owns the increment, because `startWorker` expects the
|
|
65
|
+
// attempt number it is about to run under.
|
|
66
|
+
state.attempt += 1;
|
|
67
|
+
startWorker(contract, node, state, runDir, running, retryPrompt(node, verdict), lock, states, campaignPath, { forceFresh });
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
// Handing the node back to the scheduler instead: its dispatch increments
|
|
71
|
+
// on the way out, so incrementing here too spent two attempt numbers on one
|
|
72
|
+
// retry. Observed 2026-09-13 on a resume after a killed controller — a node
|
|
73
|
+
// that ran twice reported attempt 3, with no `…2.*` logs and a
|
|
74
|
+
// `worktree.previousAttempt` naming an attempt that never existed.
|
|
75
|
+
transition(runDir, state, "pending", { phase: "worker", error: null }, lock);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
transition(runDir, state, node.gate.enabled ? "exhausted" : "failed", {
|
|
79
|
+
phase,
|
|
80
|
+
gate: verdict,
|
|
81
|
+
error: { code, message },
|
|
82
|
+
}, lock);
|
|
83
|
+
}
|
|
84
|
+
/** Deterministic verification failure settles through the shared rejection path. The verdict carries this attempt's unexpected paths, so a red attempt reports them whether it stops here or starts its revision (TECH-SPEC lean, rule 1). @param {ValidatedContract} contract @param {ValidatedNode} node @param {NodeSnapshot} state @param {string} runDir @param {Map<string, Job>|null} running @param {LockHandle} lock @param {Map<string, NodeSnapshot>} states @param {string} campaignPath @param {JudgeVerdict} [verdict] */
|
|
85
|
+
export function applyVerificationFailure(contract, node, state, runDir, running, lock, states, campaignPath, verdict = verificationFailureWithScope(verificationFailureVerdict(state), state.scope)) {
|
|
86
|
+
applyRejection(contract, node, state, runDir, running, lock, states, campaignPath, verdict, { code: "verification_failed", label: "verification" });
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Seal the current attempt, verify its candidate in a detached worktree, and
|
|
91
|
+
* only then perform the single done-state transition. The integration module
|
|
92
|
+
* owns the journal and conditional ref update; this callback owns node state.
|
|
93
|
+
*
|
|
94
|
+
* @param {ValidatedContract} contract
|
|
95
|
+
* @param {ValidatedNode} node
|
|
96
|
+
* @param {NodeSnapshot} state
|
|
97
|
+
* @param {string} runDir
|
|
98
|
+
* @param {LockHandle} lock
|
|
99
|
+
* @param {Map<string, NodeSnapshot>} states
|
|
100
|
+
* @param {string} campaignPath
|
|
101
|
+
* @param {Partial<NodeSnapshot>} [patch]
|
|
102
|
+
* @returns {Promise<import("../repo/integrate.mjs").IntegrationResult|null|undefined>}
|
|
103
|
+
*/
|
|
104
|
+
export async function settleDone(contract, node, state, runDir, lock, states, campaignPath, patch = {}) {
|
|
105
|
+
const workspace = attemptWorkspace(state);
|
|
106
|
+
if (!workspace || !state.worktree?.branch || !state.worktree.baseSha) {
|
|
107
|
+
transition(runDir, state, "failed", {
|
|
108
|
+
phase: "complete",
|
|
109
|
+
error: { code: "attempt_worktree_missing", message: "completed attempt has no isolated worktree" },
|
|
110
|
+
}, lock);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
let sealed;
|
|
114
|
+
try {
|
|
115
|
+
sealed = sealAttempt({
|
|
116
|
+
repo: contract.cwd,
|
|
117
|
+
path: workspace,
|
|
118
|
+
baseSha: state.worktree.baseSha,
|
|
119
|
+
runId: contract.id,
|
|
120
|
+
nodeId: node.id,
|
|
121
|
+
attempt: state.attempt,
|
|
122
|
+
});
|
|
123
|
+
state.worktree = { ...state.worktree, commit: sealed.sha, status: "ready" };
|
|
124
|
+
writeNode(runDir, state, lock);
|
|
125
|
+
} catch (error) {
|
|
126
|
+
transition(runDir, state, "failed", {
|
|
127
|
+
phase: "complete",
|
|
128
|
+
error: { code: errorCode(error) ?? "attempt_seal_failed", message: errorMessage(error) },
|
|
129
|
+
}, lock);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
const result = await integrateAttempt({
|
|
133
|
+
repo: contract.cwd,
|
|
134
|
+
runDir,
|
|
135
|
+
runId: contract.id,
|
|
136
|
+
nodeId: node.id,
|
|
137
|
+
attempt: state.attempt,
|
|
138
|
+
attemptSha: sealed.sha,
|
|
139
|
+
branch: state.worktree.branch,
|
|
140
|
+
verificationEvidence: state.verification,
|
|
141
|
+
verifyCandidate: (candidateWorkspace) => verifyCandidateWorkspace(contract, node, state, runDir, candidateWorkspace),
|
|
142
|
+
onAccepted: async (transaction) => {
|
|
143
|
+
const acceptedPath = state.worktree?.path ?? attemptWorktreePath(runDir, contract.id, node.id, transaction.attempt);
|
|
144
|
+
if (state.attempt === transaction.attempt && state.status !== "done") {
|
|
145
|
+
transition(runDir, state, "done", {
|
|
146
|
+
...patch,
|
|
147
|
+
integratedHead: transaction.candidateSha,
|
|
148
|
+
worktree: { ...(state.worktree ?? {}), status: "removed", commit: transaction.attemptSha, baseSha: transaction.previousRunRefTip },
|
|
149
|
+
}, lock);
|
|
150
|
+
}
|
|
151
|
+
if (state.attempt === transaction.attempt) ensureTerminalEvent(runDir, state, lock);
|
|
152
|
+
removeWorktree(contract.cwd, acceptedPath);
|
|
153
|
+
},
|
|
154
|
+
onVerificationFailure: async (transaction) => {
|
|
155
|
+
const verdict = verificationFailureWithScope(verificationFailureVerdict(state), state.scope);
|
|
156
|
+
verdict.summary = "integrated candidate verification failed";
|
|
157
|
+
const divergent = candidateOnlyFailures(state.verification, transaction.candidateEvidence);
|
|
158
|
+
verdict.findings = [...(verdict.findings ?? []), {
|
|
159
|
+
severity: "critical",
|
|
160
|
+
description: divergent.length
|
|
161
|
+
? `the integration worktree failed a verification the attempt passed (${boundedUtf8(divergent.join("; "), 512)}): the two worktrees disagree about the environment, not about the work`
|
|
162
|
+
: "the sealed candidate did not pass the node verification in its integration worktree",
|
|
163
|
+
evidence: boundedUtf8(JSON.stringify(transaction.candidateEvidence ?? {}), 4 * 1024),
|
|
164
|
+
}];
|
|
165
|
+
applyRejection(contract, node, state, runDir, null, lock, states, campaignPath, verdict, {
|
|
166
|
+
code: "verification_failed",
|
|
167
|
+
label: "candidate-verification",
|
|
168
|
+
});
|
|
169
|
+
},
|
|
170
|
+
onConflict: async (transaction) => {
|
|
171
|
+
const paths = transaction.conflictingPaths?.length ? transaction.conflictingPaths.join(", ") : "unknown paths";
|
|
172
|
+
transition(runDir, state, "blocked", {
|
|
173
|
+
phase: "complete",
|
|
174
|
+
error: { code: "integration_conflict", message: `integration conflict in: ${paths}` },
|
|
175
|
+
}, lock);
|
|
176
|
+
if (campaignPath) await raiseNodeAttention(campaignPath, runDir, state, "integration_conflict");
|
|
177
|
+
},
|
|
178
|
+
onConcurrentMove: async (transaction) => {
|
|
179
|
+
transition(runDir, state, "blocked", {
|
|
180
|
+
phase: "complete",
|
|
181
|
+
error: { code: "integration_concurrent_move", message: `run ref moved from ${transaction.previousRunRefTip} to ${transaction.currentRunRefTip ?? "unknown"}` },
|
|
182
|
+
}, lock);
|
|
183
|
+
if (campaignPath) await raiseNodeAttention(campaignPath, runDir, state, "integration_concurrent_move");
|
|
184
|
+
},
|
|
185
|
+
});
|
|
186
|
+
return result;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Surface a node attention state through the run's notify queue.
|
|
190
|
+
* @param {string} campaignPath
|
|
191
|
+
* @param {string} runDir
|
|
192
|
+
* @param {NodeSnapshot} state
|
|
193
|
+
* @param {string} code
|
|
194
|
+
*/
|
|
195
|
+
export async function raiseNodeAttention(campaignPath, runDir, state, code) {
|
|
196
|
+
const runId = basename(runDir);
|
|
197
|
+
const dedupeKey = `attention:${runId}:${state.id}:${code}`;
|
|
198
|
+
if (alreadyNotified(runDir, dedupeKey)) return;
|
|
199
|
+
await notifyQueueFor(runDir).enqueue({
|
|
200
|
+
type: "attention",
|
|
201
|
+
campaignId: campaignIdOf(campaignPath),
|
|
202
|
+
runId,
|
|
203
|
+
nodeId: state.id,
|
|
204
|
+
errorCode: code,
|
|
205
|
+
dedupeKey,
|
|
206
|
+
});
|
|
207
|
+
}
|