faberun 0.3.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/README.md +152 -100
  2. package/package.json +10 -2
  3. package/skills/faberun/SKILL.md +6 -5
  4. package/skills/faberun/references/contract.md +23 -11
  5. package/skills/faberun/references/engineering.md +3 -1
  6. package/skills/faberun/references/operations.md +19 -12
  7. package/skills/faberun/references/rules.md +3 -1
  8. package/src/campaign/chain.mjs +6 -2
  9. package/src/campaign/index.mjs +17 -1
  10. package/src/campaign/metrics.mjs +3 -3
  11. package/src/cli/brand.mjs +2 -1
  12. package/src/cli/campaign.mjs +2 -0
  13. package/src/cli/contract.mjs +2 -0
  14. package/src/cli/manual.mjs +341 -0
  15. package/src/cli/seat.mjs +2 -0
  16. package/src/cli/setup.mjs +109 -30
  17. package/src/cli/skills.mjs +310 -8
  18. package/src/cli.mjs +3 -2
  19. package/src/contract/final-verification.mjs +31 -2
  20. package/src/contract/index.mjs +28 -25
  21. package/src/contract/runtime.mjs +5 -1
  22. package/src/contract/snapshot.mjs +7 -1
  23. package/src/contract/task-packet.mjs +20 -9
  24. package/src/contract/verification.mjs +1 -1
  25. package/src/engine/backoff.mjs +1 -1
  26. package/src/engine/dispatch.mjs +31 -4
  27. package/src/engine/gate.mjs +12 -0
  28. package/src/engine/process-identity.mjs +39 -0
  29. package/src/engine/prompts.mjs +18 -0
  30. package/src/engine/resume.mjs +2 -2
  31. package/src/engine/review.mjs +9 -1
  32. package/src/engine/run-command.mjs +23 -2
  33. package/src/engine/run-identity.mjs +14 -0
  34. package/src/engine/scheduler.mjs +45 -12
  35. package/src/engine/settle.mjs +29 -0
  36. package/src/engine/supervise.mjs +32 -6
  37. package/src/engine/verify.mjs +98 -9
  38. package/src/harnesses/agy/index.mjs +3 -0
  39. package/src/harnesses/claude/index.mjs +5 -0
  40. package/src/harnesses/codex/index.mjs +3 -0
  41. package/src/harnesses/dsh/index.mjs +26 -0
  42. package/src/harnesses/exec-jsonl/index.mjs +2 -0
  43. package/src/harnesses/index.mjs +10 -3
  44. package/src/harnesses/replay/index.mjs +2 -0
  45. package/src/harnesses/zcode/index.mjs +3 -0
  46. package/src/host/preflight.mjs +5 -1
  47. package/src/notify/index.mjs +45 -2
  48. package/src/repo/source-identity.mjs +4 -3
  49. package/src/report/final.mjs +3 -2
  50. package/src/report/render.mjs +134 -51
  51. package/src/web/index.html +1 -1
@@ -19,11 +19,18 @@ const FIELDS = new Set([
19
19
  "verification",
20
20
  ]);
21
21
  const PROMPT_MAX_BYTES = 64 * 1024;
22
+ /**
23
+ * The `## Verification` preamble every worker prompt carries. The controller's
24
+ * recorded run is the proof; a self-run is optional and must be a quick,
25
+ * process-free command, so a sandbox that cannot signal processes never hangs
26
+ * on the node's own verification.
27
+ */
28
+ const VERIFICATION_PARAGRAPH = "The controller runs every command below after you report; its recorded results are the proof of this node. Running a command yourself is optional and only for one that finishes in seconds and spawns no long-lived process. Keep output bounded (pipe through `| tail -n 200`). Never wait on a background job, never run the whole test suite, and never run tests that start and terminate other processes.";
22
29
 
23
30
  /**
24
- * `validateRelativePath`'s answer when a read path is absent and the caller
25
- * asked to defer the missing-read verdict rather than throw it. Only contract
26
- * loading opts in; every other caller throws the missing-read error in place.
31
+ * `validateRelativePath`'s answer when a path is absent and the caller asked to
32
+ * defer the missing-path verdict rather than throw it. Only contract loading
33
+ * opts in; every other caller throws the missing-path error in place.
27
34
  */
28
35
  const DEFERRED_MISSING = Symbol("deferred-missing");
29
36
 
@@ -47,7 +54,7 @@ const DEFERRED_MISSING = Symbol("deferred-missing");
47
54
  * @param {string} contractDir
48
55
  * @param {string} cwd
49
56
  * @param {number} index
50
- * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string}[], persisted?: boolean}} [options]
57
+ * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string, kind: "read"|"acknowledged"}[], persisted?: boolean}} [options]
51
58
  * @returns {TaskPacket}
52
59
  */
53
60
  export function loadTaskPacket(node, contractDir, cwd, index, options = {}) {
@@ -117,7 +124,7 @@ export function renderWorkerPrompt(packet, nodeId) {
117
124
  ...bulletOrNone(packet.nonGoals),
118
125
  "",
119
126
  "## Verification",
120
- "Run each command yourself before reporting done. Keep command output bounded: pipe long output through `| tail -n 200` (or similar) and never paste full test or fuzz logs into your context or results.",
127
+ VERIFICATION_PARAGRAPH,
121
128
  ...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
122
129
  "",
123
130
  "## Required output",
@@ -134,7 +141,7 @@ export function renderWorkerPrompt(packet, nodeId) {
134
141
  * @param {unknown} packet
135
142
  * @param {number} index
136
143
  * @param {string} cwd
137
- * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string}[], persisted?: boolean}} [options]
144
+ * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string, kind: "read"|"acknowledged"}[], persisted?: boolean}} [options]
138
145
  * @returns {TaskPacket}
139
146
  */
140
147
  export function validateTaskPacket(packet, index, cwd, options = {}) {
@@ -207,18 +214,20 @@ export function validateTaskPacket(packet, index, cwd, options = {}) {
207
214
 
208
215
  if (!persisted) {
209
216
  const deferMissingReads = options.deferMissingReads === true;
210
- /** @type {{path: string, label: string}[]} */
217
+ /** @type {{path: string, label: string, kind: "read"|"acknowledged"}[]} */
211
218
  const deferredReads = options.deferredReads ?? [];
212
219
  normalizedReadFiles.forEach((path, pathIndex) => {
213
220
  const label = `nodes[${index}].taskPacket.readFiles[${pathIndex}]`;
214
221
  const result = validateRelativePath(path, label, cwd, true, { deferMissing: deferMissingReads });
215
- if (result === DEFERRED_MISSING) deferredReads.push({ path, label });
222
+ if (result === DEFERRED_MISSING) deferredReads.push({ path, label, kind: "read" });
216
223
  });
217
224
  if (writeFiles !== undefined) /** @type {string[]} */ (writeFiles).forEach((path, pathIndex) => {
218
225
  validateRelativePath(path, `nodes[${index}].taskPacket.writeFiles[${pathIndex}]`, cwd, false);
219
226
  });
220
227
  /** @type {string[]} */ (scopeAcknowledged).forEach((path, pathIndex) => {
221
- validateRelativePath(path, `nodes[${index}].taskPacket.scopeAcknowledged[${pathIndex}]`, cwd, true);
228
+ const label = `nodes[${index}].taskPacket.scopeAcknowledged[${pathIndex}]`;
229
+ const result = validateRelativePath(path, label, cwd, true, { deferMissing: deferMissingReads });
230
+ if (result === DEFERRED_MISSING) deferredReads.push({ path, label, kind: "acknowledged" });
222
231
  });
223
232
  for (const [commandIndex, command] of verification.entries()) {
224
233
  if (command.cwd !== undefined) {
@@ -426,6 +435,7 @@ function renderDiscoveryPrompt(packet, nodeId) {
426
435
  'Return exactly one worker-result JSON object, with no markdown or prose. Set status to "done", missingContext to [], and artifacts to an array containing exactly one JSON-stringified execution task packet with every required taskPacket field. The packet readFiles and writeFiles must be non-empty and scoped to this repository.',
427
436
  "",
428
437
  "## Verification",
438
+ VERIFICATION_PARAGRAPH,
429
439
  ...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
430
440
  ];
431
441
  const prompt = `${lines.join("\n")}\n`;
@@ -467,6 +477,7 @@ function renderAutonomousPrompt(packet, nodeId) {
467
477
  ...bulletOrNone(packet.nonGoals),
468
478
  "",
469
479
  "## Verification",
480
+ VERIFICATION_PARAGRAPH,
470
481
  ...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
471
482
  "",
472
483
  "## Required output",
@@ -38,7 +38,7 @@ export const VERIFICATION_LIMITS = Object.freeze({
38
38
  /**
39
39
  * Bounded evidence captured for one attempt.
40
40
  *
41
- * @typedef {{passed: boolean, stdout: string, stderr: string, error: string|null, exitCode: number|null, signal: string|null, timedOut: boolean, durationMs: number|null}} VerificationAttemptResult
41
+ * @typedef {{passed: boolean, stdout: string, stderr: string, error: string|null, exitCode: number|null, signal: string|null, timedOut: boolean, durationMs: number|null, signalDeath?: boolean}} VerificationAttemptResult
42
42
  */
43
43
 
44
44
  /**
@@ -9,7 +9,7 @@
9
9
  * third time buys nothing: change provider, or stop and say so.
10
10
  *
11
11
  * Everything here is a pure decision. Persisting it — the routing history,
12
- * the override, the node transition — stays in runner.mjs, so this module can
12
+ * the override, the node transition — stays in cli.mjs, so this module can
13
13
  * be tested without a run directory, a lease, or a provider.
14
14
  */
15
15
  import { nextHop, nextSynthesizedRuntime, synthesizedChain } from "./failover.mjs";
@@ -9,7 +9,7 @@
9
9
  * which made dispatch depend on review policy and on settlement, and that is
10
10
  * the shape that kept `engine/` a web instead of a stack.
11
11
  */
12
- import { JUDGE_SCHEMA, judgePrompt } from "./prompts.mjs";
12
+ import { JUDGE_SCHEMA, appendSandboxNotice, judgePrompt } from "./prompts.mjs";
13
13
  import { LockLostError } from "../run/lock.mjs";
14
14
  import { TOOL_OUTPUT_LIMIT_BYTES } from "../harnesses/exec-jsonl/index.mjs";
15
15
  import { appendPreviousAttempt } from "./retry.mjs";
@@ -32,7 +32,7 @@ import { emptyScope, persistedScopeBoundary, workerScope } from "./scope.mjs";
32
32
  import { hasOperationIntent, hasOperationSettlement, operationNeedsRecovery, operationNextState, persistInvocationIntent, providerReceipts, settleInvocation } from "../run/operations.mjs";
33
33
  import { invocationCost, invocationUsage } from "../run/usage.mjs";
34
34
  import { logPaths, readBoundedTail, startProcess } from "./process.mjs";
35
- import { mkdirSync } from "node:fs";
35
+ import { mkdirSync, statSync } from "node:fs";
36
36
  import { READ_LINE_LIMIT, normalizeProviderResult, providerCommand } from "../harnesses/index.mjs";
37
37
  import { readJson, writeJsonAtomic } from "../run/store.mjs";
38
38
  import { judgeReaskInstruction, reviewMode } from "../contract/review-modes.mjs";
@@ -267,6 +267,30 @@ function phaseHandoffPrompt(contract, node, state, runDir, role) {
267
267
  ].join("\n\n");
268
268
  return boundedUtf8(handoff, 60 * 1024);
269
269
  }
270
+ /**
271
+ * The declared weight of a node's readFiles at dispatch time: the sum of the
272
+ * byte sizes of the files that exist in the attempt workspace. This is the
273
+ * one quantity the controller can measure about a packet's reference load --
274
+ * the worker prompt lists readFiles and the worker reads them itself, so what
275
+ * it actually reads is the harness's business. A missing file counts 0 rather
276
+ * than throwing: a declared path can be produced by a dependency that has not
277
+ * run yet or removed by the tree since the packet was authored.
278
+ *
279
+ * @param {string[]} readFiles
280
+ * @param {string} workspace
281
+ * @returns {number}
282
+ */
283
+ export function declaredReadBytes(readFiles, workspace) {
284
+ let total = 0;
285
+ for (const path of readFiles) {
286
+ try {
287
+ total += statSync(join(workspace, path)).size;
288
+ } catch {
289
+ // Missing or unreadable file: contributes no weight.
290
+ }
291
+ }
292
+ return total;
293
+ }
270
294
  /**
271
295
  * The mechanical worker tool policy for the provider boundary: hook settings
272
296
  * on Claude-compatible commands. Only an adapter whose surface can prove
@@ -404,6 +428,8 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
404
428
  // so it is appended to the resolved prompt rather than the candidate handed
405
429
  // to phaseInvocationPlan.
406
430
  phasePlan.prompt = appendPreviousAttempt(phasePlan.prompt, state.previousAttempt);
431
+ // The resolved worker's sandbox is a dispatch-time fact, not a packet one.
432
+ phasePlan.prompt = appendSandboxNotice(phasePlan.prompt, runtime);
407
433
  // The worker prompt directs the provider to write the canonical result file;
408
434
  // make sure the directory exists before the provider is asked to.
409
435
  const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
@@ -429,6 +455,7 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
429
455
  writeJsonAtomic(snapshotPath, baseline);
430
456
  state.phase = "worker";
431
457
  state.runtime = runtime;
458
+ state.declaredReadBytes = declaredReadBytes(node.taskPacket.readFiles ?? [], workspace);
432
459
  // A new worker attempt has no accepted result yet. The canonical result
433
460
  // file is cleared when the previous attempt was explicitly rejected (failed
434
461
  // gate verdict), when no valid canonical file exists, or when the stale file
@@ -526,11 +553,11 @@ export function startResultMaterialization(contract, node, state, runDir, runnin
526
553
  const paths = logPaths(runDir, node.id, "worker", state.attempt);
527
554
  const workspace = attemptWorkspace(state) ?? contract.cwd;
528
555
  const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
529
- const prompt = [
556
+ const prompt = appendSandboxNotice([
530
557
  `${RESULT_MATERIALIZATION_PROMPT_HEADER} Do not inspect, implement, verify, or invoke tools.`,
531
558
  `Your only job in this single bounded turn is to write the required worker-result JSON object to: ${resultPath}`,
532
559
  "Then return that same JSON object as the final message.",
533
- ].join("\n\n");
560
+ ].join("\n\n"), materializationRuntime);
534
561
  let baseline;
535
562
  try {
536
563
  baseline = captureWorkspaceSnapshot(workspace);
@@ -21,8 +21,14 @@
21
21
  *
22
22
  * Nothing here is exported for a caller: this file is a program. Every name
23
23
  * below is module-local.
24
+ *
25
+ * It also exits when the release file's directory is gone (measured
26
+ * 2026-09-16: gate processes from a prior day's test runs, spawned into a
27
+ * temp directory the failed test never cleaned up, were still alive and
28
+ * waiting for a release file that could now never appear).
24
29
  */
25
30
  import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
31
+ import { dirname } from "node:path";
26
32
  import { spawn } from "node:child_process";
27
33
 
28
34
  /** @typedef {{executable: string, args: string[], cwd: string, promptTransport: "stdin"|"argv", harness: string, env: Record<string, string|null>|null, stdoutPath: string, stderrPath: string}} GateConfig */
@@ -158,8 +164,14 @@ function childEnv() {
158
164
  process.on("SIGTERM", () => stopProvider());
159
165
  process.on("SIGINT", () => stopProvider());
160
166
 
167
+ /** @returns {boolean} */
168
+ function releaseDirectoryGone() {
169
+ return !existsSync(dirname(releasePath));
170
+ }
171
+
161
172
  const timer = setInterval(() => {
162
173
  if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
174
+ if (releaseDirectoryGone()) { clearInterval(timer); stopProvider(); return; }
163
175
  if (!existsSync(releasePath)) return;
164
176
  clearInterval(timer);
165
177
  const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
@@ -13,6 +13,41 @@ import { processStartToken } from "../run/lock.mjs";
13
13
  /** @typedef {import("node:child_process").ChildProcess} ChildProcess */
14
14
  /** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
15
15
 
16
+ /**
17
+ * Whether a controller may signal a pid's process group at all. It may not when
18
+ * the pid is the calling process -- whose group is the controller's own -- or
19
+ * its parent, whose group is the runner (or host scheduler) that spawned the
20
+ * controller: either signal takes down the caller instead of the invocation.
21
+ * The refusal is recorded and returned, never thrown.
22
+ *
23
+ * The line goes to the injected sink when a caller supplied one, otherwise to
24
+ * stderr; this function owns no run directory and no event-log schema, so
25
+ * stderr is the one recorder it always has. Recording is diagnostic, so a
26
+ * failure to record must never turn the refusal into a throw.
27
+ *
28
+ * @param {number} pid
29
+ * @param {((event: Record<string, unknown>) => void)|undefined} [append] injected event sink
30
+ * @returns {boolean} true when the pid is this process or its parent and the signal was refused
31
+ */
32
+ export function refuseSelfSignal(pid, append) {
33
+ const relation = pid === process.pid ? "self" : pid === process.ppid ? "parent" : null;
34
+ if (relation === null) return false;
35
+ const event = {
36
+ type: "controller_self_signal_refused",
37
+ at: new Date().toISOString(),
38
+ pid,
39
+ processPid: process.pid,
40
+ relation,
41
+ };
42
+ try {
43
+ if (append) append(event);
44
+ else process.stderr.write(`[warn] controller_self_signal_refused ${JSON.stringify(event)}\n`);
45
+ } catch {
46
+ // The refusal is diagnostic: a failed recorder must never become a throw.
47
+ }
48
+ return true;
49
+ }
50
+
16
51
  /**
17
52
  * A process group answers a signal-0 probe. EPERM means the group exists but is
18
53
  * not this user's, which is no more "ours" than a missing group.
@@ -57,6 +92,10 @@ export function processStartTokenMatches(invocation) {
57
92
  */
58
93
  export function invocationOwned(invocation, options = {}) {
59
94
  if (!invocation?.pid || !Number.isInteger(invocation.pid)) return false;
95
+ // A controller does not spawn itself or its parent: a recorded pid that names
96
+ // either can never be an invocation this controller owns, whatever token or
97
+ // child handle accompanies it, and signalling it would hit the caller.
98
+ if (invocation.pid === process.pid || invocation.pid === process.ppid) return false;
60
99
  try {
61
100
  process.kill(invocation.pid, 0);
62
101
  } catch {
@@ -1,6 +1,7 @@
1
1
  import { validateWorkerResult } from "../contract/worker-result.mjs";
2
2
  import { scopeFindingsPromptSection } from "../contract/scope-findings.mjs";
3
3
  import { JUDGE_ENVELOPE_REASON, JUDGE_FINDING_ENVELOPE_REASON, JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
4
+ import { harnessCapabilities } from "../harnesses/index.mjs";
4
5
 
5
6
  /** @typedef {{id: string, definitionOfDone: import("../contract/definition-of-done.mjs").DefinitionOfDoneItem[], taskPacket: {mode?: "execution"|"discovery"|"autonomous", objective: string, instructions: string[], writeFiles?: string[], writeRoots?: string[], verification: {argv: string[]}[]}}} JudgeNode */
6
7
  /** @typedef {{verdict: "pass"|"fail"|"invalid_judge_output", maxSeverity: "none"|"minor"|"major"|"critical", summary: string, findings: {severity: "minor"|"major"|"critical", description: string, evidence: string}[]}} JudgeVerdict */
@@ -276,6 +277,23 @@ export function retryPrompt(node, verdict) {
276
277
  : prompt;
277
278
  }
278
279
 
280
+ /**
281
+ * Append the `## Sandbox` warning a resolved worker runtime earns. Only a
282
+ * harness whose adapter declares `signalsProcesses === false` gets it: that
283
+ * sandbox cannot signal child processes or read the process table, so a test
284
+ * that starts and terminates a child hangs until the executor's cap. An
285
+ * unmeasured harness (`null`) is left alone — no measurement justifies the
286
+ * warning.
287
+ *
288
+ * @param {string} prompt
289
+ * @param {{harness: string}} runtime
290
+ * @returns {string}
291
+ */
292
+ export function appendSandboxNotice(prompt, runtime) {
293
+ if (harnessCapabilities(runtime).signalsProcesses !== false) return prompt;
294
+ return `${prompt}\n\n## Sandbox\nYour harness runs you in a sandbox that cannot signal other processes or read the process table. A test that starts and terminates a child process hangs here until the executor's cap and is then killed as a stall. Do not run such tests; the controller runs them after you report.`;
295
+ }
296
+
279
297
  /**
280
298
  * @param {unknown} value
281
299
  * @param {number} maxBytes
@@ -17,7 +17,7 @@ import { Buffer } from "node:buffer";
17
17
  import { acquire as acquireLock } from "../run/lock.mjs";
18
18
  import { applyInvalidWorkerResult, assertRunMutable, handleProviderExhaustion } from "./lifecycle.mjs";
19
19
  import { applyJudgeResult } from "./review.mjs";
20
- import { assertSourceUnchanged, captureRunIdentity, statesFingerprint } from "./run-identity.mjs";
20
+ import { assertSourceUnchanged, captureRunIdentity, recordedBaseRef, statesFingerprint } from "./run-identity.mjs";
21
21
  import { attemptWorkspace, attemptWorktreePath, gitHead, removeWorktree, runRefName } from "../repo/worktree.mjs";
22
22
  import { canonicalWorkerResultText, isResultMaterializationInvocation, materializeAttemptResult, recoverWorkerResult } from "./result-file.mjs";
23
23
  import { checkPersistedWorkerScope, persistedScopeBoundary, reconcileAmbiguousWorkerRestart, resolveUnknownEffect } from "./scope.mjs";
@@ -114,7 +114,7 @@ export async function resumeRun(runDirPath, options = {}) {
114
114
  node.id,
115
115
  persistedScopeBoundary(contract, node, states.get(node.id), attemptWorkspace(states.get(node.id)) ?? contract.cwd),
116
116
  ]));
117
- const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries);
117
+ const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries, recordedBaseRef(storedMetadata) ?? undefined);
118
118
  const identity = assertSourceUnchanged(storedMetadata.sourceIdentity, sourceIdentity);
119
119
  // A resume is an explicit instruction to continue the run: it consumes a
120
120
  // stale cancel request instead of letting it re-cancel the retried nodes.
@@ -21,7 +21,7 @@ import {
21
21
  } from "../contract/review-modes.mjs";
22
22
 
23
23
  import { errorMessage, excerpt } from "../util.mjs";
24
- import { appendTransitionEvent, transition } from "./state.mjs";
24
+ import { appendTransitionEvent, transition, writeNode } from "./state.mjs";
25
25
  import { startJudge } from "./dispatch.mjs";
26
26
  import { applyRejection, raiseNodeAttention, settleDone } from "./settle.mjs";
27
27
 
@@ -154,6 +154,14 @@ export async function applyJudgeResult(contract, node, state, result, runDir, lo
154
154
  await raiseNodeAttention(campaignPath, runDir, state, "judge_protocol");
155
155
  return;
156
156
  }
157
+ // The verdict and the spent bound are durable before the blocked
158
+ // transition: a controller loss in this gap reads the settled re-ask from
159
+ // the node snapshot instead of spending a second one. No await separates
160
+ // the two writes, so the durable record cannot be observed only by luck.
161
+ writeNode(runDir, state, lock);
162
+ if (process.env.FABERUN_JUDGE_REASK_INTERRUPT === "after-verdict") {
163
+ throw new Error("judge re-ask interrupted after verdict persistence");
164
+ }
157
165
  transition(runDir, state, "blocked", {
158
166
  phase: "judge",
159
167
  gate: verdict,
@@ -99,9 +99,30 @@ async function runRepeatedCommand(command, baseCwd, commandIndex, options) {
99
99
  const attempts = [];
100
100
  const repeat = command.repeat ?? 1;
101
101
  for (let attempt = 1; attempt <= repeat; attempt += 1) {
102
- attempts.push(await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex));
102
+ const result = await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex);
103
+ if (signalDeathRetry(result, options.signal)) {
104
+ attempts.push({ ...result, signalDeath: true });
105
+ attempts.push(await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex));
106
+ } else {
107
+ attempts.push(result);
108
+ }
103
109
  }
104
- return { ...command, cwd: resolveVerificationCwd(baseCwd, command.cwd ?? "."), passed: attempts.every((item) => item.passed), attempts };
110
+ const passed = attempts.filter((item) => !item.signalDeath).every((item) => item.passed);
111
+ return { ...command, cwd: resolveVerificationCwd(baseCwd, command.cwd ?? "."), passed, attempts };
112
+ }
113
+ /**
114
+ * Whether an attempt died from a signal the controller itself did not send.
115
+ * `terminateGroup` only fires from the timeout and abort paths below, so a
116
+ * `signal` with neither set is evidence of an external kill (OOM, an operator
117
+ * `kill`, a flaky sandbox) rather than a verdict on the command under test,
118
+ * and gets one retry instead of failing the node outright.
119
+ *
120
+ * @param {VerificationAttemptResult} result
121
+ * @param {AbortSignal|undefined} signal
122
+ * @returns {boolean}
123
+ */
124
+ function signalDeathRetry(result, signal) {
125
+ return Boolean(result.signal) && !result.timedOut && !signal?.aborted;
105
126
  }
106
127
  /**
107
128
  * The mutation case: the entry passes on the mutant-kill fraction, and every
@@ -181,6 +181,20 @@ export function setLaunchBaseRef(baseRef) {
181
181
  pendingLaunchBaseRef = baseRef ?? null;
182
182
  }
183
183
 
184
+ /**
185
+ * The base ref a run was launched against, when it was launched with
186
+ * `--base-ref`. A run recorded before this field existed, or launched
187
+ * without the flag, has none — a resume of that run keeps comparing against
188
+ * the checkout's own HEAD.
189
+ *
190
+ * @param {RunMetadata|undefined} metadata
191
+ * @returns {string|null}
192
+ */
193
+ export function recordedBaseRef(metadata) {
194
+ const baseRef = metadata?.sourceIdentity?.baseRef;
195
+ return typeof baseRef === "string" && baseRef.length > 0 ? baseRef : null;
196
+ }
197
+
184
198
  /**
185
199
  * Capture the run's source identity, including one version-only probe per
186
200
  * distinct routed runtime (a local binary call, no model tokens) so a later
@@ -41,7 +41,7 @@ import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsa
41
41
  import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
42
42
  import { validateContract } from "../contract/index.mjs";
43
43
  import { validateNodeSnapshot } from "../contract/snapshot.mjs";
44
- import { finalVerificationCommands } from "../contract/final-verification.mjs";
44
+ import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
45
45
  import { startJudge, startWorker } from "./dispatch.mjs";
46
46
  import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
47
47
  import { blockDependents, runtimeAssignments } from "./assignment.mjs";
@@ -94,12 +94,13 @@ export function verificationBudgetMs(commands) {
94
94
  /**
95
95
  * The budget a node is judged against, in milliseconds. It is the sum of every
96
96
  * bounded phase the node can legitimately occupy without a state transition:
97
- * its worker invocation (`timeoutSec`), its packet verification and the
98
- * contract's `finalVerification` when it is phase-terminal, an integration
99
- * candidate run of that same set, and the bounded command proofs of its gate.
100
- * A frozen node is one that has been silent longer than this, not merely
101
- * longer than the worker timeout, because a legitimate verification can be
102
- * minutes long and must not be mistaken for a freeze.
97
+ * its worker invocation (`timeoutSec`), its packet verification plus the
98
+ * contract's `sharedVerification` and, when it is phase-terminal, the
99
+ * contract's `finalVerification`, an integration candidate run of that same
100
+ * set, and the bounded command proofs of its gate. A frozen node is one that
101
+ * has been silent longer than this, not merely longer than the worker timeout,
102
+ * because a legitimate verification can be minutes long and must not be
103
+ * mistaken for a freeze.
103
104
  *
104
105
  * @param {ValidatedContract} contract
105
106
  * @param {ValidatedNode} node
@@ -107,14 +108,17 @@ export function verificationBudgetMs(commands) {
107
108
  */
108
109
  export function nodeBudgetBasisMs(contract, node) {
109
110
  const defaultTimeoutMs = (node.timeoutSec ?? contract.timeoutSec ?? 60) * 1_000;
111
+ const sharedMs = verificationBudgetMs(sharedVerificationCommands(contract));
110
112
  const packetMs = verificationBudgetMs(node.taskPacket?.verification);
111
113
  const finalMs = verificationBudgetMs(finalVerificationCommands(contract, node));
112
- // The controller runs the packet set once after the worker and once against
113
- // the integration candidate, and the finalVerification set with each.
114
- const candidateMs = packetMs + finalMs;
114
+ // The controller runs the packet set (with the contract-level shared set) once
115
+ // after the worker and once against the integration candidate, and the
116
+ // finalVerification set with each.
117
+ const attemptMs = packetMs + sharedMs;
118
+ const candidateMs = attemptMs + finalMs;
115
119
  const gateTimeoutMs = Math.max(1_000, Math.min(defaultTimeoutMs, 120_000));
116
120
  const commandProofs = (node.definitionOfDone ?? []).filter((item) => item.proof?.kind === "command").length;
117
- return defaultTimeoutMs + packetMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
121
+ return defaultTimeoutMs + attemptMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
118
122
  }
119
123
 
120
124
  /**
@@ -149,6 +153,26 @@ export function enforceRunningInvariant(runDir, states, running, lock) {
149
153
  return parked;
150
154
  }
151
155
 
156
+ /**
157
+ * The fingerprint that decides whether a render is due: the shared
158
+ * `statesFingerprint` (status, phase, attempt, revisions) plus each running
159
+ * node's verification progress. A verification command completing and the
160
+ * next one starting changes neither status nor phase, so the shared
161
+ * fingerprint alone would never notice it; this is the one used to gate
162
+ * `renderStatusIfChanged` and nothing else, so a resume's drift comparison
163
+ * still uses the unextended `statesFingerprint`.
164
+ *
165
+ * @param {Map<string, NodeSnapshot>} states
166
+ * @returns {string}
167
+ */
168
+ function renderFingerprint(states) {
169
+ const progress = [...states.values()].map((state) => {
170
+ const verification = /** @type {{progress?: {index: number, total: number, argv: string}}|null|undefined} */ (state.verification);
171
+ return verification?.progress ? `${state.id}:${verification.progress.index}/${verification.progress.total}:${verification.progress.argv}` : "";
172
+ }).join("|");
173
+ return `${statesFingerprint(states)}#${progress}`;
174
+ }
175
+
152
176
  /**
153
177
  * @param {string} contractPath
154
178
  * @param {{detachedBootstrap?: boolean}} [options] `detachedBootstrap` is set
@@ -290,11 +314,19 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
290
314
  let statusFingerprint = null;
291
315
  /** @param {boolean} force @param {LockHandle|null} [renderLock] */
292
316
  const renderStatusIfChanged = (force = false, renderLock = lock) => {
293
- const fingerprint = statesFingerprint(states);
317
+ const fingerprint = renderFingerprint(states);
294
318
  if (!force && fingerprint === statusFingerprint) return;
295
319
  statusFingerprint = fingerprint;
296
320
  render(runDir, runsDir, contract, states, renderLock);
297
321
  };
322
+ // The tick that owns a running node's verification can be minutes long
323
+ // (`executeControllerVerification` is awaited on the critical path below),
324
+ // and that whole time status.json would otherwise report whatever the last
325
+ // tick left it at. A timer renders between ticks too; it is cheap even when
326
+ // idle because `renderFingerprint` still change-detects, so a quiet run
327
+ // writes nothing extra.
328
+ const statusTimer = setInterval(() => renderStatusIfChanged(), contract.pollIntervalMs);
329
+ statusTimer.unref();
298
330
  let handoffFingerprint = statesFingerprint(states);
299
331
  const renderHandoffIfChanged = () => {
300
332
  const fingerprint = statesFingerprint(states);
@@ -475,6 +507,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
475
507
  notifyQueuesByRun.delete(runDir);
476
508
  return { runDir, states, ok: false, error };
477
509
  } finally {
510
+ clearInterval(statusTimer);
478
511
  process.removeListener("SIGINT", cancel);
479
512
  process.removeListener("SIGTERM", cancel);
480
513
  process.removeListener("SIGHUP", cancel);
@@ -86,6 +86,33 @@ export function applyVerificationFailure(contract, node, state, runDir, running,
86
86
  applyRejection(contract, node, state, runDir, running, lock, states, campaignPath, verdict, { code: "verification_failed", label: "verification" });
87
87
  }
88
88
 
89
+ /**
90
+ * The bounded operator-facing note when the integration candidate needed
91
+ * `verifyCandidateWorkspace`'s one retry to agree with the attempt. The node
92
+ * snapshot schema has no free-text `note` field of its own; `gate.summary` is
93
+ * the one it already carries that `statusNote` (report/render.mjs) surfaces,
94
+ * so a retried acceptance folds its note there instead of failing on an
95
+ * unknown field the next time the state is written.
96
+ *
97
+ * @param {unknown} candidateEvidence
98
+ * @returns {string|null}
99
+ */
100
+ function candidateRetryNote(candidateEvidence) {
101
+ const record = /** @type {{retried?: unknown, commands?: Array<{argv?: string[]}>}} */ (candidateEvidence ?? {});
102
+ const indexes = Array.isArray(record.retried) ? record.retried : [];
103
+ if (!indexes.length) return null;
104
+ const argvList = indexes.map((index) => (record.commands?.[/** @type {number} */ (index)]?.argv ?? []).join(" ")).join(", ");
105
+ return boundedUtf8(`candidate verification retried: ${argvList}`, 256);
106
+ }
107
+ /**
108
+ * @param {import("../contract/index.mjs").GateResult|null|undefined} gate
109
+ * @param {string} note
110
+ * @returns {import("../contract/index.mjs").GateResult}
111
+ */
112
+ function withCandidateRetryNote(gate, note) {
113
+ if (!gate) return { verdict: "pass", maxSeverity: "none", summary: note, findings: [] };
114
+ return { ...gate, summary: boundedUtf8(`${gate.summary} · ${note}`, 4 * 1024) };
115
+ }
89
116
  /**
90
117
  * Seal the current attempt, verify its candidate in a detached worktree, and
91
118
  * only then perform the single done-state transition. The integration module
@@ -142,8 +169,10 @@ export async function settleDone(contract, node, state, runDir, lock, states, ca
142
169
  onAccepted: async (transaction) => {
143
170
  const acceptedPath = state.worktree?.path ?? attemptWorktreePath(runDir, contract.id, node.id, transaction.attempt);
144
171
  if (state.attempt === transaction.attempt && state.status !== "done") {
172
+ const retryNote = candidateRetryNote(/** @type {{verificationEvidence?: {candidate?: unknown}}} */ (transaction).verificationEvidence?.candidate);
145
173
  transition(runDir, state, "done", {
146
174
  ...patch,
175
+ ...(retryNote ? { gate: withCandidateRetryNote(/** @type {import("../contract/index.mjs").GateResult|null|undefined} */ (patch.gate ?? state.gate), retryNote) } : {}),
147
176
  integratedHead: transaction.candidateSha,
148
177
  worktree: { ...(state.worktree ?? {}), status: "removed", commit: transaction.attemptSha, baseSha: transaction.previousRunRefTip },
149
178
  }, lock);
@@ -34,6 +34,7 @@
34
34
  import { existsSync, readFileSync } from "node:fs";
35
35
  import { join } from "node:path";
36
36
  import { SETTLED, SUCCESS } from "./prompts.mjs";
37
+ import { invocationOwned, refuseSelfSignal } from "./process-identity.mjs";
37
38
  import { earliestTierReset } from "./retry.mjs";
38
39
  import { lockStale, pidAlive, readLock } from "../run/lock.mjs";
39
40
  import { listNodeSnapshots, readNodeSnapshot } from "../run/node-store.mjs";
@@ -495,30 +496,45 @@ export async function waitForGroupGone(pid, alive, graceMs, sleep, now) {
495
496
  * the previous holder is gone. The kill and liveness probes are injectable so
496
497
  * a test can prove the ordering without signalling a real process.
497
498
  *
499
+ * A lock is a claim until it is proven, exactly as the engine proves one: the
500
+ * signal goes out only after `invocationOwned` confirms the pid still answers
501
+ * a signal-0 probe and still carries the token the holder recorded. A recycled
502
+ * pid is never signalled, and a lock that recorded no token is unverifiable --
503
+ * the function returns false and the caller's takeover path handles whatever is
504
+ * left, as it would a dead controller. EPERM is not ownership and not a failure
505
+ * to surface: like ESRCH it means gone-or-not-ours and never escapes.
506
+ *
507
+ * A lock naming this process or its parent is refused one step earlier: that
508
+ * lock's group is the caller's own, so the refusal is recorded and returns
509
+ * false without ever probing or signalling it.
510
+ *
498
511
  * @param {string} runDir
499
- * @param {{kill?: (pid: number, signal: string) => void, alive?: (pid: number) => boolean, sleep?: (ms: number) => Promise<void>, now?: () => number, graceMs?: number, killGraceMs?: number}} [options]
500
- * @returns {Promise<boolean>} whether a group was found and terminated
512
+ * @param {{kill?: (pid: number, signal: string) => void, alive?: (pid: number) => boolean, sleep?: (ms: number) => Promise<void>, now?: () => number, graceMs?: number, killGraceMs?: number, append?: (event: Record<string, unknown>) => void}} [options]
513
+ * @returns {Promise<boolean>} whether a verified group was found and terminated
501
514
  */
502
515
  export async function terminateControllerGroup(runDir, options = {}) {
503
516
  const lock = readLock(runDir);
504
517
  if (!lock || /** @type {{invalid?: true}} */ (lock).invalid) return false;
505
518
  const record = /** @type {import("../run/lock.mjs").LockRecord} */ (lock);
506
519
  const pid = record.pid;
520
+ if (refuseSelfSignal(pid, options.append)) return false;
521
+ if (!invocationOwned({ pid, processGroupId: pid, processStartToken: record.processStartToken })) return false;
507
522
  const kill = options.kill ?? ((target, signal) => {
508
523
  try {
509
524
  // The controller is normally detached, so its pid is its process group
510
525
  // id; a non-detached holder has no such group, so fall back to the pid.
526
+ // A group that is gone or not ours falls through to the pid probe.
511
527
  if (process.platform !== "win32") {
512
528
  try {
513
529
  process.kill(-target, signal);
514
530
  return;
515
531
  } catch (groupError) {
516
- if (errorCode(groupError) !== "ESRCH") throw groupError;
532
+ if (errorCode(groupError) !== "ESRCH" && errorCode(groupError) !== "EPERM") throw groupError;
517
533
  }
518
534
  }
519
535
  process.kill(target, signal);
520
536
  } catch (error) {
521
- if (errorCode(error) !== "ESRCH") throw error;
537
+ if (errorCode(error) !== "ESRCH" && errorCode(error) !== "EPERM") throw error;
522
538
  }
523
539
  });
524
540
  const alive = options.alive ?? ((target) => pidAlive(target) || groupAlive(target));
@@ -527,9 +543,19 @@ export async function terminateControllerGroup(runDir, options = {}) {
527
543
  const graceMs = options.graceMs ?? DEFAULT_TERMINATE_GRACE_MS;
528
544
  const killGraceMs = options.killGraceMs ?? DEFAULT_TERMINATE_KILL_GRACE_MS;
529
545
  if (!alive(pid)) return false;
530
- kill(pid, "SIGTERM");
546
+ try {
547
+ kill(pid, "SIGTERM");
548
+ } catch (error) {
549
+ if (errorCode(error) === "ESRCH" || errorCode(error) === "EPERM") return false;
550
+ throw error;
551
+ }
531
552
  if (await waitForGroupGone(pid, alive, graceMs, sleep, now)) return true;
532
- kill(pid, "SIGKILL");
553
+ try {
554
+ kill(pid, "SIGKILL");
555
+ } catch (error) {
556
+ if (errorCode(error) === "ESRCH" || errorCode(error) === "EPERM") return false;
557
+ throw error;
558
+ }
533
559
  await waitForGroupGone(pid, alive, killGraceMs, sleep, now);
534
560
  return true;
535
561
  }