faberun 0.3.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +152 -100
  2. package/package.json +8 -2
  3. package/skills/faberun/SKILL.md +6 -5
  4. package/skills/faberun/references/contract.md +23 -11
  5. package/skills/faberun/references/engineering.md +3 -1
  6. package/skills/faberun/references/operations.md +19 -12
  7. package/skills/faberun/references/rules.md +3 -1
  8. package/src/campaign/chain.mjs +6 -2
  9. package/src/campaign/index.mjs +17 -1
  10. package/src/campaign/metrics.mjs +3 -3
  11. package/src/cli/brand.mjs +2 -1
  12. package/src/cli/setup.mjs +109 -30
  13. package/src/cli/skills.mjs +308 -8
  14. package/src/cli.mjs +2 -1
  15. package/src/contract/final-verification.mjs +31 -2
  16. package/src/contract/index.mjs +27 -24
  17. package/src/contract/runtime.mjs +5 -1
  18. package/src/contract/task-packet.mjs +20 -9
  19. package/src/contract/verification.mjs +1 -1
  20. package/src/engine/backoff.mjs +1 -1
  21. package/src/engine/dispatch.mjs +5 -3
  22. package/src/engine/gate.mjs +12 -0
  23. package/src/engine/process-identity.mjs +39 -0
  24. package/src/engine/prompts.mjs +18 -0
  25. package/src/engine/resume.mjs +2 -2
  26. package/src/engine/review.mjs +9 -1
  27. package/src/engine/run-command.mjs +23 -2
  28. package/src/engine/run-identity.mjs +14 -0
  29. package/src/engine/scheduler.mjs +45 -12
  30. package/src/engine/settle.mjs +29 -0
  31. package/src/engine/supervise.mjs +32 -6
  32. package/src/engine/verify.mjs +98 -9
  33. package/src/harnesses/agy/index.mjs +3 -0
  34. package/src/harnesses/claude/index.mjs +5 -0
  35. package/src/harnesses/codex/index.mjs +3 -0
  36. package/src/harnesses/dsh/index.mjs +26 -0
  37. package/src/harnesses/exec-jsonl/index.mjs +2 -0
  38. package/src/harnesses/index.mjs +10 -3
  39. package/src/harnesses/replay/index.mjs +2 -0
  40. package/src/harnesses/zcode/index.mjs +3 -0
  41. package/src/host/preflight.mjs +5 -1
  42. package/src/notify/index.mjs +45 -2
  43. package/src/repo/source-identity.mjs +4 -3
  44. package/src/report/render.mjs +128 -49
  45. package/src/web/index.html +1 -1
@@ -4,7 +4,7 @@ import { dirname, resolve } from "node:path";
4
4
  import { loadTaskPacket, renderWorkerPrompt } from "./task-packet.mjs";
5
5
  import { RESERVED_ARTICLES } from "./articles.mjs";
6
6
  import { validateDefinitionOfDone } from "./definition-of-done.mjs";
7
- import { validateFinalVerification } from "./final-verification.mjs";
7
+ import { validateFinalVerification, validateSharedVerification } from "./final-verification.mjs";
8
8
  import { VERIFICATION_LIMITS } from "./verification.mjs";
9
9
  import {
10
10
  validateCapabilityRequirements,
@@ -23,7 +23,7 @@ export { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION } from "../harnesses/index.mj
23
23
  const CONTRACT_FIELDS = new Set([
24
24
  "schemaVersion", "contractVersion", "id", "campaignId", "goal", "cwd", "sourceIdentity",
25
25
  "maxParallel", "pollIntervalMs", "stallTimeoutSec", "timeoutSec",
26
- "runtimeDefaults", "runtimes", "nodes", "warnings", "finalVerification", "nodeAdvisory",
26
+ "runtimeDefaults", "runtimes", "nodes", "warnings", "finalVerification", "sharedVerification", "nodeAdvisory",
27
27
  ]);
28
28
  const DEFAULTS_FIELDS = new Set(["worker", "judge"]);
29
29
  const NODE_FIELDS = new Set([
@@ -39,7 +39,7 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
39
39
 
40
40
  /** @typedef {{structuredOutput?: boolean, promptTransport?: "stdin"|"argv", sandbox?: boolean, permissions?: boolean, continuation?: boolean, tokenBudget?: boolean, costBudget?: boolean, usage?: boolean, cost?: boolean}} CapabilityRequirements */
41
41
 
42
- /** @typedef {{kind: string, id?: string, campaignId?: string, contractId?: string, nodeId?: string, cwd?: string, gitHead?: string|null, dirtyTreeFingerprint?: string|null, packetHashes?: Record<string, string>, harnessVersions?: Record<string, string|null>}} SourceIdentity */
42
+ /** @typedef {{kind: string, id?: string, campaignId?: string, contractId?: string, nodeId?: string, cwd?: string, gitHead?: string|null, dirtyTreeFingerprint?: string|null, packetHashes?: Record<string, string>, harnessVersions?: Record<string, string|null>, baseRef?: string|null}} SourceIdentity */
43
43
 
44
44
  /** @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[]}} VerificationCommand */
45
45
 
@@ -51,7 +51,7 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
51
51
 
52
52
  /** @typedef {{id: string, type: string, phase: string, runtime?: string, dependsOn: string[], taskPacket: TaskPacket, taskPacketFile?: string, prompt: string, definitionOfDone: import("./definition-of-done.mjs").DefinitionOfDoneItem[], gate: ValidatedGate, timeoutSec?: number, requiredCapabilities: CapabilityRequirements, packetHash: string, sourceIdentity: SourceIdentity, replayPolicy: "safe"|"reconcile"|"never"}} ValidatedNode */
53
53
 
54
- /** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, finalVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
54
+ /** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, finalVerification?: VerificationCommand[], sharedVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
55
55
  /** @typedef {{costUsd?: number, durationSec?: number}} NodeAdvisoryPolicy */
56
56
 
57
57
  /** @typedef {"pending"|"running"|"done"|"no-op"|"blocked"|"failed"|"exhausted"|"stalled"|"canceled"} NodeStatus */
@@ -147,7 +147,7 @@ export function validateContract(raw, contractPath, options = {}) {
147
147
  }
148
148
  const rawNodes = /** @type {JsonObject[]} */ (raw.nodes);
149
149
  const ids = new Set();
150
- /** @type {{path: string, label: string}[][]} */
150
+ /** @type {{path: string, label: string, kind: "read"|"acknowledged"}[][]} */
151
151
  const deferredReadsByNode = [];
152
152
  const nodes = rawNodes.map((node, index) => {
153
153
  assertObject(node, `nodes[${index}]`);
@@ -165,12 +165,12 @@ export function validateContract(raw, contractPath, options = {}) {
165
165
  if (!Array.isArray(dependsOn) || dependsOn.some((id) => typeof id !== "string")) {
166
166
  throw new TypeError(`nodes[${index}].dependsOn must be an array of ids`);
167
167
  }
168
- // A readFiles entry that names a file no dependency has produced yet is a
169
- // missing read today, but the graph is not known until every node is
170
- // loaded. Collect the candidate here; the second pass below resolves each
171
- // against the node's transitive closure once all packets and dependsOn
172
- // edges are in hand.
173
- /** @type {{path: string, label: string}[]} */
168
+ // A readFiles -- or scopeAcknowledged -- entry that names a file no
169
+ // dependency has produced yet is a missing path today, but the graph is not
170
+ // known until every node is loaded. Collect the candidate here; the second
171
+ // pass below resolves each against the node's transitive closure once all
172
+ // packets and dependsOn edges are in hand.
173
+ /** @type {{path: string, label: string, kind: "read"|"acknowledged"}[]} */
174
174
  const deferredReads = [];
175
175
  const taskPacket = loadTaskPacket(node, contractDir, cwd, index, { deferMissingReads: true, deferredReads, persisted });
176
176
  deferredReadsByNode.push(deferredReads);
@@ -234,21 +234,22 @@ export function validateContract(raw, contractPath, options = {}) {
234
234
  }
235
235
  assertAcyclic(nodes);
236
236
 
237
- // Second pass: a readFiles entry deferred at packet load is accepted only
238
- // when some transitive dependency produces it -- declares the identical path
239
- // in its writeFiles, or the path sits under a dependency's directory-shaped
240
- // writeRoots entry (a file-shaped entry authorizes exactly that path). Every
241
- // other caller of validateTaskPacket keeps rejecting the missing read inline;
242
- // this graph-aware deferral is a contract-loading capability only. Persisted
243
- // loads never defer -- they skipped the existence probe, so there is nothing
244
- // to resolve and nothing to stat.
237
+ // Second pass: a readFiles or scopeAcknowledged entry deferred at packet
238
+ // load is accepted only when some transitive dependency produces it --
239
+ // declares the identical path in its writeFiles, or the path sits under a
240
+ // dependency's directory-shaped writeRoots entry (a file-shaped entry
241
+ // authorizes exactly that path). Every other caller of validateTaskPacket
242
+ // keeps rejecting the missing path inline; this graph-aware deferral is a
243
+ // contract-loading capability only. Persisted loads never defer -- they
244
+ // skipped the existence probe, so there is nothing to resolve and nothing to
245
+ // stat.
245
246
  if (!persisted) {
246
247
  for (const [index, node] of nodes.entries()) {
247
248
  const deferredReads = deferredReadsByNode[index];
248
249
  if (deferredReads.length === 0) continue;
249
250
  const closure = transitiveDependencyClosure(node, nodes);
250
251
  for (const { path, label } of deferredReads) {
251
- if (!dependencyCoversRead(closure, path, cwd)) {
252
+ if (!dependencyCoversPath(closure, path, cwd)) {
252
253
  throw new TypeError(`${label} does not exist: ${path}`);
253
254
  }
254
255
  }
@@ -351,13 +352,15 @@ export function validateContract(raw, contractPath, options = {}) {
351
352
  stallTimeoutSec: positiveNumber(raw.stallTimeoutSec ?? 300, "contract.stallTimeoutSec"),
352
353
  timeoutSec: positiveNumber(raw.timeoutSec ?? 2_400, "contract.timeoutSec"),
353
354
  finalVerification: validateFinalVerification(raw.finalVerification, "contract.finalVerification"),
355
+ sharedVerification: validateSharedVerification(raw.sharedVerification, "contract.sharedVerification"),
354
356
  nodeAdvisory: validateNodeAdvisory(raw.nodeAdvisory),
355
357
  warnings,
356
358
  });
357
359
  // The persisted load is a replay, not a re-authoring: it accepts only bytes
358
360
  // whose digest matches the decision frozen at launch. A changed DAG, gate,
359
- // runtime selection, timeout, definition of done or finalVerification leaves
360
- // every packetHash untouched, so only this digest refuses it.
361
+ // runtime selection, timeout, definition of done, finalVerification or
362
+ // sharedVerification leaves every packetHash untouched, so only this digest
363
+ // refuses it.
361
364
  if (persisted && options.contractDigest !== undefined && contractDigest(raw) !== options.contractDigest) {
362
365
  throw new TypeError("persisted contract does not match the contractDigest recorded at run creation; the stored contract was modified after the run was created");
363
366
  }
@@ -593,7 +596,7 @@ function transitiveDependencyClosure(node, nodes) {
593
596
  }
594
597
 
595
598
  /**
596
- * Whether a transitive dependency produces the deferred read: it declares the
599
+ * Whether a transitive dependency produces the deferred path: it declares the
597
600
  * identical path in `writeFiles`, or the path sits under a directory-shaped
598
601
  * `writeRoots` entry. A `writeRoots` entry that names an existing regular file
599
602
  * authorizes exactly that path and nothing beneath it, mirroring the
@@ -604,7 +607,7 @@ function transitiveDependencyClosure(node, nodes) {
604
607
  * @param {string} cwd
605
608
  * @returns {boolean}
606
609
  */
607
- function dependencyCoversRead(closure, path, cwd) {
610
+ function dependencyCoversPath(closure, path, cwd) {
608
611
  for (const dependency of closure) {
609
612
  const packet = dependency.taskPacket;
610
613
  if ((packet.writeFiles ?? []).includes(path)) return true;
@@ -41,7 +41,7 @@ const PRICING_FIELDS = new Set(["inputPerMTok", "cachedInputPerMTok", "outputPer
41
41
  const SNAPSHOT_RUNTIME_FIELDS = new Set(["id", ...RUNTIME_FIELDS, "capabilities"]);
42
42
  const CAPABILITY_FIELDS = new Set([
43
43
  "structuredOutput", "promptTransport", "sandbox", "permissions", "continuation", "tokenBudget", "costBudget",
44
- "usage", "cost", "toolPolicy", "streamsOutput", "maxArgvPromptBytes",
44
+ "usage", "cost", "toolPolicy", "streamsOutput", "signalsProcesses", "maxArgvPromptBytes",
45
45
  ]);
46
46
  /** @typedef {{id: string, type?: string, runtime?: string, gate: {runtime?: string}, status?: NodeStatus, errorCode?: string, currentRuntime?: string}} RoutableNode */
47
47
  /** @typedef {{status?: NodeStatus, errorCode?: string, currentRuntime?: string, assignment?: string, availability?: Record<string, RuntimeAvailability>}} RoutingEvent */
@@ -187,6 +187,10 @@ export function validateCapabilities(value, label) {
187
187
  for (const name of ["structuredOutput", "sandbox", "permissions", "continuation", "tokenBudget", "costBudget", "usage", "cost", "toolPolicy", "streamsOutput"]) {
188
188
  if (typeof value[name] !== "boolean") throw new TypeError(`${label}.${name} must be boolean`);
189
189
  }
190
+ // `signalsProcesses` is tri-state: null is a sandbox nobody has measured.
191
+ if (value.signalsProcesses !== null && typeof value.signalsProcesses !== "boolean") {
192
+ throw new TypeError(`${label}.signalsProcesses must be boolean or null`);
193
+ }
190
194
  if (!["stdin", "argv"].includes(/** @type {string} */ (value.promptTransport))) {
191
195
  throw new TypeError(`${label}.promptTransport is invalid`);
192
196
  }
@@ -19,11 +19,18 @@ const FIELDS = new Set([
19
19
  "verification",
20
20
  ]);
21
21
  const PROMPT_MAX_BYTES = 64 * 1024;
22
+ /**
23
+ * The `## Verification` preamble every worker prompt carries. The controller's
24
+ * recorded run is the proof; a self-run is optional and must be a quick,
25
+ * process-free command, so a sandbox that cannot signal processes never hangs
26
+ * on the node's own verification.
27
+ */
28
+ const VERIFICATION_PARAGRAPH = "The controller runs every command below after you report; its recorded results are the proof of this node. Running a command yourself is optional and only for one that finishes in seconds and spawns no long-lived process. Keep output bounded (pipe through `| tail -n 200`). Never wait on a background job, never run the whole test suite, and never run tests that start and terminate other processes.";
22
29
 
23
30
  /**
24
- * `validateRelativePath`'s answer when a read path is absent and the caller
25
- * asked to defer the missing-read verdict rather than throw it. Only contract
26
- * loading opts in; every other caller throws the missing-read error in place.
31
+ * `validateRelativePath`'s answer when a path is absent and the caller asked to
32
+ * defer the missing-path verdict rather than throw it. Only contract loading
33
+ * opts in; every other caller throws the missing-path error in place.
27
34
  */
28
35
  const DEFERRED_MISSING = Symbol("deferred-missing");
29
36
 
@@ -47,7 +54,7 @@ const DEFERRED_MISSING = Symbol("deferred-missing");
47
54
  * @param {string} contractDir
48
55
  * @param {string} cwd
49
56
  * @param {number} index
50
- * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string}[], persisted?: boolean}} [options]
57
+ * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string, kind: "read"|"acknowledged"}[], persisted?: boolean}} [options]
51
58
  * @returns {TaskPacket}
52
59
  */
53
60
  export function loadTaskPacket(node, contractDir, cwd, index, options = {}) {
@@ -117,7 +124,7 @@ export function renderWorkerPrompt(packet, nodeId) {
117
124
  ...bulletOrNone(packet.nonGoals),
118
125
  "",
119
126
  "## Verification",
120
- "Run each command yourself before reporting done. Keep command output bounded: pipe long output through `| tail -n 200` (or similar) and never paste full test or fuzz logs into your context or results.",
127
+ VERIFICATION_PARAGRAPH,
121
128
  ...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
122
129
  "",
123
130
  "## Required output",
@@ -134,7 +141,7 @@ export function renderWorkerPrompt(packet, nodeId) {
134
141
  * @param {unknown} packet
135
142
  * @param {number} index
136
143
  * @param {string} cwd
137
- * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string}[], persisted?: boolean}} [options]
144
+ * @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string, kind: "read"|"acknowledged"}[], persisted?: boolean}} [options]
138
145
  * @returns {TaskPacket}
139
146
  */
140
147
  export function validateTaskPacket(packet, index, cwd, options = {}) {
@@ -207,18 +214,20 @@ export function validateTaskPacket(packet, index, cwd, options = {}) {
207
214
 
208
215
  if (!persisted) {
209
216
  const deferMissingReads = options.deferMissingReads === true;
210
- /** @type {{path: string, label: string}[]} */
217
+ /** @type {{path: string, label: string, kind: "read"|"acknowledged"}[]} */
211
218
  const deferredReads = options.deferredReads ?? [];
212
219
  normalizedReadFiles.forEach((path, pathIndex) => {
213
220
  const label = `nodes[${index}].taskPacket.readFiles[${pathIndex}]`;
214
221
  const result = validateRelativePath(path, label, cwd, true, { deferMissing: deferMissingReads });
215
- if (result === DEFERRED_MISSING) deferredReads.push({ path, label });
222
+ if (result === DEFERRED_MISSING) deferredReads.push({ path, label, kind: "read" });
216
223
  });
217
224
  if (writeFiles !== undefined) /** @type {string[]} */ (writeFiles).forEach((path, pathIndex) => {
218
225
  validateRelativePath(path, `nodes[${index}].taskPacket.writeFiles[${pathIndex}]`, cwd, false);
219
226
  });
220
227
  /** @type {string[]} */ (scopeAcknowledged).forEach((path, pathIndex) => {
221
- validateRelativePath(path, `nodes[${index}].taskPacket.scopeAcknowledged[${pathIndex}]`, cwd, true);
228
+ const label = `nodes[${index}].taskPacket.scopeAcknowledged[${pathIndex}]`;
229
+ const result = validateRelativePath(path, label, cwd, true, { deferMissing: deferMissingReads });
230
+ if (result === DEFERRED_MISSING) deferredReads.push({ path, label, kind: "acknowledged" });
222
231
  });
223
232
  for (const [commandIndex, command] of verification.entries()) {
224
233
  if (command.cwd !== undefined) {
@@ -426,6 +435,7 @@ function renderDiscoveryPrompt(packet, nodeId) {
426
435
  'Return exactly one worker-result JSON object, with no markdown or prose. Set status to "done", missingContext to [], and artifacts to an array containing exactly one JSON-stringified execution task packet with every required taskPacket field. The packet readFiles and writeFiles must be non-empty and scoped to this repository.',
427
436
  "",
428
437
  "## Verification",
438
+ VERIFICATION_PARAGRAPH,
429
439
  ...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
430
440
  ];
431
441
  const prompt = `${lines.join("\n")}\n`;
@@ -467,6 +477,7 @@ function renderAutonomousPrompt(packet, nodeId) {
467
477
  ...bulletOrNone(packet.nonGoals),
468
478
  "",
469
479
  "## Verification",
480
+ VERIFICATION_PARAGRAPH,
470
481
  ...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
471
482
  "",
472
483
  "## Required output",
@@ -38,7 +38,7 @@ export const VERIFICATION_LIMITS = Object.freeze({
38
38
  /**
39
39
  * Bounded evidence captured for one attempt.
40
40
  *
41
- * @typedef {{passed: boolean, stdout: string, stderr: string, error: string|null, exitCode: number|null, signal: string|null, timedOut: boolean, durationMs: number|null}} VerificationAttemptResult
41
+ * @typedef {{passed: boolean, stdout: string, stderr: string, error: string|null, exitCode: number|null, signal: string|null, timedOut: boolean, durationMs: number|null, signalDeath?: boolean}} VerificationAttemptResult
42
42
  */
43
43
 
44
44
  /**
@@ -9,7 +9,7 @@
9
9
  * third time buys nothing: change provider, or stop and say so.
10
10
  *
11
11
  * Everything here is a pure decision. Persisting it — the routing history,
12
- * the override, the node transition — stays in runner.mjs, so this module can
12
+ * the override, the node transition — stays in cli.mjs, so this module can
13
13
  * be tested without a run directory, a lease, or a provider.
14
14
  */
15
15
  import { nextHop, nextSynthesizedRuntime, synthesizedChain } from "./failover.mjs";
@@ -9,7 +9,7 @@
9
9
  * which made dispatch depend on review policy and on settlement, and that is
10
10
  * the shape that kept `engine/` a web instead of a stack.
11
11
  */
12
- import { JUDGE_SCHEMA, judgePrompt } from "./prompts.mjs";
12
+ import { JUDGE_SCHEMA, appendSandboxNotice, judgePrompt } from "./prompts.mjs";
13
13
  import { LockLostError } from "../run/lock.mjs";
14
14
  import { TOOL_OUTPUT_LIMIT_BYTES } from "../harnesses/exec-jsonl/index.mjs";
15
15
  import { appendPreviousAttempt } from "./retry.mjs";
@@ -404,6 +404,8 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
404
404
  // so it is appended to the resolved prompt rather than the candidate handed
405
405
  // to phaseInvocationPlan.
406
406
  phasePlan.prompt = appendPreviousAttempt(phasePlan.prompt, state.previousAttempt);
407
+ // The resolved worker's sandbox is a dispatch-time fact, not a packet one.
408
+ phasePlan.prompt = appendSandboxNotice(phasePlan.prompt, runtime);
407
409
  // The worker prompt directs the provider to write the canonical result file;
408
410
  // make sure the directory exists before the provider is asked to.
409
411
  const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
@@ -526,11 +528,11 @@ export function startResultMaterialization(contract, node, state, runDir, runnin
526
528
  const paths = logPaths(runDir, node.id, "worker", state.attempt);
527
529
  const workspace = attemptWorkspace(state) ?? contract.cwd;
528
530
  const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
529
- const prompt = [
531
+ const prompt = appendSandboxNotice([
530
532
  `${RESULT_MATERIALIZATION_PROMPT_HEADER} Do not inspect, implement, verify, or invoke tools.`,
531
533
  `Your only job in this single bounded turn is to write the required worker-result JSON object to: ${resultPath}`,
532
534
  "Then return that same JSON object as the final message.",
533
- ].join("\n\n");
535
+ ].join("\n\n"), materializationRuntime);
534
536
  let baseline;
535
537
  try {
536
538
  baseline = captureWorkspaceSnapshot(workspace);
@@ -21,8 +21,14 @@
21
21
  *
22
22
  * Nothing here is exported for a caller: this file is a program. Every name
23
23
  * below is module-local.
24
+ *
25
+ * It also exits when the release file's directory is gone (measured
26
+ * 2026-09-16: gate processes from a prior day's test runs, spawned into a
27
+ * temp directory the failed test never cleaned up, were still alive and
28
+ * waiting for a release file that could now never appear).
24
29
  */
25
30
  import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
31
+ import { dirname } from "node:path";
26
32
  import { spawn } from "node:child_process";
27
33
 
28
34
  /** @typedef {{executable: string, args: string[], cwd: string, promptTransport: "stdin"|"argv", harness: string, env: Record<string, string|null>|null, stdoutPath: string, stderrPath: string}} GateConfig */
@@ -158,8 +164,14 @@ function childEnv() {
158
164
  process.on("SIGTERM", () => stopProvider());
159
165
  process.on("SIGINT", () => stopProvider());
160
166
 
167
+ /** @returns {boolean} */
168
+ function releaseDirectoryGone() {
169
+ return !existsSync(dirname(releasePath));
170
+ }
171
+
161
172
  const timer = setInterval(() => {
162
173
  if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
174
+ if (releaseDirectoryGone()) { clearInterval(timer); stopProvider(); return; }
163
175
  if (!existsSync(releasePath)) return;
164
176
  clearInterval(timer);
165
177
  const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
@@ -13,6 +13,41 @@ import { processStartToken } from "../run/lock.mjs";
13
13
  /** @typedef {import("node:child_process").ChildProcess} ChildProcess */
14
14
  /** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
15
15
 
16
+ /**
17
+ * Whether a controller may signal a pid's process group at all. It may not when
18
+ * the pid is the calling process -- whose group is the controller's own -- or
19
+ * its parent, whose group is the runner (or host scheduler) that spawned the
20
+ * controller: either signal takes down the caller instead of the invocation.
21
+ * The refusal is recorded and returned, never thrown.
22
+ *
23
+ * The line goes to the injected sink when a caller supplied one, otherwise to
24
+ * stderr; this function owns no run directory and no event-log schema, so
25
+ * stderr is the one recorder it always has. Recording is diagnostic, so a
26
+ * failure to record must never turn the refusal into a throw.
27
+ *
28
+ * @param {number} pid
29
+ * @param {((event: Record<string, unknown>) => void)|undefined} [append] injected event sink
30
+ * @returns {boolean} true when the pid is this process or its parent and the signal was refused
31
+ */
32
+ export function refuseSelfSignal(pid, append) {
33
+ const relation = pid === process.pid ? "self" : pid === process.ppid ? "parent" : null;
34
+ if (relation === null) return false;
35
+ const event = {
36
+ type: "controller_self_signal_refused",
37
+ at: new Date().toISOString(),
38
+ pid,
39
+ processPid: process.pid,
40
+ relation,
41
+ };
42
+ try {
43
+ if (append) append(event);
44
+ else process.stderr.write(`[warn] controller_self_signal_refused ${JSON.stringify(event)}\n`);
45
+ } catch {
46
+ // The refusal is diagnostic: a failed recorder must never become a throw.
47
+ }
48
+ return true;
49
+ }
50
+
16
51
  /**
17
52
  * A process group answers a signal-0 probe. EPERM means the group exists but is
18
53
  * not this user's, which is no more "ours" than a missing group.
@@ -57,6 +92,10 @@ export function processStartTokenMatches(invocation) {
57
92
  */
58
93
  export function invocationOwned(invocation, options = {}) {
59
94
  if (!invocation?.pid || !Number.isInteger(invocation.pid)) return false;
95
+ // A controller does not spawn itself or its parent: a recorded pid that names
96
+ // either can never be an invocation this controller owns, whatever token or
97
+ // child handle accompanies it, and signalling it would hit the caller.
98
+ if (invocation.pid === process.pid || invocation.pid === process.ppid) return false;
60
99
  try {
61
100
  process.kill(invocation.pid, 0);
62
101
  } catch {
@@ -1,6 +1,7 @@
1
1
  import { validateWorkerResult } from "../contract/worker-result.mjs";
2
2
  import { scopeFindingsPromptSection } from "../contract/scope-findings.mjs";
3
3
  import { JUDGE_ENVELOPE_REASON, JUDGE_FINDING_ENVELOPE_REASON, JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
4
+ import { harnessCapabilities } from "../harnesses/index.mjs";
4
5
 
5
6
  /** @typedef {{id: string, definitionOfDone: import("../contract/definition-of-done.mjs").DefinitionOfDoneItem[], taskPacket: {mode?: "execution"|"discovery"|"autonomous", objective: string, instructions: string[], writeFiles?: string[], writeRoots?: string[], verification: {argv: string[]}[]}}} JudgeNode */
6
7
  /** @typedef {{verdict: "pass"|"fail"|"invalid_judge_output", maxSeverity: "none"|"minor"|"major"|"critical", summary: string, findings: {severity: "minor"|"major"|"critical", description: string, evidence: string}[]}} JudgeVerdict */
@@ -276,6 +277,23 @@ export function retryPrompt(node, verdict) {
276
277
  : prompt;
277
278
  }
278
279
 
280
+ /**
281
+ * Append the `## Sandbox` warning a resolved worker runtime earns. Only a
282
+ * harness whose adapter declares `signalsProcesses === false` gets it: that
283
+ * sandbox cannot signal child processes or read the process table, so a test
284
+ * that starts and terminates a child hangs until the executor's cap. An
285
+ * unmeasured harness (`null`) is left alone — no measurement justifies the
286
+ * warning.
287
+ *
288
+ * @param {string} prompt
289
+ * @param {{harness: string}} runtime
290
+ * @returns {string}
291
+ */
292
+ export function appendSandboxNotice(prompt, runtime) {
293
+ if (harnessCapabilities(runtime).signalsProcesses !== false) return prompt;
294
+ return `${prompt}\n\n## Sandbox\nYour harness runs you in a sandbox that cannot signal other processes or read the process table. A test that starts and terminates a child process hangs here until the executor's cap and is then killed as a stall. Do not run such tests; the controller runs them after you report.`;
295
+ }
296
+
279
297
  /**
280
298
  * @param {unknown} value
281
299
  * @param {number} maxBytes
@@ -17,7 +17,7 @@ import { Buffer } from "node:buffer";
17
17
  import { acquire as acquireLock } from "../run/lock.mjs";
18
18
  import { applyInvalidWorkerResult, assertRunMutable, handleProviderExhaustion } from "./lifecycle.mjs";
19
19
  import { applyJudgeResult } from "./review.mjs";
20
- import { assertSourceUnchanged, captureRunIdentity, statesFingerprint } from "./run-identity.mjs";
20
+ import { assertSourceUnchanged, captureRunIdentity, recordedBaseRef, statesFingerprint } from "./run-identity.mjs";
21
21
  import { attemptWorkspace, attemptWorktreePath, gitHead, removeWorktree, runRefName } from "../repo/worktree.mjs";
22
22
  import { canonicalWorkerResultText, isResultMaterializationInvocation, materializeAttemptResult, recoverWorkerResult } from "./result-file.mjs";
23
23
  import { checkPersistedWorkerScope, persistedScopeBoundary, reconcileAmbiguousWorkerRestart, resolveUnknownEffect } from "./scope.mjs";
@@ -114,7 +114,7 @@ export async function resumeRun(runDirPath, options = {}) {
114
114
  node.id,
115
115
  persistedScopeBoundary(contract, node, states.get(node.id), attemptWorkspace(states.get(node.id)) ?? contract.cwd),
116
116
  ]));
117
- const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries);
117
+ const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries, recordedBaseRef(storedMetadata) ?? undefined);
118
118
  const identity = assertSourceUnchanged(storedMetadata.sourceIdentity, sourceIdentity);
119
119
  // A resume is an explicit instruction to continue the run: it consumes a
120
120
  // stale cancel request instead of letting it re-cancel the retried nodes.
@@ -21,7 +21,7 @@ import {
21
21
  } from "../contract/review-modes.mjs";
22
22
 
23
23
  import { errorMessage, excerpt } from "../util.mjs";
24
- import { appendTransitionEvent, transition } from "./state.mjs";
24
+ import { appendTransitionEvent, transition, writeNode } from "./state.mjs";
25
25
  import { startJudge } from "./dispatch.mjs";
26
26
  import { applyRejection, raiseNodeAttention, settleDone } from "./settle.mjs";
27
27
 
@@ -154,6 +154,14 @@ export async function applyJudgeResult(contract, node, state, result, runDir, lo
154
154
  await raiseNodeAttention(campaignPath, runDir, state, "judge_protocol");
155
155
  return;
156
156
  }
157
+ // The verdict and the spent bound are durable before the blocked
158
+ // transition: a controller loss in this gap reads the settled re-ask from
159
+ // the node snapshot instead of spending a second one. No await separates
160
+ // the two writes, so the durable record cannot be observed only by luck.
161
+ writeNode(runDir, state, lock);
162
+ if (process.env.FABERUN_JUDGE_REASK_INTERRUPT === "after-verdict") {
163
+ throw new Error("judge re-ask interrupted after verdict persistence");
164
+ }
157
165
  transition(runDir, state, "blocked", {
158
166
  phase: "judge",
159
167
  gate: verdict,
@@ -99,9 +99,30 @@ async function runRepeatedCommand(command, baseCwd, commandIndex, options) {
99
99
  const attempts = [];
100
100
  const repeat = command.repeat ?? 1;
101
101
  for (let attempt = 1; attempt <= repeat; attempt += 1) {
102
- attempts.push(await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex));
102
+ const result = await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex);
103
+ if (signalDeathRetry(result, options.signal)) {
104
+ attempts.push({ ...result, signalDeath: true });
105
+ attempts.push(await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex));
106
+ } else {
107
+ attempts.push(result);
108
+ }
103
109
  }
104
- return { ...command, cwd: resolveVerificationCwd(baseCwd, command.cwd ?? "."), passed: attempts.every((item) => item.passed), attempts };
110
+ const passed = attempts.filter((item) => !item.signalDeath).every((item) => item.passed);
111
+ return { ...command, cwd: resolveVerificationCwd(baseCwd, command.cwd ?? "."), passed, attempts };
112
+ }
113
+ /**
114
+ * Whether an attempt died from a signal the controller itself did not send.
115
+ * `terminateGroup` only fires from the timeout and abort paths below, so a
116
+ * `signal` with neither set is evidence of an external kill (OOM, an operator
117
+ * `kill`, a flaky sandbox) rather than a verdict on the command under test,
118
+ * and gets one retry instead of failing the node outright.
119
+ *
120
+ * @param {VerificationAttemptResult} result
121
+ * @param {AbortSignal|undefined} signal
122
+ * @returns {boolean}
123
+ */
124
+ function signalDeathRetry(result, signal) {
125
+ return Boolean(result.signal) && !result.timedOut && !signal?.aborted;
105
126
  }
106
127
  /**
107
128
  * The mutation case: the entry passes on the mutant-kill fraction, and every
@@ -181,6 +181,20 @@ export function setLaunchBaseRef(baseRef) {
181
181
  pendingLaunchBaseRef = baseRef ?? null;
182
182
  }
183
183
 
184
+ /**
185
+ * The base ref a run was launched against, when it was launched with
186
+ * `--base-ref`. A run recorded before this field existed, or launched
187
+ * without the flag, has none — a resume of that run keeps comparing against
188
+ * the checkout's own HEAD.
189
+ *
190
+ * @param {RunMetadata|undefined} metadata
191
+ * @returns {string|null}
192
+ */
193
+ export function recordedBaseRef(metadata) {
194
+ const baseRef = metadata?.sourceIdentity?.baseRef;
195
+ return typeof baseRef === "string" && baseRef.length > 0 ? baseRef : null;
196
+ }
197
+
184
198
  /**
185
199
  * Capture the run's source identity, including one version-only probe per
186
200
  * distinct routed runtime (a local binary call, no model tokens) so a later
@@ -41,7 +41,7 @@ import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsa
41
41
  import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
42
42
  import { validateContract } from "../contract/index.mjs";
43
43
  import { validateNodeSnapshot } from "../contract/snapshot.mjs";
44
- import { finalVerificationCommands } from "../contract/final-verification.mjs";
44
+ import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
45
45
  import { startJudge, startWorker } from "./dispatch.mjs";
46
46
  import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
47
47
  import { blockDependents, runtimeAssignments } from "./assignment.mjs";
@@ -94,12 +94,13 @@ export function verificationBudgetMs(commands) {
94
94
  /**
95
95
  * The budget a node is judged against, in milliseconds. It is the sum of every
96
96
  * bounded phase the node can legitimately occupy without a state transition:
97
- * its worker invocation (`timeoutSec`), its packet verification and the
98
- * contract's `finalVerification` when it is phase-terminal, an integration
99
- * candidate run of that same set, and the bounded command proofs of its gate.
100
- * A frozen node is one that has been silent longer than this, not merely
101
- * longer than the worker timeout, because a legitimate verification can be
102
- * minutes long and must not be mistaken for a freeze.
97
+ * its worker invocation (`timeoutSec`), its packet verification plus the
98
+ * contract's `sharedVerification` and, when it is phase-terminal, the
99
+ * contract's `finalVerification`, an integration candidate run of that same
100
+ * set, and the bounded command proofs of its gate. A frozen node is one that
101
+ * has been silent longer than this, not merely longer than the worker timeout,
102
+ * because a legitimate verification can be minutes long and must not be
103
+ * mistaken for a freeze.
103
104
  *
104
105
  * @param {ValidatedContract} contract
105
106
  * @param {ValidatedNode} node
@@ -107,14 +108,17 @@ export function verificationBudgetMs(commands) {
107
108
  */
108
109
  export function nodeBudgetBasisMs(contract, node) {
109
110
  const defaultTimeoutMs = (node.timeoutSec ?? contract.timeoutSec ?? 60) * 1_000;
111
+ const sharedMs = verificationBudgetMs(sharedVerificationCommands(contract));
110
112
  const packetMs = verificationBudgetMs(node.taskPacket?.verification);
111
113
  const finalMs = verificationBudgetMs(finalVerificationCommands(contract, node));
112
- // The controller runs the packet set once after the worker and once against
113
- // the integration candidate, and the finalVerification set with each.
114
- const candidateMs = packetMs + finalMs;
114
+ // The controller runs the packet set (with the contract-level shared set) once
115
+ // after the worker and once against the integration candidate, and the
116
+ // finalVerification set with each.
117
+ const attemptMs = packetMs + sharedMs;
118
+ const candidateMs = attemptMs + finalMs;
115
119
  const gateTimeoutMs = Math.max(1_000, Math.min(defaultTimeoutMs, 120_000));
116
120
  const commandProofs = (node.definitionOfDone ?? []).filter((item) => item.proof?.kind === "command").length;
117
- return defaultTimeoutMs + packetMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
121
+ return defaultTimeoutMs + attemptMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
118
122
  }
119
123
 
120
124
  /**
@@ -149,6 +153,26 @@ export function enforceRunningInvariant(runDir, states, running, lock) {
149
153
  return parked;
150
154
  }
151
155
 
156
+ /**
157
+ * The fingerprint that decides whether a render is due: the shared
158
+ * `statesFingerprint` (status, phase, attempt, revisions) plus each running
159
+ * node's verification progress. A verification command completing and the
160
+ * next one starting changes neither status nor phase, so the shared
161
+ * fingerprint alone would never notice it; this is the one used to gate
162
+ * `renderStatusIfChanged` and nothing else, so a resume's drift comparison
163
+ * still uses the unextended `statesFingerprint`.
164
+ *
165
+ * @param {Map<string, NodeSnapshot>} states
166
+ * @returns {string}
167
+ */
168
+ function renderFingerprint(states) {
169
+ const progress = [...states.values()].map((state) => {
170
+ const verification = /** @type {{progress?: {index: number, total: number, argv: string}}|null|undefined} */ (state.verification);
171
+ return verification?.progress ? `${state.id}:${verification.progress.index}/${verification.progress.total}:${verification.progress.argv}` : "";
172
+ }).join("|");
173
+ return `${statesFingerprint(states)}#${progress}`;
174
+ }
175
+
152
176
  /**
153
177
  * @param {string} contractPath
154
178
  * @param {{detachedBootstrap?: boolean}} [options] `detachedBootstrap` is set
@@ -290,11 +314,19 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
290
314
  let statusFingerprint = null;
291
315
  /** @param {boolean} force @param {LockHandle|null} [renderLock] */
292
316
  const renderStatusIfChanged = (force = false, renderLock = lock) => {
293
- const fingerprint = statesFingerprint(states);
317
+ const fingerprint = renderFingerprint(states);
294
318
  if (!force && fingerprint === statusFingerprint) return;
295
319
  statusFingerprint = fingerprint;
296
320
  render(runDir, runsDir, contract, states, renderLock);
297
321
  };
322
+ // The tick that owns a running node's verification can be minutes long
323
+ // (`executeControllerVerification` is awaited on the critical path below),
324
+ // and that whole time status.json would otherwise report whatever the last
325
+ // tick left it at. A timer renders between ticks too; it is cheap even when
326
+ // idle because `renderFingerprint` still change-detects, so a quiet run
327
+ // writes nothing extra.
328
+ const statusTimer = setInterval(() => renderStatusIfChanged(), contract.pollIntervalMs);
329
+ statusTimer.unref();
298
330
  let handoffFingerprint = statesFingerprint(states);
299
331
  const renderHandoffIfChanged = () => {
300
332
  const fingerprint = statesFingerprint(states);
@@ -475,6 +507,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
475
507
  notifyQueuesByRun.delete(runDir);
476
508
  return { runDir, states, ok: false, error };
477
509
  } finally {
510
+ clearInterval(statusTimer);
478
511
  process.removeListener("SIGINT", cancel);
479
512
  process.removeListener("SIGTERM", cancel);
480
513
  process.removeListener("SIGHUP", cancel);