faberun 0.3.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +152 -100
- package/package.json +8 -2
- package/skills/faberun/SKILL.md +6 -5
- package/skills/faberun/references/contract.md +23 -11
- package/skills/faberun/references/engineering.md +3 -1
- package/skills/faberun/references/operations.md +19 -12
- package/skills/faberun/references/rules.md +3 -1
- package/src/campaign/chain.mjs +6 -2
- package/src/campaign/index.mjs +17 -1
- package/src/campaign/metrics.mjs +3 -3
- package/src/cli/brand.mjs +2 -1
- package/src/cli/setup.mjs +109 -30
- package/src/cli/skills.mjs +308 -8
- package/src/cli.mjs +2 -1
- package/src/contract/final-verification.mjs +31 -2
- package/src/contract/index.mjs +27 -24
- package/src/contract/runtime.mjs +5 -1
- package/src/contract/task-packet.mjs +20 -9
- package/src/contract/verification.mjs +1 -1
- package/src/engine/backoff.mjs +1 -1
- package/src/engine/dispatch.mjs +5 -3
- package/src/engine/gate.mjs +12 -0
- package/src/engine/process-identity.mjs +39 -0
- package/src/engine/prompts.mjs +18 -0
- package/src/engine/resume.mjs +2 -2
- package/src/engine/review.mjs +9 -1
- package/src/engine/run-command.mjs +23 -2
- package/src/engine/run-identity.mjs +14 -0
- package/src/engine/scheduler.mjs +45 -12
- package/src/engine/settle.mjs +29 -0
- package/src/engine/supervise.mjs +32 -6
- package/src/engine/verify.mjs +98 -9
- package/src/harnesses/agy/index.mjs +3 -0
- package/src/harnesses/claude/index.mjs +5 -0
- package/src/harnesses/codex/index.mjs +3 -0
- package/src/harnesses/dsh/index.mjs +26 -0
- package/src/harnesses/exec-jsonl/index.mjs +2 -0
- package/src/harnesses/index.mjs +10 -3
- package/src/harnesses/replay/index.mjs +2 -0
- package/src/harnesses/zcode/index.mjs +3 -0
- package/src/host/preflight.mjs +5 -1
- package/src/notify/index.mjs +45 -2
- package/src/repo/source-identity.mjs +4 -3
- package/src/report/render.mjs +128 -49
- package/src/web/index.html +1 -1
package/src/contract/index.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { dirname, resolve } from "node:path";
|
|
|
4
4
|
import { loadTaskPacket, renderWorkerPrompt } from "./task-packet.mjs";
|
|
5
5
|
import { RESERVED_ARTICLES } from "./articles.mjs";
|
|
6
6
|
import { validateDefinitionOfDone } from "./definition-of-done.mjs";
|
|
7
|
-
import { validateFinalVerification } from "./final-verification.mjs";
|
|
7
|
+
import { validateFinalVerification, validateSharedVerification } from "./final-verification.mjs";
|
|
8
8
|
import { VERIFICATION_LIMITS } from "./verification.mjs";
|
|
9
9
|
import {
|
|
10
10
|
validateCapabilityRequirements,
|
|
@@ -23,7 +23,7 @@ export { CONTRACT_VERSION, PROTOCOL_SCHEMA_VERSION } from "../harnesses/index.mj
|
|
|
23
23
|
const CONTRACT_FIELDS = new Set([
|
|
24
24
|
"schemaVersion", "contractVersion", "id", "campaignId", "goal", "cwd", "sourceIdentity",
|
|
25
25
|
"maxParallel", "pollIntervalMs", "stallTimeoutSec", "timeoutSec",
|
|
26
|
-
"runtimeDefaults", "runtimes", "nodes", "warnings", "finalVerification", "nodeAdvisory",
|
|
26
|
+
"runtimeDefaults", "runtimes", "nodes", "warnings", "finalVerification", "sharedVerification", "nodeAdvisory",
|
|
27
27
|
]);
|
|
28
28
|
const DEFAULTS_FIELDS = new Set(["worker", "judge"]);
|
|
29
29
|
const NODE_FIELDS = new Set([
|
|
@@ -39,7 +39,7 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
|
|
|
39
39
|
|
|
40
40
|
/** @typedef {{structuredOutput?: boolean, promptTransport?: "stdin"|"argv", sandbox?: boolean, permissions?: boolean, continuation?: boolean, tokenBudget?: boolean, costBudget?: boolean, usage?: boolean, cost?: boolean}} CapabilityRequirements */
|
|
41
41
|
|
|
42
|
-
/** @typedef {{kind: string, id?: string, campaignId?: string, contractId?: string, nodeId?: string, cwd?: string, gitHead?: string|null, dirtyTreeFingerprint?: string|null, packetHashes?: Record<string, string>, harnessVersions?: Record<string, string|null
|
|
42
|
+
/** @typedef {{kind: string, id?: string, campaignId?: string, contractId?: string, nodeId?: string, cwd?: string, gitHead?: string|null, dirtyTreeFingerprint?: string|null, packetHashes?: Record<string, string>, harnessVersions?: Record<string, string|null>, baseRef?: string|null}} SourceIdentity */
|
|
43
43
|
|
|
44
44
|
/** @typedef {{argv: string[], cwd?: string, timeoutSec?: number, repeat?: number, env?: string[]}} VerificationCommand */
|
|
45
45
|
|
|
@@ -51,7 +51,7 @@ const GATE_REVIEWS = new Set(["none", "advisory", "blocking"]);
|
|
|
51
51
|
|
|
52
52
|
/** @typedef {{id: string, type: string, phase: string, runtime?: string, dependsOn: string[], taskPacket: TaskPacket, taskPacketFile?: string, prompt: string, definitionOfDone: import("./definition-of-done.mjs").DefinitionOfDoneItem[], gate: ValidatedGate, timeoutSec?: number, requiredCapabilities: CapabilityRequirements, packetHash: string, sourceIdentity: SourceIdentity, replayPolicy: "safe"|"reconcile"|"never"}} ValidatedNode */
|
|
53
53
|
|
|
54
|
-
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, finalVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
|
|
54
|
+
/** @typedef {{schemaVersion: number, contractVersion: string, id: string, campaignId: string, goal: string, cwd: string, sourceIdentity: SourceIdentity, runtimes: Record<string, ValidatedRuntime>, runtimeDefaults: {worker?: string, judge?: string}, nodes: ValidatedNode[], maxParallel: number, pollIntervalMs: number, stallTimeoutSec: number, timeoutSec: number, finalVerification?: VerificationCommand[], sharedVerification?: VerificationCommand[], nodeAdvisory?: NodeAdvisoryPolicy, warnings: string[]}} ValidatedContract */
|
|
55
55
|
/** @typedef {{costUsd?: number, durationSec?: number}} NodeAdvisoryPolicy */
|
|
56
56
|
|
|
57
57
|
/** @typedef {"pending"|"running"|"done"|"no-op"|"blocked"|"failed"|"exhausted"|"stalled"|"canceled"} NodeStatus */
|
|
@@ -147,7 +147,7 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
147
147
|
}
|
|
148
148
|
const rawNodes = /** @type {JsonObject[]} */ (raw.nodes);
|
|
149
149
|
const ids = new Set();
|
|
150
|
-
/** @type {{path: string, label: string}[][]} */
|
|
150
|
+
/** @type {{path: string, label: string, kind: "read"|"acknowledged"}[][]} */
|
|
151
151
|
const deferredReadsByNode = [];
|
|
152
152
|
const nodes = rawNodes.map((node, index) => {
|
|
153
153
|
assertObject(node, `nodes[${index}]`);
|
|
@@ -165,12 +165,12 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
165
165
|
if (!Array.isArray(dependsOn) || dependsOn.some((id) => typeof id !== "string")) {
|
|
166
166
|
throw new TypeError(`nodes[${index}].dependsOn must be an array of ids`);
|
|
167
167
|
}
|
|
168
|
-
// A readFiles entry that names a file no
|
|
169
|
-
// missing
|
|
170
|
-
// loaded. Collect the candidate here; the second
|
|
171
|
-
// against the node's transitive closure once all
|
|
172
|
-
// edges are in hand.
|
|
173
|
-
/** @type {{path: string, label: string}[]} */
|
|
168
|
+
// A readFiles -- or scopeAcknowledged -- entry that names a file no
|
|
169
|
+
// dependency has produced yet is a missing path today, but the graph is not
|
|
170
|
+
// known until every node is loaded. Collect the candidate here; the second
|
|
171
|
+
// pass below resolves each against the node's transitive closure once all
|
|
172
|
+
// packets and dependsOn edges are in hand.
|
|
173
|
+
/** @type {{path: string, label: string, kind: "read"|"acknowledged"}[]} */
|
|
174
174
|
const deferredReads = [];
|
|
175
175
|
const taskPacket = loadTaskPacket(node, contractDir, cwd, index, { deferMissingReads: true, deferredReads, persisted });
|
|
176
176
|
deferredReadsByNode.push(deferredReads);
|
|
@@ -234,21 +234,22 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
234
234
|
}
|
|
235
235
|
assertAcyclic(nodes);
|
|
236
236
|
|
|
237
|
-
// Second pass: a readFiles entry deferred at packet
|
|
238
|
-
// when some transitive dependency produces it --
|
|
239
|
-
// in its writeFiles, or the path sits under a
|
|
240
|
-
// writeRoots entry (a file-shaped entry
|
|
241
|
-
//
|
|
242
|
-
// this graph-aware deferral is a
|
|
243
|
-
// loads never defer -- they
|
|
244
|
-
// to resolve and nothing to
|
|
237
|
+
// Second pass: a readFiles or scopeAcknowledged entry deferred at packet
|
|
238
|
+
// load is accepted only when some transitive dependency produces it --
|
|
239
|
+
// declares the identical path in its writeFiles, or the path sits under a
|
|
240
|
+
// dependency's directory-shaped writeRoots entry (a file-shaped entry
|
|
241
|
+
// authorizes exactly that path). Every other caller of validateTaskPacket
|
|
242
|
+
// keeps rejecting the missing path inline; this graph-aware deferral is a
|
|
243
|
+
// contract-loading capability only. Persisted loads never defer -- they
|
|
244
|
+
// skipped the existence probe, so there is nothing to resolve and nothing to
|
|
245
|
+
// stat.
|
|
245
246
|
if (!persisted) {
|
|
246
247
|
for (const [index, node] of nodes.entries()) {
|
|
247
248
|
const deferredReads = deferredReadsByNode[index];
|
|
248
249
|
if (deferredReads.length === 0) continue;
|
|
249
250
|
const closure = transitiveDependencyClosure(node, nodes);
|
|
250
251
|
for (const { path, label } of deferredReads) {
|
|
251
|
-
if (!
|
|
252
|
+
if (!dependencyCoversPath(closure, path, cwd)) {
|
|
252
253
|
throw new TypeError(`${label} does not exist: ${path}`);
|
|
253
254
|
}
|
|
254
255
|
}
|
|
@@ -351,13 +352,15 @@ export function validateContract(raw, contractPath, options = {}) {
|
|
|
351
352
|
stallTimeoutSec: positiveNumber(raw.stallTimeoutSec ?? 300, "contract.stallTimeoutSec"),
|
|
352
353
|
timeoutSec: positiveNumber(raw.timeoutSec ?? 2_400, "contract.timeoutSec"),
|
|
353
354
|
finalVerification: validateFinalVerification(raw.finalVerification, "contract.finalVerification"),
|
|
355
|
+
sharedVerification: validateSharedVerification(raw.sharedVerification, "contract.sharedVerification"),
|
|
354
356
|
nodeAdvisory: validateNodeAdvisory(raw.nodeAdvisory),
|
|
355
357
|
warnings,
|
|
356
358
|
});
|
|
357
359
|
// The persisted load is a replay, not a re-authoring: it accepts only bytes
|
|
358
360
|
// whose digest matches the decision frozen at launch. A changed DAG, gate,
|
|
359
|
-
// runtime selection, timeout, definition of done
|
|
360
|
-
// every packetHash untouched, so only this digest
|
|
361
|
+
// runtime selection, timeout, definition of done, finalVerification or
|
|
362
|
+
// sharedVerification leaves every packetHash untouched, so only this digest
|
|
363
|
+
// refuses it.
|
|
361
364
|
if (persisted && options.contractDigest !== undefined && contractDigest(raw) !== options.contractDigest) {
|
|
362
365
|
throw new TypeError("persisted contract does not match the contractDigest recorded at run creation; the stored contract was modified after the run was created");
|
|
363
366
|
}
|
|
@@ -593,7 +596,7 @@ function transitiveDependencyClosure(node, nodes) {
|
|
|
593
596
|
}
|
|
594
597
|
|
|
595
598
|
/**
|
|
596
|
-
* Whether a transitive dependency produces the deferred
|
|
599
|
+
* Whether a transitive dependency produces the deferred path: it declares the
|
|
597
600
|
* identical path in `writeFiles`, or the path sits under a directory-shaped
|
|
598
601
|
* `writeRoots` entry. A `writeRoots` entry that names an existing regular file
|
|
599
602
|
* authorizes exactly that path and nothing beneath it, mirroring the
|
|
@@ -604,7 +607,7 @@ function transitiveDependencyClosure(node, nodes) {
|
|
|
604
607
|
* @param {string} cwd
|
|
605
608
|
* @returns {boolean}
|
|
606
609
|
*/
|
|
607
|
-
function
|
|
610
|
+
function dependencyCoversPath(closure, path, cwd) {
|
|
608
611
|
for (const dependency of closure) {
|
|
609
612
|
const packet = dependency.taskPacket;
|
|
610
613
|
if ((packet.writeFiles ?? []).includes(path)) return true;
|
package/src/contract/runtime.mjs
CHANGED
|
@@ -41,7 +41,7 @@ const PRICING_FIELDS = new Set(["inputPerMTok", "cachedInputPerMTok", "outputPer
|
|
|
41
41
|
const SNAPSHOT_RUNTIME_FIELDS = new Set(["id", ...RUNTIME_FIELDS, "capabilities"]);
|
|
42
42
|
const CAPABILITY_FIELDS = new Set([
|
|
43
43
|
"structuredOutput", "promptTransport", "sandbox", "permissions", "continuation", "tokenBudget", "costBudget",
|
|
44
|
-
"usage", "cost", "toolPolicy", "streamsOutput", "maxArgvPromptBytes",
|
|
44
|
+
"usage", "cost", "toolPolicy", "streamsOutput", "signalsProcesses", "maxArgvPromptBytes",
|
|
45
45
|
]);
|
|
46
46
|
/** @typedef {{id: string, type?: string, runtime?: string, gate: {runtime?: string}, status?: NodeStatus, errorCode?: string, currentRuntime?: string}} RoutableNode */
|
|
47
47
|
/** @typedef {{status?: NodeStatus, errorCode?: string, currentRuntime?: string, assignment?: string, availability?: Record<string, RuntimeAvailability>}} RoutingEvent */
|
|
@@ -187,6 +187,10 @@ export function validateCapabilities(value, label) {
|
|
|
187
187
|
for (const name of ["structuredOutput", "sandbox", "permissions", "continuation", "tokenBudget", "costBudget", "usage", "cost", "toolPolicy", "streamsOutput"]) {
|
|
188
188
|
if (typeof value[name] !== "boolean") throw new TypeError(`${label}.${name} must be boolean`);
|
|
189
189
|
}
|
|
190
|
+
// `signalsProcesses` is tri-state: null is a sandbox nobody has measured.
|
|
191
|
+
if (value.signalsProcesses !== null && typeof value.signalsProcesses !== "boolean") {
|
|
192
|
+
throw new TypeError(`${label}.signalsProcesses must be boolean or null`);
|
|
193
|
+
}
|
|
190
194
|
if (!["stdin", "argv"].includes(/** @type {string} */ (value.promptTransport))) {
|
|
191
195
|
throw new TypeError(`${label}.promptTransport is invalid`);
|
|
192
196
|
}
|
|
@@ -19,11 +19,18 @@ const FIELDS = new Set([
|
|
|
19
19
|
"verification",
|
|
20
20
|
]);
|
|
21
21
|
const PROMPT_MAX_BYTES = 64 * 1024;
|
|
22
|
+
/**
|
|
23
|
+
* The `## Verification` preamble every worker prompt carries. The controller's
|
|
24
|
+
* recorded run is the proof; a self-run is optional and must be a quick,
|
|
25
|
+
* process-free command, so a sandbox that cannot signal processes never hangs
|
|
26
|
+
* on the node's own verification.
|
|
27
|
+
*/
|
|
28
|
+
const VERIFICATION_PARAGRAPH = "The controller runs every command below after you report; its recorded results are the proof of this node. Running a command yourself is optional and only for one that finishes in seconds and spawns no long-lived process. Keep output bounded (pipe through `| tail -n 200`). Never wait on a background job, never run the whole test suite, and never run tests that start and terminate other processes.";
|
|
22
29
|
|
|
23
30
|
/**
|
|
24
|
-
* `validateRelativePath`'s answer when a
|
|
25
|
-
*
|
|
26
|
-
*
|
|
31
|
+
* `validateRelativePath`'s answer when a path is absent and the caller asked to
|
|
32
|
+
* defer the missing-path verdict rather than throw it. Only contract loading
|
|
33
|
+
* opts in; every other caller throws the missing-path error in place.
|
|
27
34
|
*/
|
|
28
35
|
const DEFERRED_MISSING = Symbol("deferred-missing");
|
|
29
36
|
|
|
@@ -47,7 +54,7 @@ const DEFERRED_MISSING = Symbol("deferred-missing");
|
|
|
47
54
|
* @param {string} contractDir
|
|
48
55
|
* @param {string} cwd
|
|
49
56
|
* @param {number} index
|
|
50
|
-
* @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string}[], persisted?: boolean}} [options]
|
|
57
|
+
* @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string, kind: "read"|"acknowledged"}[], persisted?: boolean}} [options]
|
|
51
58
|
* @returns {TaskPacket}
|
|
52
59
|
*/
|
|
53
60
|
export function loadTaskPacket(node, contractDir, cwd, index, options = {}) {
|
|
@@ -117,7 +124,7 @@ export function renderWorkerPrompt(packet, nodeId) {
|
|
|
117
124
|
...bulletOrNone(packet.nonGoals),
|
|
118
125
|
"",
|
|
119
126
|
"## Verification",
|
|
120
|
-
|
|
127
|
+
VERIFICATION_PARAGRAPH,
|
|
121
128
|
...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
|
|
122
129
|
"",
|
|
123
130
|
"## Required output",
|
|
@@ -134,7 +141,7 @@ export function renderWorkerPrompt(packet, nodeId) {
|
|
|
134
141
|
* @param {unknown} packet
|
|
135
142
|
* @param {number} index
|
|
136
143
|
* @param {string} cwd
|
|
137
|
-
* @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string}[], persisted?: boolean}} [options]
|
|
144
|
+
* @param {{deferMissingReads?: boolean, deferredReads?: {path: string, label: string, kind: "read"|"acknowledged"}[], persisted?: boolean}} [options]
|
|
138
145
|
* @returns {TaskPacket}
|
|
139
146
|
*/
|
|
140
147
|
export function validateTaskPacket(packet, index, cwd, options = {}) {
|
|
@@ -207,18 +214,20 @@ export function validateTaskPacket(packet, index, cwd, options = {}) {
|
|
|
207
214
|
|
|
208
215
|
if (!persisted) {
|
|
209
216
|
const deferMissingReads = options.deferMissingReads === true;
|
|
210
|
-
/** @type {{path: string, label: string}[]} */
|
|
217
|
+
/** @type {{path: string, label: string, kind: "read"|"acknowledged"}[]} */
|
|
211
218
|
const deferredReads = options.deferredReads ?? [];
|
|
212
219
|
normalizedReadFiles.forEach((path, pathIndex) => {
|
|
213
220
|
const label = `nodes[${index}].taskPacket.readFiles[${pathIndex}]`;
|
|
214
221
|
const result = validateRelativePath(path, label, cwd, true, { deferMissing: deferMissingReads });
|
|
215
|
-
if (result === DEFERRED_MISSING) deferredReads.push({ path, label });
|
|
222
|
+
if (result === DEFERRED_MISSING) deferredReads.push({ path, label, kind: "read" });
|
|
216
223
|
});
|
|
217
224
|
if (writeFiles !== undefined) /** @type {string[]} */ (writeFiles).forEach((path, pathIndex) => {
|
|
218
225
|
validateRelativePath(path, `nodes[${index}].taskPacket.writeFiles[${pathIndex}]`, cwd, false);
|
|
219
226
|
});
|
|
220
227
|
/** @type {string[]} */ (scopeAcknowledged).forEach((path, pathIndex) => {
|
|
221
|
-
|
|
228
|
+
const label = `nodes[${index}].taskPacket.scopeAcknowledged[${pathIndex}]`;
|
|
229
|
+
const result = validateRelativePath(path, label, cwd, true, { deferMissing: deferMissingReads });
|
|
230
|
+
if (result === DEFERRED_MISSING) deferredReads.push({ path, label, kind: "acknowledged" });
|
|
222
231
|
});
|
|
223
232
|
for (const [commandIndex, command] of verification.entries()) {
|
|
224
233
|
if (command.cwd !== undefined) {
|
|
@@ -426,6 +435,7 @@ function renderDiscoveryPrompt(packet, nodeId) {
|
|
|
426
435
|
'Return exactly one worker-result JSON object, with no markdown or prose. Set status to "done", missingContext to [], and artifacts to an array containing exactly one JSON-stringified execution task packet with every required taskPacket field. The packet readFiles and writeFiles must be non-empty and scoped to this repository.',
|
|
427
436
|
"",
|
|
428
437
|
"## Verification",
|
|
438
|
+
VERIFICATION_PARAGRAPH,
|
|
429
439
|
...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
|
|
430
440
|
];
|
|
431
441
|
const prompt = `${lines.join("\n")}\n`;
|
|
@@ -467,6 +477,7 @@ function renderAutonomousPrompt(packet, nodeId) {
|
|
|
467
477
|
...bulletOrNone(packet.nonGoals),
|
|
468
478
|
"",
|
|
469
479
|
"## Verification",
|
|
480
|
+
VERIFICATION_PARAGRAPH,
|
|
470
481
|
...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
|
|
471
482
|
"",
|
|
472
483
|
"## Required output",
|
|
@@ -38,7 +38,7 @@ export const VERIFICATION_LIMITS = Object.freeze({
|
|
|
38
38
|
/**
|
|
39
39
|
* Bounded evidence captured for one attempt.
|
|
40
40
|
*
|
|
41
|
-
* @typedef {{passed: boolean, stdout: string, stderr: string, error: string|null, exitCode: number|null, signal: string|null, timedOut: boolean, durationMs: number|null}} VerificationAttemptResult
|
|
41
|
+
* @typedef {{passed: boolean, stdout: string, stderr: string, error: string|null, exitCode: number|null, signal: string|null, timedOut: boolean, durationMs: number|null, signalDeath?: boolean}} VerificationAttemptResult
|
|
42
42
|
*/
|
|
43
43
|
|
|
44
44
|
/**
|
package/src/engine/backoff.mjs
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* third time buys nothing: change provider, or stop and say so.
|
|
10
10
|
*
|
|
11
11
|
* Everything here is a pure decision. Persisting it — the routing history,
|
|
12
|
-
* the override, the node transition — stays in
|
|
12
|
+
* the override, the node transition — stays in cli.mjs, so this module can
|
|
13
13
|
* be tested without a run directory, a lease, or a provider.
|
|
14
14
|
*/
|
|
15
15
|
import { nextHop, nextSynthesizedRuntime, synthesizedChain } from "./failover.mjs";
|
package/src/engine/dispatch.mjs
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* which made dispatch depend on review policy and on settlement, and that is
|
|
10
10
|
* the shape that kept `engine/` a web instead of a stack.
|
|
11
11
|
*/
|
|
12
|
-
import { JUDGE_SCHEMA, judgePrompt } from "./prompts.mjs";
|
|
12
|
+
import { JUDGE_SCHEMA, appendSandboxNotice, judgePrompt } from "./prompts.mjs";
|
|
13
13
|
import { LockLostError } from "../run/lock.mjs";
|
|
14
14
|
import { TOOL_OUTPUT_LIMIT_BYTES } from "../harnesses/exec-jsonl/index.mjs";
|
|
15
15
|
import { appendPreviousAttempt } from "./retry.mjs";
|
|
@@ -404,6 +404,8 @@ export function startWorker(contract, node, state, runDir, running, prompt, lock
|
|
|
404
404
|
// so it is appended to the resolved prompt rather than the candidate handed
|
|
405
405
|
// to phaseInvocationPlan.
|
|
406
406
|
phasePlan.prompt = appendPreviousAttempt(phasePlan.prompt, state.previousAttempt);
|
|
407
|
+
// The resolved worker's sandbox is a dispatch-time fact, not a packet one.
|
|
408
|
+
phasePlan.prompt = appendSandboxNotice(phasePlan.prompt, runtime);
|
|
407
409
|
// The worker prompt directs the provider to write the canonical result file;
|
|
408
410
|
// make sure the directory exists before the provider is asked to.
|
|
409
411
|
const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
|
|
@@ -526,11 +528,11 @@ export function startResultMaterialization(contract, node, state, runDir, runnin
|
|
|
526
528
|
const paths = logPaths(runDir, node.id, "worker", state.attempt);
|
|
527
529
|
const workspace = attemptWorkspace(state) ?? contract.cwd;
|
|
528
530
|
const resultPath = attemptWorkerResultPath(runDir, node.id, workspace);
|
|
529
|
-
const prompt = [
|
|
531
|
+
const prompt = appendSandboxNotice([
|
|
530
532
|
`${RESULT_MATERIALIZATION_PROMPT_HEADER} Do not inspect, implement, verify, or invoke tools.`,
|
|
531
533
|
`Your only job in this single bounded turn is to write the required worker-result JSON object to: ${resultPath}`,
|
|
532
534
|
"Then return that same JSON object as the final message.",
|
|
533
|
-
].join("\n\n");
|
|
535
|
+
].join("\n\n"), materializationRuntime);
|
|
534
536
|
let baseline;
|
|
535
537
|
try {
|
|
536
538
|
baseline = captureWorkspaceSnapshot(workspace);
|
package/src/engine/gate.mjs
CHANGED
|
@@ -21,8 +21,14 @@
|
|
|
21
21
|
*
|
|
22
22
|
* Nothing here is exported for a caller: this file is a program. Every name
|
|
23
23
|
* below is module-local.
|
|
24
|
+
*
|
|
25
|
+
* It also exits when the release file's directory is gone (measured
|
|
26
|
+
* 2026-09-16: gate processes from a prior day's test runs, spawned into a
|
|
27
|
+
* temp directory the failed test never cleaned up, were still alive and
|
|
28
|
+
* waiting for a release file that could now never appear).
|
|
24
29
|
*/
|
|
25
30
|
import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
|
|
31
|
+
import { dirname } from "node:path";
|
|
26
32
|
import { spawn } from "node:child_process";
|
|
27
33
|
|
|
28
34
|
/** @typedef {{executable: string, args: string[], cwd: string, promptTransport: "stdin"|"argv", harness: string, env: Record<string, string|null>|null, stdoutPath: string, stderrPath: string}} GateConfig */
|
|
@@ -158,8 +164,14 @@ function childEnv() {
|
|
|
158
164
|
process.on("SIGTERM", () => stopProvider());
|
|
159
165
|
process.on("SIGINT", () => stopProvider());
|
|
160
166
|
|
|
167
|
+
/** @returns {boolean} */
|
|
168
|
+
function releaseDirectoryGone() {
|
|
169
|
+
return !existsSync(dirname(releasePath));
|
|
170
|
+
}
|
|
171
|
+
|
|
161
172
|
const timer = setInterval(() => {
|
|
162
173
|
if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
|
|
174
|
+
if (releaseDirectoryGone()) { clearInterval(timer); stopProvider(); return; }
|
|
163
175
|
if (!existsSync(releasePath)) return;
|
|
164
176
|
clearInterval(timer);
|
|
165
177
|
const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
|
|
@@ -13,6 +13,41 @@ import { processStartToken } from "../run/lock.mjs";
|
|
|
13
13
|
/** @typedef {import("node:child_process").ChildProcess} ChildProcess */
|
|
14
14
|
/** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
|
|
15
15
|
|
|
16
|
+
/**
|
|
17
|
+
* Whether a controller may signal a pid's process group at all. It may not when
|
|
18
|
+
* the pid is the calling process -- whose group is the controller's own -- or
|
|
19
|
+
* its parent, whose group is the runner (or host scheduler) that spawned the
|
|
20
|
+
* controller: either signal takes down the caller instead of the invocation.
|
|
21
|
+
* The refusal is recorded and returned, never thrown.
|
|
22
|
+
*
|
|
23
|
+
* The line goes to the injected sink when a caller supplied one, otherwise to
|
|
24
|
+
* stderr; this function owns no run directory and no event-log schema, so
|
|
25
|
+
* stderr is the one recorder it always has. Recording is diagnostic, so a
|
|
26
|
+
* failure to record must never turn the refusal into a throw.
|
|
27
|
+
*
|
|
28
|
+
* @param {number} pid
|
|
29
|
+
* @param {((event: Record<string, unknown>) => void)|undefined} [append] injected event sink
|
|
30
|
+
* @returns {boolean} true when the pid is this process or its parent and the signal was refused
|
|
31
|
+
*/
|
|
32
|
+
export function refuseSelfSignal(pid, append) {
|
|
33
|
+
const relation = pid === process.pid ? "self" : pid === process.ppid ? "parent" : null;
|
|
34
|
+
if (relation === null) return false;
|
|
35
|
+
const event = {
|
|
36
|
+
type: "controller_self_signal_refused",
|
|
37
|
+
at: new Date().toISOString(),
|
|
38
|
+
pid,
|
|
39
|
+
processPid: process.pid,
|
|
40
|
+
relation,
|
|
41
|
+
};
|
|
42
|
+
try {
|
|
43
|
+
if (append) append(event);
|
|
44
|
+
else process.stderr.write(`[warn] controller_self_signal_refused ${JSON.stringify(event)}\n`);
|
|
45
|
+
} catch {
|
|
46
|
+
// The refusal is diagnostic: a failed recorder must never become a throw.
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
|
|
16
51
|
/**
|
|
17
52
|
* A process group answers a signal-0 probe. EPERM means the group exists but is
|
|
18
53
|
* not this user's, which is no more "ours" than a missing group.
|
|
@@ -57,6 +92,10 @@ export function processStartTokenMatches(invocation) {
|
|
|
57
92
|
*/
|
|
58
93
|
export function invocationOwned(invocation, options = {}) {
|
|
59
94
|
if (!invocation?.pid || !Number.isInteger(invocation.pid)) return false;
|
|
95
|
+
// A controller does not spawn itself or its parent: a recorded pid that names
|
|
96
|
+
// either can never be an invocation this controller owns, whatever token or
|
|
97
|
+
// child handle accompanies it, and signalling it would hit the caller.
|
|
98
|
+
if (invocation.pid === process.pid || invocation.pid === process.ppid) return false;
|
|
60
99
|
try {
|
|
61
100
|
process.kill(invocation.pid, 0);
|
|
62
101
|
} catch {
|
package/src/engine/prompts.mjs
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { validateWorkerResult } from "../contract/worker-result.mjs";
|
|
2
2
|
import { scopeFindingsPromptSection } from "../contract/scope-findings.mjs";
|
|
3
3
|
import { JUDGE_ENVELOPE_REASON, JUDGE_FINDING_ENVELOPE_REASON, JUDGE_LIMITS } from "../contract/judge-envelope.mjs";
|
|
4
|
+
import { harnessCapabilities } from "../harnesses/index.mjs";
|
|
4
5
|
|
|
5
6
|
/** @typedef {{id: string, definitionOfDone: import("../contract/definition-of-done.mjs").DefinitionOfDoneItem[], taskPacket: {mode?: "execution"|"discovery"|"autonomous", objective: string, instructions: string[], writeFiles?: string[], writeRoots?: string[], verification: {argv: string[]}[]}}} JudgeNode */
|
|
6
7
|
/** @typedef {{verdict: "pass"|"fail"|"invalid_judge_output", maxSeverity: "none"|"minor"|"major"|"critical", summary: string, findings: {severity: "minor"|"major"|"critical", description: string, evidence: string}[]}} JudgeVerdict */
|
|
@@ -276,6 +277,23 @@ export function retryPrompt(node, verdict) {
|
|
|
276
277
|
: prompt;
|
|
277
278
|
}
|
|
278
279
|
|
|
280
|
+
/**
|
|
281
|
+
* Append the `## Sandbox` warning a resolved worker runtime earns. Only a
|
|
282
|
+
* harness whose adapter declares `signalsProcesses === false` gets it: that
|
|
283
|
+
* sandbox cannot signal child processes or read the process table, so a test
|
|
284
|
+
* that starts and terminates a child hangs until the executor's cap. An
|
|
285
|
+
* unmeasured harness (`null`) is left alone — no measurement justifies the
|
|
286
|
+
* warning.
|
|
287
|
+
*
|
|
288
|
+
* @param {string} prompt
|
|
289
|
+
* @param {{harness: string}} runtime
|
|
290
|
+
* @returns {string}
|
|
291
|
+
*/
|
|
292
|
+
export function appendSandboxNotice(prompt, runtime) {
|
|
293
|
+
if (harnessCapabilities(runtime).signalsProcesses !== false) return prompt;
|
|
294
|
+
return `${prompt}\n\n## Sandbox\nYour harness runs you in a sandbox that cannot signal other processes or read the process table. A test that starts and terminates a child process hangs here until the executor's cap and is then killed as a stall. Do not run such tests; the controller runs them after you report.`;
|
|
295
|
+
}
|
|
296
|
+
|
|
279
297
|
/**
|
|
280
298
|
* @param {unknown} value
|
|
281
299
|
* @param {number} maxBytes
|
package/src/engine/resume.mjs
CHANGED
|
@@ -17,7 +17,7 @@ import { Buffer } from "node:buffer";
|
|
|
17
17
|
import { acquire as acquireLock } from "../run/lock.mjs";
|
|
18
18
|
import { applyInvalidWorkerResult, assertRunMutable, handleProviderExhaustion } from "./lifecycle.mjs";
|
|
19
19
|
import { applyJudgeResult } from "./review.mjs";
|
|
20
|
-
import { assertSourceUnchanged, captureRunIdentity, statesFingerprint } from "./run-identity.mjs";
|
|
20
|
+
import { assertSourceUnchanged, captureRunIdentity, recordedBaseRef, statesFingerprint } from "./run-identity.mjs";
|
|
21
21
|
import { attemptWorkspace, attemptWorktreePath, gitHead, removeWorktree, runRefName } from "../repo/worktree.mjs";
|
|
22
22
|
import { canonicalWorkerResultText, isResultMaterializationInvocation, materializeAttemptResult, recoverWorkerResult } from "./result-file.mjs";
|
|
23
23
|
import { checkPersistedWorkerScope, persistedScopeBoundary, reconcileAmbiguousWorkerRestart, resolveUnknownEffect } from "./scope.mjs";
|
|
@@ -114,7 +114,7 @@ export async function resumeRun(runDirPath, options = {}) {
|
|
|
114
114
|
node.id,
|
|
115
115
|
persistedScopeBoundary(contract, node, states.get(node.id), attemptWorkspace(states.get(node.id)) ?? contract.cwd),
|
|
116
116
|
]));
|
|
117
|
-
const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries);
|
|
117
|
+
const sourceIdentity = await captureRunIdentity(contract, scopeBoundaries, recordedBaseRef(storedMetadata) ?? undefined);
|
|
118
118
|
const identity = assertSourceUnchanged(storedMetadata.sourceIdentity, sourceIdentity);
|
|
119
119
|
// A resume is an explicit instruction to continue the run: it consumes a
|
|
120
120
|
// stale cancel request instead of letting it re-cancel the retried nodes.
|
package/src/engine/review.mjs
CHANGED
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
} from "../contract/review-modes.mjs";
|
|
22
22
|
|
|
23
23
|
import { errorMessage, excerpt } from "../util.mjs";
|
|
24
|
-
import { appendTransitionEvent, transition } from "./state.mjs";
|
|
24
|
+
import { appendTransitionEvent, transition, writeNode } from "./state.mjs";
|
|
25
25
|
import { startJudge } from "./dispatch.mjs";
|
|
26
26
|
import { applyRejection, raiseNodeAttention, settleDone } from "./settle.mjs";
|
|
27
27
|
|
|
@@ -154,6 +154,14 @@ export async function applyJudgeResult(contract, node, state, result, runDir, lo
|
|
|
154
154
|
await raiseNodeAttention(campaignPath, runDir, state, "judge_protocol");
|
|
155
155
|
return;
|
|
156
156
|
}
|
|
157
|
+
// The verdict and the spent bound are durable before the blocked
|
|
158
|
+
// transition: a controller loss in this gap reads the settled re-ask from
|
|
159
|
+
// the node snapshot instead of spending a second one. No await separates
|
|
160
|
+
// the two writes, so the durable record cannot be observed only by luck.
|
|
161
|
+
writeNode(runDir, state, lock);
|
|
162
|
+
if (process.env.FABERUN_JUDGE_REASK_INTERRUPT === "after-verdict") {
|
|
163
|
+
throw new Error("judge re-ask interrupted after verdict persistence");
|
|
164
|
+
}
|
|
157
165
|
transition(runDir, state, "blocked", {
|
|
158
166
|
phase: "judge",
|
|
159
167
|
gate: verdict,
|
|
@@ -99,9 +99,30 @@ async function runRepeatedCommand(command, baseCwd, commandIndex, options) {
|
|
|
99
99
|
const attempts = [];
|
|
100
100
|
const repeat = command.repeat ?? 1;
|
|
101
101
|
for (let attempt = 1; attempt <= repeat; attempt += 1) {
|
|
102
|
-
|
|
102
|
+
const result = await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex);
|
|
103
|
+
if (signalDeathRetry(result, options.signal)) {
|
|
104
|
+
attempts.push({ ...result, signalDeath: true });
|
|
105
|
+
attempts.push(await runCommand(command, baseCwd, command.cwd ?? ".", attempt, options.signal, options, commandIndex));
|
|
106
|
+
} else {
|
|
107
|
+
attempts.push(result);
|
|
108
|
+
}
|
|
103
109
|
}
|
|
104
|
-
|
|
110
|
+
const passed = attempts.filter((item) => !item.signalDeath).every((item) => item.passed);
|
|
111
|
+
return { ...command, cwd: resolveVerificationCwd(baseCwd, command.cwd ?? "."), passed, attempts };
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Whether an attempt died from a signal the controller itself did not send.
|
|
115
|
+
* `terminateGroup` only fires from the timeout and abort paths below, so a
|
|
116
|
+
* `signal` with neither set is evidence of an external kill (OOM, an operator
|
|
117
|
+
* `kill`, a flaky sandbox) rather than a verdict on the command under test,
|
|
118
|
+
* and gets one retry instead of failing the node outright.
|
|
119
|
+
*
|
|
120
|
+
* @param {VerificationAttemptResult} result
|
|
121
|
+
* @param {AbortSignal|undefined} signal
|
|
122
|
+
* @returns {boolean}
|
|
123
|
+
*/
|
|
124
|
+
function signalDeathRetry(result, signal) {
|
|
125
|
+
return Boolean(result.signal) && !result.timedOut && !signal?.aborted;
|
|
105
126
|
}
|
|
106
127
|
/**
|
|
107
128
|
* The mutation case: the entry passes on the mutant-kill fraction, and every
|
|
@@ -181,6 +181,20 @@ export function setLaunchBaseRef(baseRef) {
|
|
|
181
181
|
pendingLaunchBaseRef = baseRef ?? null;
|
|
182
182
|
}
|
|
183
183
|
|
|
184
|
+
/**
|
|
185
|
+
* The base ref a run was launched against, when it was launched with
|
|
186
|
+
* `--base-ref`. A run recorded before this field existed, or launched
|
|
187
|
+
* without the flag, has none — a resume of that run keeps comparing against
|
|
188
|
+
* the checkout's own HEAD.
|
|
189
|
+
*
|
|
190
|
+
* @param {RunMetadata|undefined} metadata
|
|
191
|
+
* @returns {string|null}
|
|
192
|
+
*/
|
|
193
|
+
export function recordedBaseRef(metadata) {
|
|
194
|
+
const baseRef = metadata?.sourceIdentity?.baseRef;
|
|
195
|
+
return typeof baseRef === "string" && baseRef.length > 0 ? baseRef : null;
|
|
196
|
+
}
|
|
197
|
+
|
|
184
198
|
/**
|
|
185
199
|
* Capture the run's source identity, including one version-only probe per
|
|
186
200
|
* distinct routed runtime (a local binary call, no model tokens) so a later
|
package/src/engine/scheduler.mjs
CHANGED
|
@@ -41,7 +41,7 @@ import { appendUsageRecord, invocationCost, invocationUsage, recordInvocationUsa
|
|
|
41
41
|
import { captureNodeScopeBoundaries, checkWorkerScope, emptyScope } from "./scope.mjs";
|
|
42
42
|
import { validateContract } from "../contract/index.mjs";
|
|
43
43
|
import { validateNodeSnapshot } from "../contract/snapshot.mjs";
|
|
44
|
-
import { finalVerificationCommands } from "../contract/final-verification.mjs";
|
|
44
|
+
import { finalVerificationCommands, sharedVerificationCommands } from "../contract/final-verification.mjs";
|
|
45
45
|
import { startJudge, startWorker } from "./dispatch.mjs";
|
|
46
46
|
import { assertEnvironmentReady, captureRunIdentity, createRunMetadata, serializableContract, statesFingerprint } from "./run-identity.mjs";
|
|
47
47
|
import { blockDependents, runtimeAssignments } from "./assignment.mjs";
|
|
@@ -94,12 +94,13 @@ export function verificationBudgetMs(commands) {
|
|
|
94
94
|
/**
|
|
95
95
|
* The budget a node is judged against, in milliseconds. It is the sum of every
|
|
96
96
|
* bounded phase the node can legitimately occupy without a state transition:
|
|
97
|
-
* its worker invocation (`timeoutSec`), its packet verification
|
|
98
|
-
* contract's `
|
|
99
|
-
* candidate run of that same
|
|
100
|
-
*
|
|
101
|
-
* longer than
|
|
102
|
-
* minutes long and must not be
|
|
97
|
+
* its worker invocation (`timeoutSec`), its packet verification plus the
|
|
98
|
+
* contract's `sharedVerification` and, when it is phase-terminal, the
|
|
99
|
+
* contract's `finalVerification`, an integration candidate run of that same
|
|
100
|
+
* set, and the bounded command proofs of its gate. A frozen node is one that
|
|
101
|
+
* has been silent longer than this, not merely longer than the worker timeout,
|
|
102
|
+
* because a legitimate verification can be minutes long and must not be
|
|
103
|
+
* mistaken for a freeze.
|
|
103
104
|
*
|
|
104
105
|
* @param {ValidatedContract} contract
|
|
105
106
|
* @param {ValidatedNode} node
|
|
@@ -107,14 +108,17 @@ export function verificationBudgetMs(commands) {
|
|
|
107
108
|
*/
|
|
108
109
|
export function nodeBudgetBasisMs(contract, node) {
|
|
109
110
|
const defaultTimeoutMs = (node.timeoutSec ?? contract.timeoutSec ?? 60) * 1_000;
|
|
111
|
+
const sharedMs = verificationBudgetMs(sharedVerificationCommands(contract));
|
|
110
112
|
const packetMs = verificationBudgetMs(node.taskPacket?.verification);
|
|
111
113
|
const finalMs = verificationBudgetMs(finalVerificationCommands(contract, node));
|
|
112
|
-
// The controller runs the packet set
|
|
113
|
-
// the integration candidate, and the
|
|
114
|
-
|
|
114
|
+
// The controller runs the packet set (with the contract-level shared set) once
|
|
115
|
+
// after the worker and once against the integration candidate, and the
|
|
116
|
+
// finalVerification set with each.
|
|
117
|
+
const attemptMs = packetMs + sharedMs;
|
|
118
|
+
const candidateMs = attemptMs + finalMs;
|
|
115
119
|
const gateTimeoutMs = Math.max(1_000, Math.min(defaultTimeoutMs, 120_000));
|
|
116
120
|
const commandProofs = (node.definitionOfDone ?? []).filter((item) => item.proof?.kind === "command").length;
|
|
117
|
-
return defaultTimeoutMs +
|
|
121
|
+
return defaultTimeoutMs + attemptMs + candidateMs + commandProofs * gateTimeoutMs + finalMs;
|
|
118
122
|
}
|
|
119
123
|
|
|
120
124
|
/**
|
|
@@ -149,6 +153,26 @@ export function enforceRunningInvariant(runDir, states, running, lock) {
|
|
|
149
153
|
return parked;
|
|
150
154
|
}
|
|
151
155
|
|
|
156
|
+
/**
|
|
157
|
+
* The fingerprint that decides whether a render is due: the shared
|
|
158
|
+
* `statesFingerprint` (status, phase, attempt, revisions) plus each running
|
|
159
|
+
* node's verification progress. A verification command completing and the
|
|
160
|
+
* next one starting changes neither status nor phase, so the shared
|
|
161
|
+
* fingerprint alone would never notice it; this is the one used to gate
|
|
162
|
+
* `renderStatusIfChanged` and nothing else, so a resume's drift comparison
|
|
163
|
+
* still uses the unextended `statesFingerprint`.
|
|
164
|
+
*
|
|
165
|
+
* @param {Map<string, NodeSnapshot>} states
|
|
166
|
+
* @returns {string}
|
|
167
|
+
*/
|
|
168
|
+
function renderFingerprint(states) {
|
|
169
|
+
const progress = [...states.values()].map((state) => {
|
|
170
|
+
const verification = /** @type {{progress?: {index: number, total: number, argv: string}}|null|undefined} */ (state.verification);
|
|
171
|
+
return verification?.progress ? `${state.id}:${verification.progress.index}/${verification.progress.total}:${verification.progress.argv}` : "";
|
|
172
|
+
}).join("|");
|
|
173
|
+
return `${statesFingerprint(states)}#${progress}`;
|
|
174
|
+
}
|
|
175
|
+
|
|
152
176
|
/**
|
|
153
177
|
* @param {string} contractPath
|
|
154
178
|
* @param {{detachedBootstrap?: boolean}} [options] `detachedBootstrap` is set
|
|
@@ -290,11 +314,19 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
290
314
|
let statusFingerprint = null;
|
|
291
315
|
/** @param {boolean} force @param {LockHandle|null} [renderLock] */
|
|
292
316
|
const renderStatusIfChanged = (force = false, renderLock = lock) => {
|
|
293
|
-
const fingerprint =
|
|
317
|
+
const fingerprint = renderFingerprint(states);
|
|
294
318
|
if (!force && fingerprint === statusFingerprint) return;
|
|
295
319
|
statusFingerprint = fingerprint;
|
|
296
320
|
render(runDir, runsDir, contract, states, renderLock);
|
|
297
321
|
};
|
|
322
|
+
// The tick that owns a running node's verification can be minutes long
|
|
323
|
+
// (`executeControllerVerification` is awaited on the critical path below),
|
|
324
|
+
// and that whole time status.json would otherwise report whatever the last
|
|
325
|
+
// tick left it at. A timer renders between ticks too; it is cheap even when
|
|
326
|
+
// idle because `renderFingerprint` still change-detects, so a quiet run
|
|
327
|
+
// writes nothing extra.
|
|
328
|
+
const statusTimer = setInterval(() => renderStatusIfChanged(), contract.pollIntervalMs);
|
|
329
|
+
statusTimer.unref();
|
|
298
330
|
let handoffFingerprint = statesFingerprint(states);
|
|
299
331
|
const renderHandoffIfChanged = () => {
|
|
300
332
|
const fingerprint = statesFingerprint(states);
|
|
@@ -475,6 +507,7 @@ export async function driveRun(contract, runDir, states, campaign, lock, sourceI
|
|
|
475
507
|
notifyQueuesByRun.delete(runDir);
|
|
476
508
|
return { runDir, states, ok: false, error };
|
|
477
509
|
} finally {
|
|
510
|
+
clearInterval(statusTimer);
|
|
478
511
|
process.removeListener("SIGINT", cancel);
|
|
479
512
|
process.removeListener("SIGTERM", cancel);
|
|
480
513
|
process.removeListener("SIGHUP", cancel);
|