faberun 0.19.2 → 0.19.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "faberun",
3
- "version": "0.19.2",
3
+ "version": "0.19.3",
4
4
  "description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -0,0 +1,197 @@
1
+ /**
2
+ * `campaign watch --wake`: the loop that turns a campaign's own records into
3
+ * the lines an operator is woken with, and the single-watcher lock that keeps
4
+ * two of them from doubling every wake.
5
+ *
6
+ * Separate from `cli/campaign.mjs` because that file owns argv, dispatch and
7
+ * usage for the campaign verb, and this is neither: it is a long-running
8
+ * supervisor with a poll interval, an idle policy, a durable dedupe through
9
+ * the inbox, and a lock with its own staleness rule. `cli/campaign.mjs` keeps
10
+ * the thin `watch` operation that reads the flags and calls in here.
11
+ */
12
+ import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
13
+ import { join } from "node:path";
14
+ import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
15
+ import { readCampaign } from "./record.mjs";
16
+ import { notifyQueueFor } from "../engine/notify-queue.mjs";
17
+ import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
18
+ import { errorCode, readJsonTolerant } from "../util.mjs";
19
+
20
+ /** How often the watcher re-reads the campaign when the operator named no interval. */
21
+ export const DEFAULT_WAKE_POLL_MS = 30_000;
22
+
23
+ /** How long without a material event before the watcher reports the campaign idle. */
24
+ const WAKE_IDLE_AFTER_MS = 20 * 60_000;
25
+
26
+ const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
27
+ const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
28
+
29
+ /**
30
+ * The watcher loop. Each line is announced through `notify`, which by default
31
+ * records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
32
+ * `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
33
+ * line already recorded is never re-sent. The injectable seams exist so a
34
+ * test can drive the loop deterministically.
35
+ *
36
+ * @param {string} campaignPath
37
+ * @param {string} runsDir
38
+ * @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
39
+ * @returns {Promise<void>}
40
+ */
41
+ export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
42
+ const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
43
+ const now = options.now ?? (() => Date.now());
44
+ const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
45
+ const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
46
+ const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
47
+ type: "attention",
48
+ campaignId: event.campaignId,
49
+ dedupeKey: event.dedupeKey,
50
+ summary: event.summary,
51
+ runId: event.runId ?? null,
52
+ nodeId: event.nodeId ?? null,
53
+ status: event.status ?? null,
54
+ errorCode: event.errorCode ?? null,
55
+ }));
56
+ const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
57
+ const lock = options.lock ?? acquireWatchLock(campaignPath);
58
+ emit(wakeCapabilityNotice());
59
+ /** @type {Map<string, string>} */
60
+ const runSignatures = new Map();
61
+ let lastActiveAt = now();
62
+ let first = true;
63
+ try {
64
+ for (;;) {
65
+ const campaign = readCampaign(campaignPath);
66
+ if (campaign.status !== "active") {
67
+ emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
68
+ return;
69
+ }
70
+ /**
71
+ * Persist first, deliver second: the inbox entry is the durable dedupe,
72
+ * so a restart or a second watcher skips a line already recorded even
73
+ * when delivery is injected.
74
+ *
75
+ * @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
76
+ */
77
+ const announce = async (dedupeKey, summary, extra = {}) => {
78
+ if (seen.has(dedupeKey)) return;
79
+ seen.add(dedupeKey);
80
+ const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
81
+ if (!appended.appended) return;
82
+ emit(summary);
83
+ await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
84
+ };
85
+ let anyActive = false;
86
+ for (const runId of campaign.linkedRunIds) {
87
+ const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
88
+ if (!status || !Array.isArray(status.nodes)) continue;
89
+ const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
90
+ const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
91
+ const previous = runSignatures.get(runId);
92
+ runSignatures.set(runId, signature);
93
+ if (!terminal) {
94
+ anyActive = true;
95
+ const runLock = readLock(join(runsDir, runId));
96
+ const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
97
+ if (stale && !first) {
98
+ await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
99
+ }
100
+ }
101
+ if (!first && previous !== signature) {
102
+ for (const node of status.nodes) {
103
+ const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
104
+ || (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
105
+ if (attention) {
106
+ const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
107
+ await announce(
108
+ key,
109
+ `campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
110
+ { runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
111
+ );
112
+ }
113
+ }
114
+ }
115
+ if (terminal) {
116
+ await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
117
+ }
118
+ }
119
+ const nowMs = now();
120
+ if (anyActive) lastActiveAt = nowMs;
121
+ else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
122
+ const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
123
+ await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
124
+ }
125
+ first = false;
126
+ if (options.once === true) return;
127
+ await sleep(pollMs);
128
+ }
129
+ } finally {
130
+ lock.release();
131
+ }
132
+ }
133
+
134
+ const WATCH_LOCK_FILE = "watch.lock";
135
+
136
+ /**
137
+ * A durable campaign-watch lock, one watcher per campaign across processes.
138
+ * A live holder is never taken over; a dead or recycled pid's lock is stale
139
+ * and is replaced, so a restart after a crash is not blocked. The same
140
+ * liveness rule as the controller lock: a pid is dead only when the probe
141
+ * proves it.
142
+ *
143
+ * @param {string} campaignPath
144
+ * @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
145
+ */
146
+ export function acquireWatchLock(campaignPath) {
147
+ const path = join(campaignPath, WATCH_LOCK_FILE);
148
+ /** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
149
+ let occupant = {};
150
+ for (let attempt = 0; attempt < 20; attempt += 1) {
151
+ const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
152
+ try {
153
+ const fd = openSync(path, "wx", 0o600);
154
+ try {
155
+ writeSync(fd, JSON.stringify(record));
156
+ } finally {
157
+ closeSync(fd);
158
+ }
159
+ return {
160
+ ...record,
161
+ release() {
162
+ try {
163
+ unlinkSync(path);
164
+ } catch (error) {
165
+ if (errorCode(error) !== "ENOENT") throw error;
166
+ }
167
+ },
168
+ };
169
+ } catch (error) {
170
+ if (errorCode(error) !== "EEXIST") throw error;
171
+ }
172
+ try {
173
+ occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
174
+ } catch {
175
+ occupant = {};
176
+ }
177
+ if (!watchLockStale(occupant)) {
178
+ throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
179
+ }
180
+ try {
181
+ unlinkSync(path);
182
+ } catch (error) {
183
+ if (errorCode(error) !== "ENOENT") throw error;
184
+ }
185
+ }
186
+ throw new Error("campaign watch lock contention did not settle");
187
+ }
188
+
189
+ /**
190
+ * @param {{pid?: number, processStartToken?: string|null}} occupant
191
+ * @returns {boolean}
192
+ */
193
+ function watchLockStale(occupant) {
194
+ if (typeof occupant.pid !== "number") return true;
195
+ if (!pidAlive(occupant.pid)) return true;
196
+ return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
197
+ }
@@ -1,5 +1,5 @@
1
1
  import { randomUUID } from "node:crypto";
2
- import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
2
+ import { readFileSync } from "node:fs";
3
3
  import { join, resolve } from "node:path";
4
4
  import { parseArgs as parseFlags } from "node:util";
5
5
  import {
@@ -11,25 +11,19 @@ import {
11
11
  resolveCampaign,
12
12
  } from "../campaign/index.mjs";
13
13
  import { addContract, replaceContract } from "./campaign-contract.mjs";
14
- import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
15
14
  import { runsRoot } from "../run/paths.mjs";
16
15
  import { syncAgentSignal } from "../repo/signal.mjs";
17
16
  import { acknowledgeJournalEvent, appendJournal, appendSeatAllowanceEvent, readJournal, watchJournal } from "../campaign/journal.mjs";
18
17
  import { driveCampaignChain } from "../campaign/chain.mjs";
19
18
  import { unparkCampaign } from "../campaign/unpark.mjs";
20
19
  import { readCampaign } from "../campaign/record.mjs";
21
- import { notifyQueueFor } from "../engine/notify-queue.mjs";
22
- import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
20
+ import { DEFAULT_WAKE_POLL_MS, watchCampaignWake } from "../campaign/watch.mjs";
23
21
  import { allowanceEventFields, sampleAllowance } from "../seat/allowance.mjs";
24
22
  import { detectOperatorHarness } from "../seat/harnesses.mjs";
25
23
  import { detachArgv, detachSelf, waitForBootstrap } from "./launch.mjs";
26
- import { errorCode, readJsonTolerant } from "../util.mjs";
24
+ import { readJsonTolerant } from "../util.mjs";
27
25
 
28
26
  const SYNC_OUTPUT_MAX_BYTES = 8000;
29
- const DEFAULT_WAKE_POLL_MS = 30_000;
30
- const WAKE_IDLE_AFTER_MS = 20 * 60_000;
31
- const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
32
- const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
33
27
 
34
28
  const NOTE_KINDS = new Set([
35
29
  "intent",
@@ -159,177 +153,6 @@ async function watch(campaignId, values) {
159
153
  await watchCampaignWake(path, runsDir, { pollMs, once: values.once === true });
160
154
  }
161
155
 
162
- /**
163
- * The watcher loop. Each line is announced through `notify`, which by default
164
- * records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
165
- * `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
166
- * line already recorded is never re-sent. The injectable seams exist so a
167
- * test can drive the loop deterministically.
168
- *
169
- * @param {string} campaignPath
170
- * @param {string} runsDir
171
- * @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
172
- * @returns {Promise<void>}
173
- */
174
- export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
175
- const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
176
- const now = options.now ?? (() => Date.now());
177
- const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
178
- const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
179
- const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
180
- type: "attention",
181
- campaignId: event.campaignId,
182
- dedupeKey: event.dedupeKey,
183
- summary: event.summary,
184
- runId: event.runId ?? null,
185
- nodeId: event.nodeId ?? null,
186
- status: event.status ?? null,
187
- errorCode: event.errorCode ?? null,
188
- }));
189
- const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
190
- const lock = options.lock ?? acquireWatchLock(campaignPath);
191
- emit(wakeCapabilityNotice());
192
- /** @type {Map<string, string>} */
193
- const runSignatures = new Map();
194
- let lastActiveAt = now();
195
- let first = true;
196
- try {
197
- for (;;) {
198
- const campaign = readCampaign(campaignPath);
199
- if (campaign.status !== "active") {
200
- emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
201
- return;
202
- }
203
- /**
204
- * Persist first, deliver second: the inbox entry is the durable dedupe,
205
- * so a restart or a second watcher skips a line already recorded even
206
- * when delivery is injected.
207
- *
208
- * @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
209
- */
210
- const announce = async (dedupeKey, summary, extra = {}) => {
211
- if (seen.has(dedupeKey)) return;
212
- seen.add(dedupeKey);
213
- const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
214
- if (!appended.appended) return;
215
- emit(summary);
216
- await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
217
- };
218
- let anyActive = false;
219
- for (const runId of campaign.linkedRunIds) {
220
- const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
221
- if (!status || !Array.isArray(status.nodes)) continue;
222
- const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
223
- const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
224
- const previous = runSignatures.get(runId);
225
- runSignatures.set(runId, signature);
226
- if (!terminal) {
227
- anyActive = true;
228
- const runLock = readLock(join(runsDir, runId));
229
- const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
230
- if (stale && !first) {
231
- await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
232
- }
233
- }
234
- if (!first && previous !== signature) {
235
- for (const node of status.nodes) {
236
- const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
237
- || (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
238
- if (attention) {
239
- const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
240
- await announce(
241
- key,
242
- `campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
243
- { runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
244
- );
245
- }
246
- }
247
- }
248
- if (terminal) {
249
- await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
250
- }
251
- }
252
- const nowMs = now();
253
- if (anyActive) lastActiveAt = nowMs;
254
- else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
255
- const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
256
- await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
257
- }
258
- first = false;
259
- if (options.once === true) return;
260
- await sleep(pollMs);
261
- }
262
- } finally {
263
- lock.release();
264
- }
265
- }
266
-
267
- const WATCH_LOCK_FILE = "watch.lock";
268
-
269
- /**
270
- * A durable campaign-watch lock, one watcher per campaign across processes.
271
- * A live holder is never taken over; a dead or recycled pid's lock is stale
272
- * and is replaced, so a restart after a crash is not blocked. The same
273
- * liveness rule as the controller lock: a pid is dead only when the probe
274
- * proves it.
275
- *
276
- * @param {string} campaignPath
277
- * @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
278
- */
279
- export function acquireWatchLock(campaignPath) {
280
- const path = join(campaignPath, WATCH_LOCK_FILE);
281
- /** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
282
- let occupant = {};
283
- for (let attempt = 0; attempt < 20; attempt += 1) {
284
- const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
285
- try {
286
- const fd = openSync(path, "wx", 0o600);
287
- try {
288
- writeSync(fd, JSON.stringify(record));
289
- } finally {
290
- closeSync(fd);
291
- }
292
- return {
293
- ...record,
294
- release() {
295
- try {
296
- unlinkSync(path);
297
- } catch (error) {
298
- if (errorCode(error) !== "ENOENT") throw error;
299
- }
300
- },
301
- };
302
- } catch (error) {
303
- if (errorCode(error) !== "EEXIST") throw error;
304
- }
305
- try {
306
- occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
307
- } catch {
308
- occupant = {};
309
- }
310
- if (!watchLockStale(occupant)) {
311
- throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
312
- }
313
- try {
314
- unlinkSync(path);
315
- } catch (error) {
316
- if (errorCode(error) !== "ENOENT") throw error;
317
- }
318
- }
319
- throw new Error("campaign watch lock contention did not settle");
320
- }
321
-
322
- /**
323
- * @param {{pid?: number, processStartToken?: string|null}} occupant
324
- * @returns {boolean}
325
- */
326
- function watchLockStale(occupant) {
327
- if (typeof occupant.pid !== "number") return true;
328
- if (!pidAlive(occupant.pid)) return true;
329
- return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
330
- }
331
-
332
-
333
156
  /**
334
157
  * @param {string} campaignId
335
158
  * @param {CliValues} values
@@ -467,9 +290,38 @@ function close(campaignId, values) {
467
290
  renderHandoff(path, runsDir);
468
291
  process.stdout.write(`[campaign] ${closed.campaign.id} closed\n`);
469
292
  process.stdout.write(`[campaign] ledger · docs/campaigns/${closed.campaign.id}/ledger · ${closed.ledgerFiles.length} files\n`);
293
+ reportRequirementClosure(closed.campaign.requirements ?? []);
470
294
  if (syncAgentSignal(runsDir)) process.stdout.write(`[campaign] AGENTS.md signal updated\n`);
471
295
  }
472
296
 
297
+ /**
298
+ * Say out loud what the close just wrote, and what it measured.
299
+ *
300
+ * The record is a correlation between the requirement ids a linked run's
301
+ * contract declared and the node snapshots that reached `done` carrying them.
302
+ * It reads the run directory and never the branch, so a requirement the
303
+ * operator delivered by hand -- because its node blocked, which happens --
304
+ * lands in the record as `open` while the work is on main. Measured
305
+ * 2026-09-22 on `availability-is-verified-not-assumed`: R4 read `open` with
306
+ * an empty `nodes` array and was merged in `a6a5d43`.
307
+ *
308
+ * Nothing printed this before, so the only way to meet the closure was to
309
+ * open the JSON, where `open` reads as "not delivered" instead of "no node of
310
+ * this campaign proved it". Naming the measurement is the whole fix: a close
311
+ * cannot know what it did not run.
312
+ *
313
+ * @param {import("../campaign/index.mjs").RequirementClosure[]} requirements
314
+ */
315
+ function reportRequirementClosure(requirements) {
316
+ if (requirements.length === 0) return;
317
+ const open = requirements.filter((entry) => entry.status !== "covered");
318
+ const covered = requirements.length - open.length;
319
+ process.stdout.write(`[campaign] requirements · ${covered}/${requirements.length} carried by a node that reached done\n`);
320
+ if (open.length === 0) return;
321
+ process.stdout.write(`[campaign] no done node carried ${open.map((entry) => entry.requirementId).join(", ")}\n`);
322
+ process.stdout.write("[campaign] closure reads node snapshots, never the branch: a requirement delivered outside a node reads open here\n");
323
+ }
324
+
473
325
  /**
474
326
  * `campaign supervise <id>` (also spelled `supervise campaign <id>`): the
475
327
  * idempotent re-invocation that drives the manifest. It takes the campaign's
@@ -55,8 +55,14 @@ export function detachSelf(command, target, extraArgs = []) {
55
55
  * exactly as for every detached controller: a foreground launcher is the only
56
56
  * moment an operator is present.
57
57
  *
58
+ * `stdio` overrides the discarded default for a caller that has somewhere
59
+ * durable to put the child's output. A detached *run* wants the default: its
60
+ * bootstrap record is the channel, and a controller's real output belongs in
61
+ * the run directory. A detached *plan* has no bootstrap record, so discarding
62
+ * its stdio discarded the only account of why it died.
63
+ *
58
64
  * @param {string[]} argv
59
- * @param {{nonce?: string, env?: NodeJS.ProcessEnv}} [options]
65
+ * @param {{nonce?: string, env?: NodeJS.ProcessEnv, stdio?: import("node:child_process").StdioOptions}} [options]
60
66
  * @returns {DetachedChild}
61
67
  */
62
68
  export function detachArgv(argv, options = {}) {
@@ -74,7 +80,7 @@ export function detachArgv(argv, options = {}) {
74
80
  // pending forever. There it also means DETACHED_PROCESS: no console
75
81
  // window, which is what "ignore" stdio already implies.
76
82
  detached: true,
77
- stdio: "ignore",
83
+ stdio: options.stdio ?? "ignore",
78
84
  }));
79
85
  child.unref();
80
86
  child.bootstrapNonce = nonce;
package/src/cli/plan.mjs CHANGED
@@ -16,6 +16,7 @@ import { colorLevel, statusToken } from "./brand.mjs";
16
16
  import { delay } from "../util.mjs";
17
17
  import { runPlanningPipeline } from "../plan/pipeline.mjs";
18
18
  import { campaignTree, runDirectory } from "../run/paths.mjs";
19
+ import { readCampaign } from "../campaign/record.mjs";
19
20
 
20
21
  /** How often a foreground `plan` polls a launched stage's run directory. */
21
22
  const DEFAULT_POLL_MS = 1_000;
@@ -125,7 +126,7 @@ export function loadVerificationSuites(path) {
125
126
 
126
127
  /**
127
128
  * @param {string} target
128
- * @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, package?: string, detach?: boolean, json?: boolean}} values
129
+ * @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, package?: string, "targeted-fix"?: boolean, detach?: boolean, json?: boolean}} values
129
130
  * @returns {Promise<void>}
130
131
  */
131
132
  export async function planCli(target, values) {
@@ -151,7 +152,14 @@ export async function planCli(target, values) {
151
152
  if (typeof values.runtimes === "string" && values.runtimes) argv.push("--runtimes", resolve(values.runtimes));
152
153
  if (typeof values.verification === "string" && values.verification) argv.push("--verification", resolve(values.verification));
153
154
  if (packageMode !== "implementation") argv.push("--package", packageMode);
155
+ if (values["targeted-fix"] === true) argv.push("--targeted-fix");
154
156
  const failurePath = planBootstrapFailurePath(process.cwd(), campaignId, phase);
157
+ // Read the campaign before creating anything: the failure record lives
158
+ // inside the campaign tree, so a typo in --campaign would otherwise leave
159
+ // a campaign directory with no record in it for `discoverCampaigns` to
160
+ // find. The child reads it too; this is the launcher refusing what it can
161
+ // see for itself rather than detaching into a certain failure.
162
+ readCampaign(campaignTree(process.cwd(), campaignId));
155
163
  mkdirSync(dirname(failurePath), { recursive: true });
156
164
  rmSync(failurePath, { force: true });
157
165
  const child = detachArgv(argv);
@@ -188,6 +196,7 @@ export async function planCli(target, values) {
188
196
  runtimeDefaults,
189
197
  runtimes,
190
198
  verification,
199
+ targetedFix: values["targeted-fix"] === true,
191
200
  packageMode,
192
201
  launch: async (contractPath, contract) => {
193
202
  const child = detachSelf("run", contractPath);
package/src/cli.mjs CHANGED
@@ -124,6 +124,7 @@ export const COMMAND_OPTIONS = {
124
124
  runtimes: { type: "string" },
125
125
  verification: { type: "string" },
126
126
  package: { type: "string" },
127
+ "targeted-fix": { type: "boolean" },
127
128
  detach: { type: "boolean" },
128
129
  json: { type: "boolean" },
129
130
  },
@@ -8,7 +8,18 @@ export const VERIFICATION_LIMITS = Object.freeze({
8
8
  stderrBytes: 16 * 1024,
9
9
  maxCommands: 32,
10
10
  maxRepeat: 8,
11
- maxTimeoutSec: 600,
11
+ // A verification command may declare up to half an hour. It was 600s, which
12
+ // is below this repository's own suite: measured 2026-09-22, `npm test`
13
+ // takes 434-473s at the default parallelism on the author's machine and 650s
14
+ // on a Windows CI runner, and `node --test --test-concurrency=1 test/engine/`
15
+ // -- the way a packet's verification actually runs it -- takes 1035-1058s.
16
+ // So the one command that proves the engine could not be declared at all,
17
+ // and a packet author's way out was `--test-name-pattern`, which exits 0
18
+ // when it matches nothing (see AGENTS.md). A cap that pushes authors toward
19
+ // a proof that certifies nothing is worse than a longer runaway. The node's
20
+ // own wall clock (`contract.timeoutSec`, 2400s by default) still bounds the
21
+ // attempt above this.
22
+ maxTimeoutSec: 1_800,
12
23
  stateStdoutBytes: 2 * 1024,
13
24
  stateCommands: 16,
14
25
  stateAttempts: 4,
@@ -25,7 +25,8 @@
25
25
  * It also exits when the release file's directory is gone (measured
26
26
  * 2026-09-16: gate processes from a prior day's test runs, spawned into a
27
27
  * temp directory the failed test never cleaned up, were still alive and
28
- * waiting for a release file that could now never appear).
28
+ * waiting for a release file that could now never appear), and it keeps
29
+ * asking both of those questions after the provider starts, not only before.
29
30
  */
30
31
  import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
31
32
  import { dirname } from "node:path";
@@ -186,11 +187,32 @@ function releaseDirectoryGone() {
186
187
  return !existsSync(dirname(releasePath));
187
188
  }
188
189
 
190
+ /**
191
+ * How often the gate re-asks its two liveness questions once the provider is
192
+ * running. Before release the tick below asks them every 10ms, because it is
193
+ * also polling for the release file; after release it used to stop asking
194
+ * entirely, leaving the provider's own exit as the gate's only remaining
195
+ * liveness check. A controller that died without cleaning up, or a run
196
+ * directory deleted underneath a live attempt, therefore left the provider
197
+ * running with nobody watching -- the stranded-process shape ADR 0010
198
+ * describes, in the one window the pre-release check does not cover.
199
+ *
200
+ * A second, not ten milliseconds: this watches a provider that runs for
201
+ * minutes, and three syscalls a second is the whole cost of never stranding
202
+ * one.
203
+ */
204
+ const WATCHDOG_INTERVAL_MS = 1_000;
205
+
189
206
  const timer = setInterval(() => {
190
207
  if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
191
208
  if (releaseDirectoryGone()) { clearInterval(timer); stopProvider(); return; }
192
209
  if (!existsSync(releasePath)) return;
193
210
  clearInterval(timer);
211
+ const watchdog = setInterval(() => {
212
+ if (parentAlive() && !releaseDirectoryGone()) return;
213
+ clearInterval(watchdog);
214
+ stopProvider();
215
+ }, WATCHDOG_INTERVAL_MS);
194
216
  const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
195
217
  const stderrFd = openSync(config.stderrPath, "wx", 0o600);
196
218
  const invocation = spawnInvocation(config.executable, config.args, { cwd: config.cwd });
@@ -207,6 +229,7 @@ const timer = setInterval(() => {
207
229
  }
208
230
  provider.once("error", () => process.exitCode = 127);
209
231
  provider.once("close", (code) => {
232
+ clearInterval(watchdog);
210
233
  capLog(config.stdoutPath, config.harness === "codex");
211
234
  capLog(config.stderrPath);
212
235
  process.exit(code ?? 1);
@@ -147,8 +147,19 @@ export function terminalErrorCode(state) {
147
147
  * `turn_limit` is here and not among the timeout codes below: a turn the CLI
148
148
  * stopped itself at `--max-turns` exits cleanly with no seal yet, and the
149
149
  * next dispatch seals its worktree as it does for any previous attempt.
150
+ *
151
+ * `incomplete_stream` is a stream that ended before its terminal envelope --
152
+ * Claude with no `result`, codex with no `turn.completed`, dsh with neither
153
+ * terminal record (`harnesses/protocol.mjs`). That is the provider's transport
154
+ * dying mid-turn, not the run's own doing, so it is the same class as
155
+ * `provider_error` and earns the same one retry. It was absent, and a worker
156
+ * whose provider dropped its connection parked on the first attempt while a
157
+ * worker whose provider returned an error envelope got a second one -- the
158
+ * harsher treatment for the less informative failure. The judge role is
159
+ * unaffected: `engine/settle-judge.mjs` routes this code to its own bounded
160
+ * re-ask before anything parks.
150
161
  */
151
- export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
162
+ export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "incomplete_stream", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
152
163
 
153
164
  /**
154
165
  * Timeout codes earn their automatic retry only when phase 5b sealed work
@@ -82,13 +82,17 @@ export const DEFAULT_NODE_BUDGET_MS = 600_000;
82
82
  const APPROVE_BELOW_VALUES = new Set(["standard", "high", "none"]);
83
83
 
84
84
  /**
85
- * @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
85
+ * @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, targetedFix?: boolean, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
86
+ * `targetedFix` allows a plan with a single node. Sizing refuses one by
87
+ * default because a phase that decomposes into one node is usually a plan
88
+ * that was never decomposed; a targeted fix is the case where one node is
89
+ * the honest answer, and the operator says so.
86
90
  * @returns {Promise<FrozenPipelineResult|ContestedPipelineResult>}
87
91
  */
88
92
  export async function runPlanningPipeline(options) {
89
93
  const {
90
94
  specPath, campaignId, phase, runtimes, launch, wait,
91
- reviewRounds = 2, runtimeDefaults = {}, verification = {},
95
+ reviewRounds = 2, runtimeDefaults = {}, verification = {}, targetedFix = false,
92
96
  } = options;
93
97
  const ask = options.ask ?? askPlanningRuntimes;
94
98
  // Implementation work is sized by what it writes; exploratory work -- an
@@ -258,7 +262,7 @@ export async function runPlanningPipeline(options) {
258
262
  stage = "sizing";
259
263
  const sizing = applySizingRules(
260
264
  { nodes: currentPlan.nodes.map(toSizingNode), justification: currentPlan.justification },
261
- { nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)) },
265
+ { nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)), targetedFix },
262
266
  );
263
267
  stage = "routing";
264
268
  const routing = resolveRuntimes(sizing.plan.nodes, {