faberun 0.19.2 → 0.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/watch.mjs +197 -0
- package/src/cli/campaign.mjs +32 -180
- package/src/cli/launch.mjs +8 -2
- package/src/cli/plan.mjs +10 -1
- package/src/cli.mjs +1 -0
- package/src/contract/verification.mjs +12 -1
- package/src/engine/gate.mjs +24 -1
- package/src/engine/lifecycle.mjs +12 -1
- package/src/plan/pipeline.mjs +7 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.3",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `campaign watch --wake`: the loop that turns a campaign's own records into
|
|
3
|
+
* the lines an operator is woken with, and the single-watcher lock that keeps
|
|
4
|
+
* two of them from doubling every wake.
|
|
5
|
+
*
|
|
6
|
+
* Separate from `cli/campaign.mjs` because that file owns argv, dispatch and
|
|
7
|
+
* usage for the campaign verb, and this is neither: it is a long-running
|
|
8
|
+
* supervisor with a poll interval, an idle policy, a durable dedupe through
|
|
9
|
+
* the inbox, and a lock with its own staleness rule. `cli/campaign.mjs` keeps
|
|
10
|
+
* the thin `watch` operation that reads the flags and calls in here.
|
|
11
|
+
*/
|
|
12
|
+
import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
15
|
+
import { readCampaign } from "./record.mjs";
|
|
16
|
+
import { notifyQueueFor } from "../engine/notify-queue.mjs";
|
|
17
|
+
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
18
|
+
import { errorCode, readJsonTolerant } from "../util.mjs";
|
|
19
|
+
|
|
20
|
+
/** How often the watcher re-reads the campaign when the operator named no interval. */
|
|
21
|
+
export const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
22
|
+
|
|
23
|
+
/** How long without a material event before the watcher reports the campaign idle. */
|
|
24
|
+
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
25
|
+
|
|
26
|
+
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
27
|
+
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The watcher loop. Each line is announced through `notify`, which by default
|
|
31
|
+
* records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
|
|
32
|
+
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
|
|
33
|
+
* line already recorded is never re-sent. The injectable seams exist so a
|
|
34
|
+
* test can drive the loop deterministically.
|
|
35
|
+
*
|
|
36
|
+
* @param {string} campaignPath
|
|
37
|
+
* @param {string} runsDir
|
|
38
|
+
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
39
|
+
* @returns {Promise<void>}
|
|
40
|
+
*/
|
|
41
|
+
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
42
|
+
const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
|
|
43
|
+
const now = options.now ?? (() => Date.now());
|
|
44
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
|
|
45
|
+
const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
|
|
46
|
+
const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
|
|
47
|
+
type: "attention",
|
|
48
|
+
campaignId: event.campaignId,
|
|
49
|
+
dedupeKey: event.dedupeKey,
|
|
50
|
+
summary: event.summary,
|
|
51
|
+
runId: event.runId ?? null,
|
|
52
|
+
nodeId: event.nodeId ?? null,
|
|
53
|
+
status: event.status ?? null,
|
|
54
|
+
errorCode: event.errorCode ?? null,
|
|
55
|
+
}));
|
|
56
|
+
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
57
|
+
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
58
|
+
emit(wakeCapabilityNotice());
|
|
59
|
+
/** @type {Map<string, string>} */
|
|
60
|
+
const runSignatures = new Map();
|
|
61
|
+
let lastActiveAt = now();
|
|
62
|
+
let first = true;
|
|
63
|
+
try {
|
|
64
|
+
for (;;) {
|
|
65
|
+
const campaign = readCampaign(campaignPath);
|
|
66
|
+
if (campaign.status !== "active") {
|
|
67
|
+
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
72
|
+
* so a restart or a second watcher skips a line already recorded even
|
|
73
|
+
* when delivery is injected.
|
|
74
|
+
*
|
|
75
|
+
* @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
|
|
76
|
+
*/
|
|
77
|
+
const announce = async (dedupeKey, summary, extra = {}) => {
|
|
78
|
+
if (seen.has(dedupeKey)) return;
|
|
79
|
+
seen.add(dedupeKey);
|
|
80
|
+
const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
81
|
+
if (!appended.appended) return;
|
|
82
|
+
emit(summary);
|
|
83
|
+
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
84
|
+
};
|
|
85
|
+
let anyActive = false;
|
|
86
|
+
for (const runId of campaign.linkedRunIds) {
|
|
87
|
+
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
88
|
+
if (!status || !Array.isArray(status.nodes)) continue;
|
|
89
|
+
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
90
|
+
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
91
|
+
const previous = runSignatures.get(runId);
|
|
92
|
+
runSignatures.set(runId, signature);
|
|
93
|
+
if (!terminal) {
|
|
94
|
+
anyActive = true;
|
|
95
|
+
const runLock = readLock(join(runsDir, runId));
|
|
96
|
+
const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
|
|
97
|
+
if (stale && !first) {
|
|
98
|
+
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (!first && previous !== signature) {
|
|
102
|
+
for (const node of status.nodes) {
|
|
103
|
+
const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
|
|
104
|
+
|| (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
|
|
105
|
+
if (attention) {
|
|
106
|
+
const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
|
|
107
|
+
await announce(
|
|
108
|
+
key,
|
|
109
|
+
`campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
|
|
110
|
+
{ runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (terminal) {
|
|
116
|
+
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const nowMs = now();
|
|
120
|
+
if (anyActive) lastActiveAt = nowMs;
|
|
121
|
+
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
122
|
+
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
123
|
+
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
124
|
+
}
|
|
125
|
+
first = false;
|
|
126
|
+
if (options.once === true) return;
|
|
127
|
+
await sleep(pollMs);
|
|
128
|
+
}
|
|
129
|
+
} finally {
|
|
130
|
+
lock.release();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const WATCH_LOCK_FILE = "watch.lock";
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* A durable campaign-watch lock, one watcher per campaign across processes.
|
|
138
|
+
* A live holder is never taken over; a dead or recycled pid's lock is stale
|
|
139
|
+
* and is replaced, so a restart after a crash is not blocked. The same
|
|
140
|
+
* liveness rule as the controller lock: a pid is dead only when the probe
|
|
141
|
+
* proves it.
|
|
142
|
+
*
|
|
143
|
+
* @param {string} campaignPath
|
|
144
|
+
* @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
|
|
145
|
+
*/
|
|
146
|
+
export function acquireWatchLock(campaignPath) {
|
|
147
|
+
const path = join(campaignPath, WATCH_LOCK_FILE);
|
|
148
|
+
/** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
|
|
149
|
+
let occupant = {};
|
|
150
|
+
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
151
|
+
const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
|
|
152
|
+
try {
|
|
153
|
+
const fd = openSync(path, "wx", 0o600);
|
|
154
|
+
try {
|
|
155
|
+
writeSync(fd, JSON.stringify(record));
|
|
156
|
+
} finally {
|
|
157
|
+
closeSync(fd);
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
...record,
|
|
161
|
+
release() {
|
|
162
|
+
try {
|
|
163
|
+
unlinkSync(path);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
} catch (error) {
|
|
170
|
+
if (errorCode(error) !== "EEXIST") throw error;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
174
|
+
} catch {
|
|
175
|
+
occupant = {};
|
|
176
|
+
}
|
|
177
|
+
if (!watchLockStale(occupant)) {
|
|
178
|
+
throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
|
|
179
|
+
}
|
|
180
|
+
try {
|
|
181
|
+
unlinkSync(path);
|
|
182
|
+
} catch (error) {
|
|
183
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
throw new Error("campaign watch lock contention did not settle");
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* @param {{pid?: number, processStartToken?: string|null}} occupant
|
|
191
|
+
* @returns {boolean}
|
|
192
|
+
*/
|
|
193
|
+
function watchLockStale(occupant) {
|
|
194
|
+
if (typeof occupant.pid !== "number") return true;
|
|
195
|
+
if (!pidAlive(occupant.pid)) return true;
|
|
196
|
+
return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
|
|
197
|
+
}
|
package/src/cli/campaign.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
3
|
import { join, resolve } from "node:path";
|
|
4
4
|
import { parseArgs as parseFlags } from "node:util";
|
|
5
5
|
import {
|
|
@@ -11,25 +11,19 @@ import {
|
|
|
11
11
|
resolveCampaign,
|
|
12
12
|
} from "../campaign/index.mjs";
|
|
13
13
|
import { addContract, replaceContract } from "./campaign-contract.mjs";
|
|
14
|
-
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
15
14
|
import { runsRoot } from "../run/paths.mjs";
|
|
16
15
|
import { syncAgentSignal } from "../repo/signal.mjs";
|
|
17
16
|
import { acknowledgeJournalEvent, appendJournal, appendSeatAllowanceEvent, readJournal, watchJournal } from "../campaign/journal.mjs";
|
|
18
17
|
import { driveCampaignChain } from "../campaign/chain.mjs";
|
|
19
18
|
import { unparkCampaign } from "../campaign/unpark.mjs";
|
|
20
19
|
import { readCampaign } from "../campaign/record.mjs";
|
|
21
|
-
import {
|
|
22
|
-
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
20
|
+
import { DEFAULT_WAKE_POLL_MS, watchCampaignWake } from "../campaign/watch.mjs";
|
|
23
21
|
import { allowanceEventFields, sampleAllowance } from "../seat/allowance.mjs";
|
|
24
22
|
import { detectOperatorHarness } from "../seat/harnesses.mjs";
|
|
25
23
|
import { detachArgv, detachSelf, waitForBootstrap } from "./launch.mjs";
|
|
26
|
-
import {
|
|
24
|
+
import { readJsonTolerant } from "../util.mjs";
|
|
27
25
|
|
|
28
26
|
const SYNC_OUTPUT_MAX_BYTES = 8000;
|
|
29
|
-
const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
30
|
-
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
31
|
-
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
32
|
-
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
33
27
|
|
|
34
28
|
const NOTE_KINDS = new Set([
|
|
35
29
|
"intent",
|
|
@@ -159,177 +153,6 @@ async function watch(campaignId, values) {
|
|
|
159
153
|
await watchCampaignWake(path, runsDir, { pollMs, once: values.once === true });
|
|
160
154
|
}
|
|
161
155
|
|
|
162
|
-
/**
|
|
163
|
-
* The watcher loop. Each line is announced through `notify`, which by default
|
|
164
|
-
* records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
|
|
165
|
-
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
|
|
166
|
-
* line already recorded is never re-sent. The injectable seams exist so a
|
|
167
|
-
* test can drive the loop deterministically.
|
|
168
|
-
*
|
|
169
|
-
* @param {string} campaignPath
|
|
170
|
-
* @param {string} runsDir
|
|
171
|
-
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
172
|
-
* @returns {Promise<void>}
|
|
173
|
-
*/
|
|
174
|
-
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
175
|
-
const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
|
|
176
|
-
const now = options.now ?? (() => Date.now());
|
|
177
|
-
const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
|
|
178
|
-
const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
|
|
179
|
-
const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
|
|
180
|
-
type: "attention",
|
|
181
|
-
campaignId: event.campaignId,
|
|
182
|
-
dedupeKey: event.dedupeKey,
|
|
183
|
-
summary: event.summary,
|
|
184
|
-
runId: event.runId ?? null,
|
|
185
|
-
nodeId: event.nodeId ?? null,
|
|
186
|
-
status: event.status ?? null,
|
|
187
|
-
errorCode: event.errorCode ?? null,
|
|
188
|
-
}));
|
|
189
|
-
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
190
|
-
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
191
|
-
emit(wakeCapabilityNotice());
|
|
192
|
-
/** @type {Map<string, string>} */
|
|
193
|
-
const runSignatures = new Map();
|
|
194
|
-
let lastActiveAt = now();
|
|
195
|
-
let first = true;
|
|
196
|
-
try {
|
|
197
|
-
for (;;) {
|
|
198
|
-
const campaign = readCampaign(campaignPath);
|
|
199
|
-
if (campaign.status !== "active") {
|
|
200
|
-
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
201
|
-
return;
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
205
|
-
* so a restart or a second watcher skips a line already recorded even
|
|
206
|
-
* when delivery is injected.
|
|
207
|
-
*
|
|
208
|
-
* @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
|
|
209
|
-
*/
|
|
210
|
-
const announce = async (dedupeKey, summary, extra = {}) => {
|
|
211
|
-
if (seen.has(dedupeKey)) return;
|
|
212
|
-
seen.add(dedupeKey);
|
|
213
|
-
const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
214
|
-
if (!appended.appended) return;
|
|
215
|
-
emit(summary);
|
|
216
|
-
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
217
|
-
};
|
|
218
|
-
let anyActive = false;
|
|
219
|
-
for (const runId of campaign.linkedRunIds) {
|
|
220
|
-
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
221
|
-
if (!status || !Array.isArray(status.nodes)) continue;
|
|
222
|
-
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
223
|
-
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
224
|
-
const previous = runSignatures.get(runId);
|
|
225
|
-
runSignatures.set(runId, signature);
|
|
226
|
-
if (!terminal) {
|
|
227
|
-
anyActive = true;
|
|
228
|
-
const runLock = readLock(join(runsDir, runId));
|
|
229
|
-
const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
|
|
230
|
-
if (stale && !first) {
|
|
231
|
-
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
if (!first && previous !== signature) {
|
|
235
|
-
for (const node of status.nodes) {
|
|
236
|
-
const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
|
|
237
|
-
|| (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
|
|
238
|
-
if (attention) {
|
|
239
|
-
const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
|
|
240
|
-
await announce(
|
|
241
|
-
key,
|
|
242
|
-
`campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
|
|
243
|
-
{ runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
if (terminal) {
|
|
249
|
-
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
const nowMs = now();
|
|
253
|
-
if (anyActive) lastActiveAt = nowMs;
|
|
254
|
-
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
255
|
-
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
256
|
-
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
257
|
-
}
|
|
258
|
-
first = false;
|
|
259
|
-
if (options.once === true) return;
|
|
260
|
-
await sleep(pollMs);
|
|
261
|
-
}
|
|
262
|
-
} finally {
|
|
263
|
-
lock.release();
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
const WATCH_LOCK_FILE = "watch.lock";
|
|
268
|
-
|
|
269
|
-
/**
|
|
270
|
-
* A durable campaign-watch lock, one watcher per campaign across processes.
|
|
271
|
-
* A live holder is never taken over; a dead or recycled pid's lock is stale
|
|
272
|
-
* and is replaced, so a restart after a crash is not blocked. The same
|
|
273
|
-
* liveness rule as the controller lock: a pid is dead only when the probe
|
|
274
|
-
* proves it.
|
|
275
|
-
*
|
|
276
|
-
* @param {string} campaignPath
|
|
277
|
-
* @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
|
|
278
|
-
*/
|
|
279
|
-
export function acquireWatchLock(campaignPath) {
|
|
280
|
-
const path = join(campaignPath, WATCH_LOCK_FILE);
|
|
281
|
-
/** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
|
|
282
|
-
let occupant = {};
|
|
283
|
-
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
284
|
-
const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
|
|
285
|
-
try {
|
|
286
|
-
const fd = openSync(path, "wx", 0o600);
|
|
287
|
-
try {
|
|
288
|
-
writeSync(fd, JSON.stringify(record));
|
|
289
|
-
} finally {
|
|
290
|
-
closeSync(fd);
|
|
291
|
-
}
|
|
292
|
-
return {
|
|
293
|
-
...record,
|
|
294
|
-
release() {
|
|
295
|
-
try {
|
|
296
|
-
unlinkSync(path);
|
|
297
|
-
} catch (error) {
|
|
298
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
299
|
-
}
|
|
300
|
-
},
|
|
301
|
-
};
|
|
302
|
-
} catch (error) {
|
|
303
|
-
if (errorCode(error) !== "EEXIST") throw error;
|
|
304
|
-
}
|
|
305
|
-
try {
|
|
306
|
-
occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
307
|
-
} catch {
|
|
308
|
-
occupant = {};
|
|
309
|
-
}
|
|
310
|
-
if (!watchLockStale(occupant)) {
|
|
311
|
-
throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
|
|
312
|
-
}
|
|
313
|
-
try {
|
|
314
|
-
unlinkSync(path);
|
|
315
|
-
} catch (error) {
|
|
316
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
throw new Error("campaign watch lock contention did not settle");
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
/**
|
|
323
|
-
* @param {{pid?: number, processStartToken?: string|null}} occupant
|
|
324
|
-
* @returns {boolean}
|
|
325
|
-
*/
|
|
326
|
-
function watchLockStale(occupant) {
|
|
327
|
-
if (typeof occupant.pid !== "number") return true;
|
|
328
|
-
if (!pidAlive(occupant.pid)) return true;
|
|
329
|
-
return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
|
|
333
156
|
/**
|
|
334
157
|
* @param {string} campaignId
|
|
335
158
|
* @param {CliValues} values
|
|
@@ -467,9 +290,38 @@ function close(campaignId, values) {
|
|
|
467
290
|
renderHandoff(path, runsDir);
|
|
468
291
|
process.stdout.write(`[campaign] ${closed.campaign.id} closed\n`);
|
|
469
292
|
process.stdout.write(`[campaign] ledger · docs/campaigns/${closed.campaign.id}/ledger · ${closed.ledgerFiles.length} files\n`);
|
|
293
|
+
reportRequirementClosure(closed.campaign.requirements ?? []);
|
|
470
294
|
if (syncAgentSignal(runsDir)) process.stdout.write(`[campaign] AGENTS.md signal updated\n`);
|
|
471
295
|
}
|
|
472
296
|
|
|
297
|
+
/**
|
|
298
|
+
* Say out loud what the close just wrote, and what it measured.
|
|
299
|
+
*
|
|
300
|
+
* The record is a correlation between the requirement ids a linked run's
|
|
301
|
+
* contract declared and the node snapshots that reached `done` carrying them.
|
|
302
|
+
* It reads the run directory and never the branch, so a requirement the
|
|
303
|
+
* operator delivered by hand -- because its node blocked, which happens --
|
|
304
|
+
* lands in the record as `open` while the work is on main. Measured
|
|
305
|
+
* 2026-09-22 on `availability-is-verified-not-assumed`: R4 read `open` with
|
|
306
|
+
* an empty `nodes` array and was merged in `a6a5d43`.
|
|
307
|
+
*
|
|
308
|
+
* Nothing printed this before, so the only way to meet the closure was to
|
|
309
|
+
* open the JSON, where `open` reads as "not delivered" instead of "no node of
|
|
310
|
+
* this campaign proved it". Naming the measurement is the whole fix: a close
|
|
311
|
+
* cannot know what it did not run.
|
|
312
|
+
*
|
|
313
|
+
* @param {import("../campaign/index.mjs").RequirementClosure[]} requirements
|
|
314
|
+
*/
|
|
315
|
+
function reportRequirementClosure(requirements) {
|
|
316
|
+
if (requirements.length === 0) return;
|
|
317
|
+
const open = requirements.filter((entry) => entry.status !== "covered");
|
|
318
|
+
const covered = requirements.length - open.length;
|
|
319
|
+
process.stdout.write(`[campaign] requirements · ${covered}/${requirements.length} carried by a node that reached done\n`);
|
|
320
|
+
if (open.length === 0) return;
|
|
321
|
+
process.stdout.write(`[campaign] no done node carried ${open.map((entry) => entry.requirementId).join(", ")}\n`);
|
|
322
|
+
process.stdout.write("[campaign] closure reads node snapshots, never the branch: a requirement delivered outside a node reads open here\n");
|
|
323
|
+
}
|
|
324
|
+
|
|
473
325
|
/**
|
|
474
326
|
* `campaign supervise <id>` (also spelled `supervise campaign <id>`): the
|
|
475
327
|
* idempotent re-invocation that drives the manifest. It takes the campaign's
|
package/src/cli/launch.mjs
CHANGED
|
@@ -55,8 +55,14 @@ export function detachSelf(command, target, extraArgs = []) {
|
|
|
55
55
|
* exactly as for every detached controller: a foreground launcher is the only
|
|
56
56
|
* moment an operator is present.
|
|
57
57
|
*
|
|
58
|
+
* `stdio` overrides the discarded default for a caller that has somewhere
|
|
59
|
+
* durable to put the child's output. A detached *run* wants the default: its
|
|
60
|
+
* bootstrap record is the channel, and a controller's real output belongs in
|
|
61
|
+
* the run directory. A detached *plan* has no bootstrap record, so discarding
|
|
62
|
+
* its stdio discarded the only account of why it died.
|
|
63
|
+
*
|
|
58
64
|
* @param {string[]} argv
|
|
59
|
-
* @param {{nonce?: string, env?: NodeJS.ProcessEnv}} [options]
|
|
65
|
+
* @param {{nonce?: string, env?: NodeJS.ProcessEnv, stdio?: import("node:child_process").StdioOptions}} [options]
|
|
60
66
|
* @returns {DetachedChild}
|
|
61
67
|
*/
|
|
62
68
|
export function detachArgv(argv, options = {}) {
|
|
@@ -74,7 +80,7 @@ export function detachArgv(argv, options = {}) {
|
|
|
74
80
|
// pending forever. There it also means DETACHED_PROCESS: no console
|
|
75
81
|
// window, which is what "ignore" stdio already implies.
|
|
76
82
|
detached: true,
|
|
77
|
-
stdio: "ignore",
|
|
83
|
+
stdio: options.stdio ?? "ignore",
|
|
78
84
|
}));
|
|
79
85
|
child.unref();
|
|
80
86
|
child.bootstrapNonce = nonce;
|
package/src/cli/plan.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { colorLevel, statusToken } from "./brand.mjs";
|
|
|
16
16
|
import { delay } from "../util.mjs";
|
|
17
17
|
import { runPlanningPipeline } from "../plan/pipeline.mjs";
|
|
18
18
|
import { campaignTree, runDirectory } from "../run/paths.mjs";
|
|
19
|
+
import { readCampaign } from "../campaign/record.mjs";
|
|
19
20
|
|
|
20
21
|
/** How often a foreground `plan` polls a launched stage's run directory. */
|
|
21
22
|
const DEFAULT_POLL_MS = 1_000;
|
|
@@ -125,7 +126,7 @@ export function loadVerificationSuites(path) {
|
|
|
125
126
|
|
|
126
127
|
/**
|
|
127
128
|
* @param {string} target
|
|
128
|
-
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, package?: string, detach?: boolean, json?: boolean}} values
|
|
129
|
+
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, package?: string, "targeted-fix"?: boolean, detach?: boolean, json?: boolean}} values
|
|
129
130
|
* @returns {Promise<void>}
|
|
130
131
|
*/
|
|
131
132
|
export async function planCli(target, values) {
|
|
@@ -151,7 +152,14 @@ export async function planCli(target, values) {
|
|
|
151
152
|
if (typeof values.runtimes === "string" && values.runtimes) argv.push("--runtimes", resolve(values.runtimes));
|
|
152
153
|
if (typeof values.verification === "string" && values.verification) argv.push("--verification", resolve(values.verification));
|
|
153
154
|
if (packageMode !== "implementation") argv.push("--package", packageMode);
|
|
155
|
+
if (values["targeted-fix"] === true) argv.push("--targeted-fix");
|
|
154
156
|
const failurePath = planBootstrapFailurePath(process.cwd(), campaignId, phase);
|
|
157
|
+
// Read the campaign before creating anything: the failure record lives
|
|
158
|
+
// inside the campaign tree, so a typo in --campaign would otherwise leave
|
|
159
|
+
// a campaign directory with no record in it for `discoverCampaigns` to
|
|
160
|
+
// find. The child reads it too; this is the launcher refusing what it can
|
|
161
|
+
// see for itself rather than detaching into a certain failure.
|
|
162
|
+
readCampaign(campaignTree(process.cwd(), campaignId));
|
|
155
163
|
mkdirSync(dirname(failurePath), { recursive: true });
|
|
156
164
|
rmSync(failurePath, { force: true });
|
|
157
165
|
const child = detachArgv(argv);
|
|
@@ -188,6 +196,7 @@ export async function planCli(target, values) {
|
|
|
188
196
|
runtimeDefaults,
|
|
189
197
|
runtimes,
|
|
190
198
|
verification,
|
|
199
|
+
targetedFix: values["targeted-fix"] === true,
|
|
191
200
|
packageMode,
|
|
192
201
|
launch: async (contractPath, contract) => {
|
|
193
202
|
const child = detachSelf("run", contractPath);
|
package/src/cli.mjs
CHANGED
|
@@ -8,7 +8,18 @@ export const VERIFICATION_LIMITS = Object.freeze({
|
|
|
8
8
|
stderrBytes: 16 * 1024,
|
|
9
9
|
maxCommands: 32,
|
|
10
10
|
maxRepeat: 8,
|
|
11
|
-
|
|
11
|
+
// A verification command may declare up to half an hour. It was 600s, which
|
|
12
|
+
// is below this repository's own suite: measured 2026-09-22, `npm test`
|
|
13
|
+
// takes 434-473s at the default parallelism on the author's machine and 650s
|
|
14
|
+
// on a Windows CI runner, and `node --test --test-concurrency=1 test/engine/`
|
|
15
|
+
// -- the way a packet's verification actually runs it -- takes 1035-1058s.
|
|
16
|
+
// So the one command that proves the engine could not be declared at all,
|
|
17
|
+
// and a packet author's way out was `--test-name-pattern`, which exits 0
|
|
18
|
+
// when it matches nothing (see AGENTS.md). A cap that pushes authors toward
|
|
19
|
+
// a proof that certifies nothing is worse than a longer runaway. The node's
|
|
20
|
+
// own wall clock (`contract.timeoutSec`, 2400s by default) still bounds the
|
|
21
|
+
// attempt above this.
|
|
22
|
+
maxTimeoutSec: 1_800,
|
|
12
23
|
stateStdoutBytes: 2 * 1024,
|
|
13
24
|
stateCommands: 16,
|
|
14
25
|
stateAttempts: 4,
|
package/src/engine/gate.mjs
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
* It also exits when the release file's directory is gone (measured
|
|
26
26
|
* 2026-09-16: gate processes from a prior day's test runs, spawned into a
|
|
27
27
|
* temp directory the failed test never cleaned up, were still alive and
|
|
28
|
-
* waiting for a release file that could now never appear)
|
|
28
|
+
* waiting for a release file that could now never appear), and it keeps
|
|
29
|
+
* asking both of those questions after the provider starts, not only before.
|
|
29
30
|
*/
|
|
30
31
|
import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
|
|
31
32
|
import { dirname } from "node:path";
|
|
@@ -186,11 +187,32 @@ function releaseDirectoryGone() {
|
|
|
186
187
|
return !existsSync(dirname(releasePath));
|
|
187
188
|
}
|
|
188
189
|
|
|
190
|
+
/**
|
|
191
|
+
* How often the gate re-asks its two liveness questions once the provider is
|
|
192
|
+
* running. Before release the tick below asks them every 10ms, because it is
|
|
193
|
+
* also polling for the release file; after release it used to stop asking
|
|
194
|
+
* entirely, leaving the provider's own exit as the gate's only remaining
|
|
195
|
+
* liveness check. A controller that died without cleaning up, or a run
|
|
196
|
+
* directory deleted underneath a live attempt, therefore left the provider
|
|
197
|
+
* running with nobody watching -- the stranded-process shape ADR 0010
|
|
198
|
+
* describes, in the one window the pre-release check does not cover.
|
|
199
|
+
*
|
|
200
|
+
* A second, not ten milliseconds: this watches a provider that runs for
|
|
201
|
+
* minutes, and three syscalls a second is the whole cost of never stranding
|
|
202
|
+
* one.
|
|
203
|
+
*/
|
|
204
|
+
const WATCHDOG_INTERVAL_MS = 1_000;
|
|
205
|
+
|
|
189
206
|
const timer = setInterval(() => {
|
|
190
207
|
if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
|
|
191
208
|
if (releaseDirectoryGone()) { clearInterval(timer); stopProvider(); return; }
|
|
192
209
|
if (!existsSync(releasePath)) return;
|
|
193
210
|
clearInterval(timer);
|
|
211
|
+
const watchdog = setInterval(() => {
|
|
212
|
+
if (parentAlive() && !releaseDirectoryGone()) return;
|
|
213
|
+
clearInterval(watchdog);
|
|
214
|
+
stopProvider();
|
|
215
|
+
}, WATCHDOG_INTERVAL_MS);
|
|
194
216
|
const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
|
|
195
217
|
const stderrFd = openSync(config.stderrPath, "wx", 0o600);
|
|
196
218
|
const invocation = spawnInvocation(config.executable, config.args, { cwd: config.cwd });
|
|
@@ -207,6 +229,7 @@ const timer = setInterval(() => {
|
|
|
207
229
|
}
|
|
208
230
|
provider.once("error", () => process.exitCode = 127);
|
|
209
231
|
provider.once("close", (code) => {
|
|
232
|
+
clearInterval(watchdog);
|
|
210
233
|
capLog(config.stdoutPath, config.harness === "codex");
|
|
211
234
|
capLog(config.stderrPath);
|
|
212
235
|
process.exit(code ?? 1);
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -147,8 +147,19 @@ export function terminalErrorCode(state) {
|
|
|
147
147
|
* `turn_limit` is here and not among the timeout codes below: a turn the CLI
|
|
148
148
|
* stopped itself at `--max-turns` exits cleanly with no seal yet, and the
|
|
149
149
|
* next dispatch seals its worktree as it does for any previous attempt.
|
|
150
|
+
*
|
|
151
|
+
* `incomplete_stream` is a stream that ended before its terminal envelope --
|
|
152
|
+
* Claude with no `result`, codex with no `turn.completed`, dsh with neither
|
|
153
|
+
* terminal record (`harnesses/protocol.mjs`). That is the provider's transport
|
|
154
|
+
* dying mid-turn, not the run's own doing, so it is the same class as
|
|
155
|
+
* `provider_error` and earns the same one retry. It was absent, and a worker
|
|
156
|
+
* whose provider dropped its connection parked on the first attempt while a
|
|
157
|
+
* worker whose provider returned an error envelope got a second one -- the
|
|
158
|
+
* harsher treatment for the less informative failure. The judge role is
|
|
159
|
+
* unaffected: `engine/settle-judge.mjs` routes this code to its own bounded
|
|
160
|
+
* re-ask before anything parks.
|
|
150
161
|
*/
|
|
151
|
-
export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
|
|
162
|
+
export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "incomplete_stream", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
|
|
152
163
|
|
|
153
164
|
/**
|
|
154
165
|
* Timeout codes earn their automatic retry only when phase 5b sealed work
|
package/src/plan/pipeline.mjs
CHANGED
|
@@ -82,13 +82,17 @@ export const DEFAULT_NODE_BUDGET_MS = 600_000;
|
|
|
82
82
|
const APPROVE_BELOW_VALUES = new Set(["standard", "high", "none"]);
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
|
-
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
|
|
85
|
+
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, targetedFix?: boolean, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
|
|
86
|
+
* `targetedFix` allows a plan with a single node. Sizing refuses one by
|
|
87
|
+
* default because a phase that decomposes into one node is usually a plan
|
|
88
|
+
* that was never decomposed; a targeted fix is the case where one node is
|
|
89
|
+
* the honest answer, and the operator says so.
|
|
86
90
|
* @returns {Promise<FrozenPipelineResult|ContestedPipelineResult>}
|
|
87
91
|
*/
|
|
88
92
|
export async function runPlanningPipeline(options) {
|
|
89
93
|
const {
|
|
90
94
|
specPath, campaignId, phase, runtimes, launch, wait,
|
|
91
|
-
reviewRounds = 2, runtimeDefaults = {}, verification = {},
|
|
95
|
+
reviewRounds = 2, runtimeDefaults = {}, verification = {}, targetedFix = false,
|
|
92
96
|
} = options;
|
|
93
97
|
const ask = options.ask ?? askPlanningRuntimes;
|
|
94
98
|
// Implementation work is sized by what it writes; exploratory work -- an
|
|
@@ -258,7 +262,7 @@ export async function runPlanningPipeline(options) {
|
|
|
258
262
|
stage = "sizing";
|
|
259
263
|
const sizing = applySizingRules(
|
|
260
264
|
{ nodes: currentPlan.nodes.map(toSizingNode), justification: currentPlan.justification },
|
|
261
|
-
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)) },
|
|
265
|
+
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)), targetedFix },
|
|
262
266
|
);
|
|
263
267
|
stage = "routing";
|
|
264
268
|
const routing = resolveRuntimes(sizing.plan.nodes, {
|