faberun 0.19.2 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/watch.mjs +197 -0
- package/src/cli/campaign.mjs +32 -180
- package/src/cli/launch.mjs +8 -2
- package/src/cli/plan.mjs +10 -1
- package/src/cli.mjs +1 -0
- package/src/contract/verification.mjs +12 -1
- package/src/engine/dispatch.mjs +8 -1
- package/src/engine/gate.mjs +24 -1
- package/src/engine/lifecycle.mjs +12 -1
- package/src/engine/process.mjs +36 -8
- package/src/engine/transcript.mjs +43 -1
- package/src/harnesses/index.mjs +1 -1
- package/src/harnesses/zcode/index.mjs +49 -4
- package/src/plan/pipeline.mjs +7 -3
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.20.0",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `campaign watch --wake`: the loop that turns a campaign's own records into
|
|
3
|
+
* the lines an operator is woken with, and the single-watcher lock that keeps
|
|
4
|
+
* two of them from doubling every wake.
|
|
5
|
+
*
|
|
6
|
+
* Separate from `cli/campaign.mjs` because that file owns argv, dispatch and
|
|
7
|
+
* usage for the campaign verb, and this is neither: it is a long-running
|
|
8
|
+
* supervisor with a poll interval, an idle policy, a durable dedupe through
|
|
9
|
+
* the inbox, and a lock with its own staleness rule. `cli/campaign.mjs` keeps
|
|
10
|
+
* the thin `watch` operation that reads the flags and calls in here.
|
|
11
|
+
*/
|
|
12
|
+
import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
15
|
+
import { readCampaign } from "./record.mjs";
|
|
16
|
+
import { notifyQueueFor } from "../engine/notify-queue.mjs";
|
|
17
|
+
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
18
|
+
import { errorCode, readJsonTolerant } from "../util.mjs";
|
|
19
|
+
|
|
20
|
+
/** How often the watcher re-reads the campaign when the operator named no interval. */
|
|
21
|
+
export const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
22
|
+
|
|
23
|
+
/** How long without a material event before the watcher reports the campaign idle. */
|
|
24
|
+
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
25
|
+
|
|
26
|
+
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
27
|
+
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The watcher loop. Each line is announced through `notify`, which by default
|
|
31
|
+
* records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
|
|
32
|
+
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
|
|
33
|
+
* line already recorded is never re-sent. The injectable seams exist so a
|
|
34
|
+
* test can drive the loop deterministically.
|
|
35
|
+
*
|
|
36
|
+
* @param {string} campaignPath
|
|
37
|
+
* @param {string} runsDir
|
|
38
|
+
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
39
|
+
* @returns {Promise<void>}
|
|
40
|
+
*/
|
|
41
|
+
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
42
|
+
const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
|
|
43
|
+
const now = options.now ?? (() => Date.now());
|
|
44
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
|
|
45
|
+
const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
|
|
46
|
+
const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
|
|
47
|
+
type: "attention",
|
|
48
|
+
campaignId: event.campaignId,
|
|
49
|
+
dedupeKey: event.dedupeKey,
|
|
50
|
+
summary: event.summary,
|
|
51
|
+
runId: event.runId ?? null,
|
|
52
|
+
nodeId: event.nodeId ?? null,
|
|
53
|
+
status: event.status ?? null,
|
|
54
|
+
errorCode: event.errorCode ?? null,
|
|
55
|
+
}));
|
|
56
|
+
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
57
|
+
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
58
|
+
emit(wakeCapabilityNotice());
|
|
59
|
+
/** @type {Map<string, string>} */
|
|
60
|
+
const runSignatures = new Map();
|
|
61
|
+
let lastActiveAt = now();
|
|
62
|
+
let first = true;
|
|
63
|
+
try {
|
|
64
|
+
for (;;) {
|
|
65
|
+
const campaign = readCampaign(campaignPath);
|
|
66
|
+
if (campaign.status !== "active") {
|
|
67
|
+
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
72
|
+
* so a restart or a second watcher skips a line already recorded even
|
|
73
|
+
* when delivery is injected.
|
|
74
|
+
*
|
|
75
|
+
* @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
|
|
76
|
+
*/
|
|
77
|
+
const announce = async (dedupeKey, summary, extra = {}) => {
|
|
78
|
+
if (seen.has(dedupeKey)) return;
|
|
79
|
+
seen.add(dedupeKey);
|
|
80
|
+
const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
81
|
+
if (!appended.appended) return;
|
|
82
|
+
emit(summary);
|
|
83
|
+
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
84
|
+
};
|
|
85
|
+
let anyActive = false;
|
|
86
|
+
for (const runId of campaign.linkedRunIds) {
|
|
87
|
+
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
88
|
+
if (!status || !Array.isArray(status.nodes)) continue;
|
|
89
|
+
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
90
|
+
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
91
|
+
const previous = runSignatures.get(runId);
|
|
92
|
+
runSignatures.set(runId, signature);
|
|
93
|
+
if (!terminal) {
|
|
94
|
+
anyActive = true;
|
|
95
|
+
const runLock = readLock(join(runsDir, runId));
|
|
96
|
+
const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
|
|
97
|
+
if (stale && !first) {
|
|
98
|
+
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (!first && previous !== signature) {
|
|
102
|
+
for (const node of status.nodes) {
|
|
103
|
+
const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
|
|
104
|
+
|| (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
|
|
105
|
+
if (attention) {
|
|
106
|
+
const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
|
|
107
|
+
await announce(
|
|
108
|
+
key,
|
|
109
|
+
`campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
|
|
110
|
+
{ runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (terminal) {
|
|
116
|
+
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const nowMs = now();
|
|
120
|
+
if (anyActive) lastActiveAt = nowMs;
|
|
121
|
+
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
122
|
+
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
123
|
+
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
124
|
+
}
|
|
125
|
+
first = false;
|
|
126
|
+
if (options.once === true) return;
|
|
127
|
+
await sleep(pollMs);
|
|
128
|
+
}
|
|
129
|
+
} finally {
|
|
130
|
+
lock.release();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const WATCH_LOCK_FILE = "watch.lock";
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* A durable campaign-watch lock, one watcher per campaign across processes.
|
|
138
|
+
* A live holder is never taken over; a dead or recycled pid's lock is stale
|
|
139
|
+
* and is replaced, so a restart after a crash is not blocked. The same
|
|
140
|
+
* liveness rule as the controller lock: a pid is dead only when the probe
|
|
141
|
+
* proves it.
|
|
142
|
+
*
|
|
143
|
+
* @param {string} campaignPath
|
|
144
|
+
* @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
|
|
145
|
+
*/
|
|
146
|
+
export function acquireWatchLock(campaignPath) {
|
|
147
|
+
const path = join(campaignPath, WATCH_LOCK_FILE);
|
|
148
|
+
/** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
|
|
149
|
+
let occupant = {};
|
|
150
|
+
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
151
|
+
const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
|
|
152
|
+
try {
|
|
153
|
+
const fd = openSync(path, "wx", 0o600);
|
|
154
|
+
try {
|
|
155
|
+
writeSync(fd, JSON.stringify(record));
|
|
156
|
+
} finally {
|
|
157
|
+
closeSync(fd);
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
...record,
|
|
161
|
+
release() {
|
|
162
|
+
try {
|
|
163
|
+
unlinkSync(path);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
} catch (error) {
|
|
170
|
+
if (errorCode(error) !== "EEXIST") throw error;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
174
|
+
} catch {
|
|
175
|
+
occupant = {};
|
|
176
|
+
}
|
|
177
|
+
if (!watchLockStale(occupant)) {
|
|
178
|
+
throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
|
|
179
|
+
}
|
|
180
|
+
try {
|
|
181
|
+
unlinkSync(path);
|
|
182
|
+
} catch (error) {
|
|
183
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
throw new Error("campaign watch lock contention did not settle");
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* @param {{pid?: number, processStartToken?: string|null}} occupant
|
|
191
|
+
* @returns {boolean}
|
|
192
|
+
*/
|
|
193
|
+
function watchLockStale(occupant) {
|
|
194
|
+
if (typeof occupant.pid !== "number") return true;
|
|
195
|
+
if (!pidAlive(occupant.pid)) return true;
|
|
196
|
+
return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
|
|
197
|
+
}
|
package/src/cli/campaign.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
3
|
import { join, resolve } from "node:path";
|
|
4
4
|
import { parseArgs as parseFlags } from "node:util";
|
|
5
5
|
import {
|
|
@@ -11,25 +11,19 @@ import {
|
|
|
11
11
|
resolveCampaign,
|
|
12
12
|
} from "../campaign/index.mjs";
|
|
13
13
|
import { addContract, replaceContract } from "./campaign-contract.mjs";
|
|
14
|
-
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
15
14
|
import { runsRoot } from "../run/paths.mjs";
|
|
16
15
|
import { syncAgentSignal } from "../repo/signal.mjs";
|
|
17
16
|
import { acknowledgeJournalEvent, appendJournal, appendSeatAllowanceEvent, readJournal, watchJournal } from "../campaign/journal.mjs";
|
|
18
17
|
import { driveCampaignChain } from "../campaign/chain.mjs";
|
|
19
18
|
import { unparkCampaign } from "../campaign/unpark.mjs";
|
|
20
19
|
import { readCampaign } from "../campaign/record.mjs";
|
|
21
|
-
import {
|
|
22
|
-
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
20
|
+
import { DEFAULT_WAKE_POLL_MS, watchCampaignWake } from "../campaign/watch.mjs";
|
|
23
21
|
import { allowanceEventFields, sampleAllowance } from "../seat/allowance.mjs";
|
|
24
22
|
import { detectOperatorHarness } from "../seat/harnesses.mjs";
|
|
25
23
|
import { detachArgv, detachSelf, waitForBootstrap } from "./launch.mjs";
|
|
26
|
-
import {
|
|
24
|
+
import { readJsonTolerant } from "../util.mjs";
|
|
27
25
|
|
|
28
26
|
const SYNC_OUTPUT_MAX_BYTES = 8000;
|
|
29
|
-
const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
30
|
-
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
31
|
-
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
32
|
-
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
33
27
|
|
|
34
28
|
const NOTE_KINDS = new Set([
|
|
35
29
|
"intent",
|
|
@@ -159,177 +153,6 @@ async function watch(campaignId, values) {
|
|
|
159
153
|
await watchCampaignWake(path, runsDir, { pollMs, once: values.once === true });
|
|
160
154
|
}
|
|
161
155
|
|
|
162
|
-
/**
|
|
163
|
-
* The watcher loop. Each line is announced through `notify`, which by default
|
|
164
|
-
* records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
|
|
165
|
-
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
|
|
166
|
-
* line already recorded is never re-sent. The injectable seams exist so a
|
|
167
|
-
* test can drive the loop deterministically.
|
|
168
|
-
*
|
|
169
|
-
* @param {string} campaignPath
|
|
170
|
-
* @param {string} runsDir
|
|
171
|
-
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
172
|
-
* @returns {Promise<void>}
|
|
173
|
-
*/
|
|
174
|
-
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
175
|
-
const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
|
|
176
|
-
const now = options.now ?? (() => Date.now());
|
|
177
|
-
const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
|
|
178
|
-
const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
|
|
179
|
-
const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
|
|
180
|
-
type: "attention",
|
|
181
|
-
campaignId: event.campaignId,
|
|
182
|
-
dedupeKey: event.dedupeKey,
|
|
183
|
-
summary: event.summary,
|
|
184
|
-
runId: event.runId ?? null,
|
|
185
|
-
nodeId: event.nodeId ?? null,
|
|
186
|
-
status: event.status ?? null,
|
|
187
|
-
errorCode: event.errorCode ?? null,
|
|
188
|
-
}));
|
|
189
|
-
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
190
|
-
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
191
|
-
emit(wakeCapabilityNotice());
|
|
192
|
-
/** @type {Map<string, string>} */
|
|
193
|
-
const runSignatures = new Map();
|
|
194
|
-
let lastActiveAt = now();
|
|
195
|
-
let first = true;
|
|
196
|
-
try {
|
|
197
|
-
for (;;) {
|
|
198
|
-
const campaign = readCampaign(campaignPath);
|
|
199
|
-
if (campaign.status !== "active") {
|
|
200
|
-
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
201
|
-
return;
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
205
|
-
* so a restart or a second watcher skips a line already recorded even
|
|
206
|
-
* when delivery is injected.
|
|
207
|
-
*
|
|
208
|
-
* @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
|
|
209
|
-
*/
|
|
210
|
-
const announce = async (dedupeKey, summary, extra = {}) => {
|
|
211
|
-
if (seen.has(dedupeKey)) return;
|
|
212
|
-
seen.add(dedupeKey);
|
|
213
|
-
const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
214
|
-
if (!appended.appended) return;
|
|
215
|
-
emit(summary);
|
|
216
|
-
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
217
|
-
};
|
|
218
|
-
let anyActive = false;
|
|
219
|
-
for (const runId of campaign.linkedRunIds) {
|
|
220
|
-
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
221
|
-
if (!status || !Array.isArray(status.nodes)) continue;
|
|
222
|
-
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
223
|
-
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
224
|
-
const previous = runSignatures.get(runId);
|
|
225
|
-
runSignatures.set(runId, signature);
|
|
226
|
-
if (!terminal) {
|
|
227
|
-
anyActive = true;
|
|
228
|
-
const runLock = readLock(join(runsDir, runId));
|
|
229
|
-
const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
|
|
230
|
-
if (stale && !first) {
|
|
231
|
-
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
if (!first && previous !== signature) {
|
|
235
|
-
for (const node of status.nodes) {
|
|
236
|
-
const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
|
|
237
|
-
|| (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
|
|
238
|
-
if (attention) {
|
|
239
|
-
const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
|
|
240
|
-
await announce(
|
|
241
|
-
key,
|
|
242
|
-
`campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
|
|
243
|
-
{ runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
if (terminal) {
|
|
249
|
-
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
const nowMs = now();
|
|
253
|
-
if (anyActive) lastActiveAt = nowMs;
|
|
254
|
-
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
255
|
-
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
256
|
-
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
257
|
-
}
|
|
258
|
-
first = false;
|
|
259
|
-
if (options.once === true) return;
|
|
260
|
-
await sleep(pollMs);
|
|
261
|
-
}
|
|
262
|
-
} finally {
|
|
263
|
-
lock.release();
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
const WATCH_LOCK_FILE = "watch.lock";
|
|
268
|
-
|
|
269
|
-
/**
|
|
270
|
-
* A durable campaign-watch lock, one watcher per campaign across processes.
|
|
271
|
-
* A live holder is never taken over; a dead or recycled pid's lock is stale
|
|
272
|
-
* and is replaced, so a restart after a crash is not blocked. The same
|
|
273
|
-
* liveness rule as the controller lock: a pid is dead only when the probe
|
|
274
|
-
* proves it.
|
|
275
|
-
*
|
|
276
|
-
* @param {string} campaignPath
|
|
277
|
-
* @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
|
|
278
|
-
*/
|
|
279
|
-
export function acquireWatchLock(campaignPath) {
|
|
280
|
-
const path = join(campaignPath, WATCH_LOCK_FILE);
|
|
281
|
-
/** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
|
|
282
|
-
let occupant = {};
|
|
283
|
-
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
284
|
-
const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
|
|
285
|
-
try {
|
|
286
|
-
const fd = openSync(path, "wx", 0o600);
|
|
287
|
-
try {
|
|
288
|
-
writeSync(fd, JSON.stringify(record));
|
|
289
|
-
} finally {
|
|
290
|
-
closeSync(fd);
|
|
291
|
-
}
|
|
292
|
-
return {
|
|
293
|
-
...record,
|
|
294
|
-
release() {
|
|
295
|
-
try {
|
|
296
|
-
unlinkSync(path);
|
|
297
|
-
} catch (error) {
|
|
298
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
299
|
-
}
|
|
300
|
-
},
|
|
301
|
-
};
|
|
302
|
-
} catch (error) {
|
|
303
|
-
if (errorCode(error) !== "EEXIST") throw error;
|
|
304
|
-
}
|
|
305
|
-
try {
|
|
306
|
-
occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
307
|
-
} catch {
|
|
308
|
-
occupant = {};
|
|
309
|
-
}
|
|
310
|
-
if (!watchLockStale(occupant)) {
|
|
311
|
-
throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
|
|
312
|
-
}
|
|
313
|
-
try {
|
|
314
|
-
unlinkSync(path);
|
|
315
|
-
} catch (error) {
|
|
316
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
throw new Error("campaign watch lock contention did not settle");
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
/**
|
|
323
|
-
* @param {{pid?: number, processStartToken?: string|null}} occupant
|
|
324
|
-
* @returns {boolean}
|
|
325
|
-
*/
|
|
326
|
-
function watchLockStale(occupant) {
|
|
327
|
-
if (typeof occupant.pid !== "number") return true;
|
|
328
|
-
if (!pidAlive(occupant.pid)) return true;
|
|
329
|
-
return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
|
|
333
156
|
/**
|
|
334
157
|
* @param {string} campaignId
|
|
335
158
|
* @param {CliValues} values
|
|
@@ -467,9 +290,38 @@ function close(campaignId, values) {
|
|
|
467
290
|
renderHandoff(path, runsDir);
|
|
468
291
|
process.stdout.write(`[campaign] ${closed.campaign.id} closed\n`);
|
|
469
292
|
process.stdout.write(`[campaign] ledger · docs/campaigns/${closed.campaign.id}/ledger · ${closed.ledgerFiles.length} files\n`);
|
|
293
|
+
reportRequirementClosure(closed.campaign.requirements ?? []);
|
|
470
294
|
if (syncAgentSignal(runsDir)) process.stdout.write(`[campaign] AGENTS.md signal updated\n`);
|
|
471
295
|
}
|
|
472
296
|
|
|
297
|
+
/**
|
|
298
|
+
* Say out loud what the close just wrote, and what it measured.
|
|
299
|
+
*
|
|
300
|
+
* The record is a correlation between the requirement ids a linked run's
|
|
301
|
+
* contract declared and the node snapshots that reached `done` carrying them.
|
|
302
|
+
* It reads the run directory and never the branch, so a requirement the
|
|
303
|
+
* operator delivered by hand -- because its node blocked, which happens --
|
|
304
|
+
* lands in the record as `open` while the work is on main. Measured
|
|
305
|
+
* 2026-09-22 on `availability-is-verified-not-assumed`: R4 read `open` with
|
|
306
|
+
* an empty `nodes` array and was merged in `a6a5d43`.
|
|
307
|
+
*
|
|
308
|
+
* Nothing printed this before, so the only way to meet the closure was to
|
|
309
|
+
* open the JSON, where `open` reads as "not delivered" instead of "no node of
|
|
310
|
+
* this campaign proved it". Naming the measurement is the whole fix: a close
|
|
311
|
+
* cannot know what it did not run.
|
|
312
|
+
*
|
|
313
|
+
* @param {import("../campaign/index.mjs").RequirementClosure[]} requirements
|
|
314
|
+
*/
|
|
315
|
+
function reportRequirementClosure(requirements) {
|
|
316
|
+
if (requirements.length === 0) return;
|
|
317
|
+
const open = requirements.filter((entry) => entry.status !== "covered");
|
|
318
|
+
const covered = requirements.length - open.length;
|
|
319
|
+
process.stdout.write(`[campaign] requirements · ${covered}/${requirements.length} carried by a node that reached done\n`);
|
|
320
|
+
if (open.length === 0) return;
|
|
321
|
+
process.stdout.write(`[campaign] no done node carried ${open.map((entry) => entry.requirementId).join(", ")}\n`);
|
|
322
|
+
process.stdout.write("[campaign] closure reads node snapshots, never the branch: a requirement delivered outside a node reads open here\n");
|
|
323
|
+
}
|
|
324
|
+
|
|
473
325
|
/**
|
|
474
326
|
* `campaign supervise <id>` (also spelled `supervise campaign <id>`): the
|
|
475
327
|
* idempotent re-invocation that drives the manifest. It takes the campaign's
|
package/src/cli/launch.mjs
CHANGED
|
@@ -55,8 +55,14 @@ export function detachSelf(command, target, extraArgs = []) {
|
|
|
55
55
|
* exactly as for every detached controller: a foreground launcher is the only
|
|
56
56
|
* moment an operator is present.
|
|
57
57
|
*
|
|
58
|
+
* `stdio` overrides the discarded default for a caller that has somewhere
|
|
59
|
+
* durable to put the child's output. A detached *run* wants the default: its
|
|
60
|
+
* bootstrap record is the channel, and a controller's real output belongs in
|
|
61
|
+
* the run directory. A detached *plan* has no bootstrap record, so discarding
|
|
62
|
+
* its stdio discarded the only account of why it died.
|
|
63
|
+
*
|
|
58
64
|
* @param {string[]} argv
|
|
59
|
-
* @param {{nonce?: string, env?: NodeJS.ProcessEnv}} [options]
|
|
65
|
+
* @param {{nonce?: string, env?: NodeJS.ProcessEnv, stdio?: import("node:child_process").StdioOptions}} [options]
|
|
60
66
|
* @returns {DetachedChild}
|
|
61
67
|
*/
|
|
62
68
|
export function detachArgv(argv, options = {}) {
|
|
@@ -74,7 +80,7 @@ export function detachArgv(argv, options = {}) {
|
|
|
74
80
|
// pending forever. There it also means DETACHED_PROCESS: no console
|
|
75
81
|
// window, which is what "ignore" stdio already implies.
|
|
76
82
|
detached: true,
|
|
77
|
-
stdio: "ignore",
|
|
83
|
+
stdio: options.stdio ?? "ignore",
|
|
78
84
|
}));
|
|
79
85
|
child.unref();
|
|
80
86
|
child.bootstrapNonce = nonce;
|
package/src/cli/plan.mjs
CHANGED
|
@@ -16,6 +16,7 @@ import { colorLevel, statusToken } from "./brand.mjs";
|
|
|
16
16
|
import { delay } from "../util.mjs";
|
|
17
17
|
import { runPlanningPipeline } from "../plan/pipeline.mjs";
|
|
18
18
|
import { campaignTree, runDirectory } from "../run/paths.mjs";
|
|
19
|
+
import { readCampaign } from "../campaign/record.mjs";
|
|
19
20
|
|
|
20
21
|
/** How often a foreground `plan` polls a launched stage's run directory. */
|
|
21
22
|
const DEFAULT_POLL_MS = 1_000;
|
|
@@ -125,7 +126,7 @@ export function loadVerificationSuites(path) {
|
|
|
125
126
|
|
|
126
127
|
/**
|
|
127
128
|
* @param {string} target
|
|
128
|
-
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, package?: string, detach?: boolean, json?: boolean}} values
|
|
129
|
+
* @param {{campaign?: string, phase?: string, "review-rounds"?: string, "approve-below"?: string, "runtime-defaults"?: string, runtimes?: string, verification?: string, package?: string, "targeted-fix"?: boolean, detach?: boolean, json?: boolean}} values
|
|
129
130
|
* @returns {Promise<void>}
|
|
130
131
|
*/
|
|
131
132
|
export async function planCli(target, values) {
|
|
@@ -151,7 +152,14 @@ export async function planCli(target, values) {
|
|
|
151
152
|
if (typeof values.runtimes === "string" && values.runtimes) argv.push("--runtimes", resolve(values.runtimes));
|
|
152
153
|
if (typeof values.verification === "string" && values.verification) argv.push("--verification", resolve(values.verification));
|
|
153
154
|
if (packageMode !== "implementation") argv.push("--package", packageMode);
|
|
155
|
+
if (values["targeted-fix"] === true) argv.push("--targeted-fix");
|
|
154
156
|
const failurePath = planBootstrapFailurePath(process.cwd(), campaignId, phase);
|
|
157
|
+
// Read the campaign before creating anything: the failure record lives
|
|
158
|
+
// inside the campaign tree, so a typo in --campaign would otherwise leave
|
|
159
|
+
// a campaign directory with no record in it for `discoverCampaigns` to
|
|
160
|
+
// find. The child reads it too; this is the launcher refusing what it can
|
|
161
|
+
// see for itself rather than detaching into a certain failure.
|
|
162
|
+
readCampaign(campaignTree(process.cwd(), campaignId));
|
|
155
163
|
mkdirSync(dirname(failurePath), { recursive: true });
|
|
156
164
|
rmSync(failurePath, { force: true });
|
|
157
165
|
const child = detachArgv(argv);
|
|
@@ -188,6 +196,7 @@ export async function planCli(target, values) {
|
|
|
188
196
|
runtimeDefaults,
|
|
189
197
|
runtimes,
|
|
190
198
|
verification,
|
|
199
|
+
targetedFix: values["targeted-fix"] === true,
|
|
191
200
|
packageMode,
|
|
192
201
|
launch: async (contractPath, contract) => {
|
|
193
202
|
const child = detachSelf("run", contractPath);
|
package/src/cli.mjs
CHANGED
|
@@ -8,7 +8,18 @@ export const VERIFICATION_LIMITS = Object.freeze({
|
|
|
8
8
|
stderrBytes: 16 * 1024,
|
|
9
9
|
maxCommands: 32,
|
|
10
10
|
maxRepeat: 8,
|
|
11
|
-
|
|
11
|
+
// A verification command may declare up to half an hour. It was 600s, which
|
|
12
|
+
// is below this repository's own suite: measured 2026-09-22, `npm test`
|
|
13
|
+
// takes 434-473s at the default parallelism on the author's machine and 650s
|
|
14
|
+
// on a Windows CI runner, and `node --test --test-concurrency=1 test/engine/`
|
|
15
|
+
// -- the way a packet's verification actually runs it -- takes 1035-1058s.
|
|
16
|
+
// So the one command that proves the engine could not be declared at all,
|
|
17
|
+
// and a packet author's way out was `--test-name-pattern`, which exits 0
|
|
18
|
+
// when it matches nothing (see AGENTS.md). A cap that pushes authors toward
|
|
19
|
+
// a proof that certifies nothing is worse than a longer runaway. The node's
|
|
20
|
+
// own wall clock (`contract.timeoutSec`, 2400s by default) still bounds the
|
|
21
|
+
// attempt above this.
|
|
22
|
+
maxTimeoutSec: 1_800,
|
|
12
23
|
stateStdoutBytes: 2 * 1024,
|
|
13
24
|
stateCommands: 16,
|
|
14
25
|
stateAttempts: 4,
|
package/src/engine/dispatch.mjs
CHANGED
|
@@ -34,7 +34,7 @@ import { invocationCost, invocationUsage } from "../run/usage.mjs";
|
|
|
34
34
|
import { logPaths, startProcess } from "./process.mjs";
|
|
35
35
|
import { readBoundedTail } from "./transcript.mjs";
|
|
36
36
|
import { mkdirSync, statSync } from "node:fs";
|
|
37
|
-
import { READ_BYTE_LIMIT, READ_LINE_LIMIT, normalizeProviderResult } from "../harnesses/index.mjs";
|
|
37
|
+
import { READ_BYTE_LIMIT, READ_LINE_LIMIT, harnessCapabilities, normalizeProviderResult } from "../harnesses/index.mjs";
|
|
38
38
|
import { writeJsonAtomic } from "../run/store.mjs";
|
|
39
39
|
import { judgeReaskInstruction, reviewMode } from "../contract/review-modes.mjs";
|
|
40
40
|
import { routeRuntimeForState, runtimeSnapshot } from "./failover.mjs";
|
|
@@ -153,12 +153,19 @@ function workerToolPolicy(runtime, node, workspace) {
|
|
|
153
153
|
* @returns {import("../harnesses/index.mjs").CommandOptions}
|
|
154
154
|
*/
|
|
155
155
|
function invocationCommandOptions(contract, node, state, runtime, phasePlan, runDir, lock, extra = {}) {
|
|
156
|
+
// A harness that streams its stdout proves liveness through the event
|
|
157
|
+
// monitor; a buffered one (zcode's `--json` writes only at exit) has one
|
|
158
|
+
// live surface left, its own log stream, and the adapter decides whether
|
|
159
|
+
// this dir means anything to it. Streaming harnesses get none: their log
|
|
160
|
+
// dir would be dead weight the engine never watches.
|
|
161
|
+
const streaming = harnessCapabilities(runtime).streamsOutput === true;
|
|
156
162
|
return {
|
|
157
163
|
...extra,
|
|
158
164
|
continuationId: runtime.capabilities.continuation === true ? phasePlan.continuationId : null,
|
|
159
165
|
// The attempt's request ceiling. An adapter that can enforce it natively
|
|
160
166
|
// takes it as a flag; the monitor enforces it for every streaming harness.
|
|
161
167
|
maxTurns: node.maxTurns ?? contract.maxTurns,
|
|
168
|
+
logDir: streaming ? null : join(runDir, "logs", `${node.id}.${state.attempt}.provider`),
|
|
162
169
|
};
|
|
163
170
|
}
|
|
164
171
|
/**
|
package/src/engine/gate.mjs
CHANGED
|
@@ -25,7 +25,8 @@
|
|
|
25
25
|
* It also exits when the release file's directory is gone (measured
|
|
26
26
|
* 2026-09-16: gate processes from a prior day's test runs, spawned into a
|
|
27
27
|
* temp directory the failed test never cleaned up, were still alive and
|
|
28
|
-
* waiting for a release file that could now never appear)
|
|
28
|
+
* waiting for a release file that could now never appear), and it keeps
|
|
29
|
+
* asking both of those questions after the provider starts, not only before.
|
|
29
30
|
*/
|
|
30
31
|
import { existsSync, readFileSync, statSync, openSync, closeSync, readSync, writeSync } from "node:fs";
|
|
31
32
|
import { dirname } from "node:path";
|
|
@@ -186,11 +187,32 @@ function releaseDirectoryGone() {
|
|
|
186
187
|
return !existsSync(dirname(releasePath));
|
|
187
188
|
}
|
|
188
189
|
|
|
190
|
+
/**
|
|
191
|
+
* How often the gate re-asks its two liveness questions once the provider is
|
|
192
|
+
* running. Before release the tick below asks them every 10ms, because it is
|
|
193
|
+
* also polling for the release file; after release it used to stop asking
|
|
194
|
+
* entirely, leaving the provider's own exit as the gate's only remaining
|
|
195
|
+
* liveness check. A controller that died without cleaning up, or a run
|
|
196
|
+
* directory deleted underneath a live attempt, therefore left the provider
|
|
197
|
+
* running with nobody watching -- the stranded-process shape ADR 0010
|
|
198
|
+
* describes, in the one window the pre-release check does not cover.
|
|
199
|
+
*
|
|
200
|
+
* A second, not ten milliseconds: this watches a provider that runs for
|
|
201
|
+
* minutes, and three syscalls a second is the whole cost of never stranding
|
|
202
|
+
* one.
|
|
203
|
+
*/
|
|
204
|
+
const WATCHDOG_INTERVAL_MS = 1_000;
|
|
205
|
+
|
|
189
206
|
const timer = setInterval(() => {
|
|
190
207
|
if (!parentAlive()) { clearInterval(timer); stopProvider(); return; }
|
|
191
208
|
if (releaseDirectoryGone()) { clearInterval(timer); stopProvider(); return; }
|
|
192
209
|
if (!existsSync(releasePath)) return;
|
|
193
210
|
clearInterval(timer);
|
|
211
|
+
const watchdog = setInterval(() => {
|
|
212
|
+
if (parentAlive() && !releaseDirectoryGone()) return;
|
|
213
|
+
clearInterval(watchdog);
|
|
214
|
+
stopProvider();
|
|
215
|
+
}, WATCHDOG_INTERVAL_MS);
|
|
194
216
|
const stdoutFd = openSync(config.stdoutPath, "wx", 0o600);
|
|
195
217
|
const stderrFd = openSync(config.stderrPath, "wx", 0o600);
|
|
196
218
|
const invocation = spawnInvocation(config.executable, config.args, { cwd: config.cwd });
|
|
@@ -207,6 +229,7 @@ const timer = setInterval(() => {
|
|
|
207
229
|
}
|
|
208
230
|
provider.once("error", () => process.exitCode = 127);
|
|
209
231
|
provider.once("close", (code) => {
|
|
232
|
+
clearInterval(watchdog);
|
|
210
233
|
capLog(config.stdoutPath, config.harness === "codex");
|
|
211
234
|
capLog(config.stderrPath);
|
|
212
235
|
process.exit(code ?? 1);
|
package/src/engine/lifecycle.mjs
CHANGED
|
@@ -147,8 +147,19 @@ export function terminalErrorCode(state) {
|
|
|
147
147
|
* `turn_limit` is here and not among the timeout codes below: a turn the CLI
|
|
148
148
|
* stopped itself at `--max-turns` exits cleanly with no seal yet, and the
|
|
149
149
|
* next dispatch seals its worktree as it does for any previous attempt.
|
|
150
|
+
*
|
|
151
|
+
* `incomplete_stream` is a stream that ended before its terminal envelope --
|
|
152
|
+
* Claude with no `result`, codex with no `turn.completed`, dsh with neither
|
|
153
|
+
* terminal record (`harnesses/protocol.mjs`). That is the provider's transport
|
|
154
|
+
* dying mid-turn, not the run's own doing, so it is the same class as
|
|
155
|
+
* `provider_error` and earns the same one retry. It was absent, and a worker
|
|
156
|
+
* whose provider dropped its connection parked on the first attempt while a
|
|
157
|
+
* worker whose provider returned an error envelope got a second one -- the
|
|
158
|
+
* harsher treatment for the less informative failure. The judge role is
|
|
159
|
+
* unaffected: `engine/settle-judge.mjs` routes this code to its own bounded
|
|
160
|
+
* re-ask before anything parks.
|
|
150
161
|
*/
|
|
151
|
-
export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
|
|
162
|
+
export const AUTO_RETRY_CODES = new Set(["judge_unavailable", "provider_error", "incomplete_stream", "stall_timeout", "wall_clock_timeout", "turn_limit"]);
|
|
152
163
|
|
|
153
164
|
/**
|
|
154
165
|
* Timeout codes earn their automatic retry only when phase 5b sealed work
|
package/src/engine/process.mjs
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* a judge decides, or when a run is done. That separation is the point: a stuck
|
|
8
8
|
* provider is killed by the same code whatever it was asked to do.
|
|
9
9
|
*/
|
|
10
|
-
import { boundedRegion, monitorInvocation } from "./transcript.mjs";
|
|
10
|
+
import { boundedRegion, latestLogWriteMs, monitorInvocation } from "./transcript.mjs";
|
|
11
11
|
import { closeSync, existsSync, fsyncSync, openSync, unlinkSync, writeFileSync, writeSync } from "node:fs";
|
|
12
12
|
import { dirname, join } from "node:path";
|
|
13
13
|
import { errorCode, errorMessage } from "../util.mjs";
|
|
@@ -36,7 +36,7 @@ import { killTarget } from "../host/platform.mjs";
|
|
|
36
36
|
/** @typedef {{prompt: string|null, stdout: string, stderr: string}} PathSet */
|
|
37
37
|
/** @typedef {{id: string, pid: number, processGroupId: number|null, processStartToken: string|null, harness: string, runtimeId: string|null, runtimeFingerprint?: string, revision?: number, phase: string, promptPath: string|null, stdoutPath: string, stderrPath: string, startedAt: string, deadlineAt: string|null, updatedAt: string, closedAt: string|null, exitCode: number|null, signal: string|null, status: "active"|"closed"|"terminated", executable: string, snapshotPath?: string, usage?: Usage, usageEstimated?: boolean, costUsd?: number|null, costProvenance?: "priced", runId?: string, campaignId?: string, nodeId?: string, attempt?: number, workspace?: string, worktreeBranch?: string|null, worktreeBaseSha?: string|null, planPhase?: string, role?: "worker"|"judge", model?: string, reasoning?: string|null, sandbox?: string|null, continuationId?: string|null, continuationMode?: "fresh"|"reuse"|"rotate", session?: import("../harnesses/session-metrics.mjs").SessionLedger|null}} Invocation */
|
|
38
38
|
/** @typedef {{pid: number|null, processGroupId?: number|null, processStartToken?: string|null}} InvocationProbe */
|
|
39
|
-
/** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, lastMonitorOffset?: number, observedOnce?: boolean, turnCapWarned?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
|
|
39
|
+
/** @typedef {{child: ChildProcess, contract: ValidatedContract, node: ValidatedNode, state: NodeSnapshot, runtime: HarnessRuntime & {id: string|null}, cwd: string, paths: PathSet, phase: string, invocation: Invocation, startedAt: string, startedTicks: bigint, progressTicks: bigint, lastOutputAt: number, closed: boolean, exitCode: number|null, signal: string|null, spawnError: Error|null, terminating: Promise<void>|null, gateConfigPath: string, gateReleasePath: string, logDir?: string|null, scopeBaseline?: unknown, scopeChecked?: boolean, scopeViolation?: boolean, resultMaterialization?: boolean, recoveryBaseline?: unknown, observeTimer?: ReturnType<typeof setInterval>, monitorOffset?: number, monitorParser?: import("../harnesses/session-metrics.mjs").SessionMetricsParser, lastEventCount?: number, lastMonitorOffset?: number, lastLogWriteMs?: number, observedOnce?: boolean, turnCapWarned?: boolean, onClose?: (invocation: Invocation) => void, onInvocationUpdate?: (invocation: Invocation) => void, onProgress?: (state: NodeSnapshot) => void}} Job */
|
|
40
40
|
/** @typedef {{graceMs?: number, killGraceMs?: number, escalate?: boolean, runDir?: string, kill?: (pid: number, signal: string|number) => unknown, child?: ChildProcess|null}} TerminateOptions */
|
|
41
41
|
|
|
42
42
|
const HERE = dirname(fileURLToPath(import.meta.url));
|
|
@@ -48,6 +48,16 @@ const GATE_PATH = join(HERE, "gate.mjs");
|
|
|
48
48
|
*/
|
|
49
49
|
export function startProcess({ contract, node, state, runtime, prompt, paths, phase, workspace = contract.cwd, commandOptions = {}, onInvocation, onInvocationUpdate, onProgress }) {
|
|
50
50
|
const command = providerCommand(runtime, prompt, commandOptions);
|
|
51
|
+
// The engine offers a log dir to every non-streaming harness; only an
|
|
52
|
+
// adapter that wants one creates it (zcode does, inside `command()` above,
|
|
53
|
+
// which has already run). Taking the offer is therefore observable, and the
|
|
54
|
+
// stall detector must key on the adapter's answer rather than on the
|
|
55
|
+
// engine's offer: `exec-jsonl` and `replay` are non-streaming too, declare
|
|
56
|
+
// no `stallTimeoutSec`, and were deliberately not stall-tracked at all. A
|
|
57
|
+
// path they never write to would otherwise have made them tracked against a
|
|
58
|
+
// log that stays empty forever -- measured 2026-09-22 against the contract
|
|
59
|
+
// default of 300s, a healthy exec-jsonl worker was killed as stalled.
|
|
60
|
+
const logDir = commandOptions.logDir && existsSync(commandOptions.logDir) ? commandOptions.logDir : null;
|
|
51
61
|
if (paths.prompt) writeFileSync(paths.prompt, prompt, { flag: "wx", mode: 0o600 });
|
|
52
62
|
const gateConfigPath = `${paths.prompt}.gate.json`;
|
|
53
63
|
const gateReleasePath = `${paths.prompt}.gate.release`;
|
|
@@ -125,6 +135,7 @@ export function startProcess({ contract, node, state, runtime, prompt, paths, ph
|
|
|
125
135
|
terminating: null,
|
|
126
136
|
gateConfigPath,
|
|
127
137
|
gateReleasePath,
|
|
138
|
+
logDir,
|
|
128
139
|
onInvocationUpdate,
|
|
129
140
|
onProgress,
|
|
130
141
|
};
|
|
@@ -433,11 +444,13 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
|
|
|
433
444
|
// calling tools is alive even when it writes no workspace file, and a
|
|
434
445
|
// buffered harness (zcode's `--json`) writes its whole transcript only at
|
|
435
446
|
// exit, so its mtime proves nothing. A harness that never streams is
|
|
436
|
-
// stall-tracked
|
|
437
|
-
//
|
|
447
|
+
// stall-tracked through its provider log dir when its adapter keeps one
|
|
448
|
+
// (zcode points the CLI's `ZCODE_LOG_DIR` at it), through a runtime that
|
|
449
|
+
// declares its own threshold, or by neither — the wall clock above is
|
|
450
|
+
// then the only budget it is held to.
|
|
438
451
|
const streaming = harnessCapabilities(job.runtime).streamsOutput;
|
|
439
452
|
const declaredStall = typeof (/** @type {{stallTimeoutSec?: unknown}} */ (job.runtime)?.stallTimeoutSec) === "number";
|
|
440
|
-
if (!streaming && !declaredStall) continue;
|
|
453
|
+
if (!streaming && !declaredStall && !job.logDir) continue;
|
|
441
454
|
const stallTimeoutSec = stallTimeoutSecFor(job.runtime, contract);
|
|
442
455
|
if (streaming) {
|
|
443
456
|
const monitored = monitorInvocation(job);
|
|
@@ -499,9 +512,24 @@ export async function detectStalls(contract, running, onTimeout, onProgress, onB
|
|
|
499
512
|
await onTimeout(job, "exhausted", limit);
|
|
500
513
|
continue;
|
|
501
514
|
}
|
|
502
|
-
} else
|
|
503
|
-
|
|
504
|
-
|
|
515
|
+
} else {
|
|
516
|
+
// The buffered-harness liveness signal: the CLI's log stream appending
|
|
517
|
+
// inside the attempt's log dir. An mtime that advances past the newest
|
|
518
|
+
// one this loop has seen is a write, and a write is progress — the same
|
|
519
|
+
// lower-bound discipline as the polling fixtures: it can only delay a
|
|
520
|
+
// stall verdict, never prove speed.
|
|
521
|
+
if (job.logDir) {
|
|
522
|
+
const written = latestLogWriteMs(job.logDir);
|
|
523
|
+
if (written > (job.lastLogWriteMs ?? 0)) {
|
|
524
|
+
job.lastLogWriteMs = written;
|
|
525
|
+
job.progressTicks = now;
|
|
526
|
+
job.lastOutputAt = Date.now();
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
if (job.observedOnce !== true) {
|
|
530
|
+
job.progressTicks = now;
|
|
531
|
+
job.lastOutputAt = Date.now();
|
|
532
|
+
}
|
|
505
533
|
}
|
|
506
534
|
job.observedOnce = true;
|
|
507
535
|
if (elapsedSeconds(job.progressTicks, now) < stallTimeoutSec) continue;
|
|
@@ -6,10 +6,11 @@
|
|
|
6
6
|
* because reading the log never touches the process, and because process.mjs
|
|
7
7
|
* crossed the 800-line ceiling carrying both jobs.
|
|
8
8
|
*/
|
|
9
|
-
import { closeSync, openSync, readFileSync, readSync, statSync } from "node:fs";
|
|
9
|
+
import { closeSync, openSync, readFileSync, readSync, readdirSync, statSync } from "node:fs";
|
|
10
10
|
import { Buffer } from "node:buffer";
|
|
11
11
|
import { SessionMetricsParser } from "../harnesses/session-metrics.mjs";
|
|
12
12
|
import { errorCode } from "../util.mjs";
|
|
13
|
+
import { join } from "node:path";
|
|
13
14
|
|
|
14
15
|
/** @typedef {import("./process.mjs").Job} Job */
|
|
15
16
|
|
|
@@ -143,3 +144,44 @@ function dropPartialLogLine(value) {
|
|
|
143
144
|
const newline = String(value).indexOf("\n");
|
|
144
145
|
return newline < 0 ? "" : String(value).slice(newline + 1);
|
|
145
146
|
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Newest write inside the attempt's provider log dir, 0 when nothing is there
|
|
150
|
+
* yet. The CLI lays its session logs out as files (it may nest a directory),
|
|
151
|
+
* so the scan walks three levels and reads only mtimes — never contents,
|
|
152
|
+
* which is the monitor's job for streaming harnesses and nobody else's.
|
|
153
|
+
* Every stat failure is an ordinary not-yet: a dir with nothing readable in
|
|
154
|
+
* it proves no liveness, which is exactly the right answer.
|
|
155
|
+
*
|
|
156
|
+
* Cheap enough to run on every tick: measured 2026-09-22 on macOS, a dir of
|
|
157
|
+
* 250 files across two levels scans in 0.80 ms, against the 1000 ms default
|
|
158
|
+
* `pollIntervalMs` — so the cost is under a tenth of a percent of one job's
|
|
159
|
+
* poll, and the scan reads no bytes.
|
|
160
|
+
*
|
|
161
|
+
* @param {string} dir
|
|
162
|
+
* @param {number} [depth]
|
|
163
|
+
* @returns {number}
|
|
164
|
+
*/
|
|
165
|
+
export function latestLogWriteMs(dir, depth = 0) {
|
|
166
|
+
let newest = 0;
|
|
167
|
+
let entries;
|
|
168
|
+
try {
|
|
169
|
+
entries = readdirSync(dir, { withFileTypes: true });
|
|
170
|
+
} catch {
|
|
171
|
+
return 0;
|
|
172
|
+
}
|
|
173
|
+
for (const entry of entries) {
|
|
174
|
+
const path = join(dir, entry.name);
|
|
175
|
+
if (entry.isDirectory()) {
|
|
176
|
+
if (depth < 3) newest = Math.max(newest, latestLogWriteMs(path, depth + 1));
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
try {
|
|
180
|
+
newest = Math.max(newest, statSync(path).mtimeMs);
|
|
181
|
+
} catch {
|
|
182
|
+
// Raced a rotation or a permission change: this entry proves nothing,
|
|
183
|
+
// the rest of the scan still does.
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return newest;
|
|
187
|
+
}
|
package/src/harnesses/index.mjs
CHANGED
|
@@ -95,7 +95,7 @@ export const READ_LINE_LIMIT = 1500;
|
|
|
95
95
|
*/
|
|
96
96
|
export const READ_BYTE_LIMIT = 32 * 1024;
|
|
97
97
|
|
|
98
|
-
/** @typedef {{schema?: object, schemaPath?: string, continuationId?: string|null, toolPolicy?: ToolPolicy, env?: Record<string, string>, maxTurns?: number}} CommandOptions */
|
|
98
|
+
/** @typedef {{schema?: object, schemaPath?: string, continuationId?: string|null, toolPolicy?: ToolPolicy, env?: Record<string, string>, maxTurns?: number, logDir?: string|null}} CommandOptions */
|
|
99
99
|
|
|
100
100
|
/** @typedef {{preferStructured?: boolean, exitCode?: number|null, signal?: string|null, stderr?: string}} NormalizeOptions */
|
|
101
101
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { accessSync, chmodSync, constants, existsSync, lstatSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
1
|
+
import { accessSync, chmodSync, constants, existsSync, lstatSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { delimiter, join } from "node:path";
|
|
4
4
|
import { normalizeZcodeResult, parseVersion } from "../protocol.mjs";
|
|
@@ -20,6 +20,20 @@ const ZCODE_MACOS_BUNDLE = Object.freeze({
|
|
|
20
20
|
cli: "/Applications/ZCode.app/Contents/Resources/glm/zcode.cjs",
|
|
21
21
|
});
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* Linux install layouts: an Electron `.deb`/`.rpm` unpacks the app under
|
|
25
|
+
* `/opt/<App>` (some packagers use `/usr/lib/<app>`) with the same
|
|
26
|
+
* `resources/` shape the macOS bundle has. None of these is documented by the
|
|
27
|
+
* vendor — `docs/harnesses/zcode-cli.md` records only the macOS layout — so the
|
|
28
|
+
* list is a probe, and `FABERUN_ZCODE_APP_DIR` names an install this list
|
|
29
|
+
* does not know.
|
|
30
|
+
*/
|
|
31
|
+
const ZCODE_LINUX_BUNDLES = Object.freeze([
|
|
32
|
+
{ electron: "/opt/ZCode/zcode", cli: "/opt/ZCode/resources/glm/zcode.cjs" },
|
|
33
|
+
{ electron: "/opt/zcode/zcode", cli: "/opt/zcode/resources/glm/zcode.cjs" },
|
|
34
|
+
{ electron: "/usr/lib/zcode/zcode", cli: "/usr/lib/zcode/resources/glm/zcode.cjs" },
|
|
35
|
+
]);
|
|
36
|
+
|
|
23
37
|
/** Provider id in the ZCODE_MODEL target; it also derives the auth env var name. */
|
|
24
38
|
const ZCODE_DEFAULT_PROVIDER = "glm";
|
|
25
39
|
|
|
@@ -68,7 +82,9 @@ export const zcodeHarness = {
|
|
|
68
82
|
// once at exit: a live worker node was killed at 420s stall_timeout with
|
|
69
83
|
// its stdout/stderr at zero bytes, while a completed 1m26s invocation's
|
|
70
84
|
// log held its full 26 lines only once the process exited. Stall
|
|
71
|
-
// detection must not watch this harness's stdout/stderr mtime.
|
|
85
|
+
// detection must not watch this harness's stdout/stderr mtime. Liveness
|
|
86
|
+
// comes from the CLI's own log stream instead — see `command()`'s
|
|
87
|
+
// ZCODE_LOG_DIR wiring, which the engine's stall clock watches.
|
|
72
88
|
streamsOutput: false,
|
|
73
89
|
// Unmeasured: no run has proven whether zcode's sandbox can signal child
|
|
74
90
|
// processes or read the process table.
|
|
@@ -134,6 +150,19 @@ export const zcodeHarness = {
|
|
|
134
150
|
// resolves (e.g. GLM_API_KEY); an unresolved token is omitted, not blanked.
|
|
135
151
|
const apiKeyVar = providerApiKeyVar(provider);
|
|
136
152
|
if (token !== null && apiKeyVar !== null) env[apiKeyVar] = token;
|
|
153
|
+
// The one live surface a buffered harness has: the CLI's own log stream.
|
|
154
|
+
// `--json` writes stdout only at exit, but `ZCODE_LOG_DIR` in `json`
|
|
155
|
+
// format receives session events as they happen, so the engine's stall
|
|
156
|
+
// clock can watch the log dir instead of holding the attempt to the wall
|
|
157
|
+
// clock alone. The engine supplies one dir per attempt (see
|
|
158
|
+
// `invocationCommandOptions`); the adapter creates it and points the CLI
|
|
159
|
+
// at it, and console logging stays off so stderr stays a pure error path.
|
|
160
|
+
if (options.logDir) {
|
|
161
|
+
mkdirSync(options.logDir, { recursive: true });
|
|
162
|
+
env.ZCODE_LOG_DIR = options.logDir;
|
|
163
|
+
env.ZCODE_LOG_FORMAT = "json";
|
|
164
|
+
env.ZCODE_LOG_CONSOLE = "false";
|
|
165
|
+
}
|
|
137
166
|
return { executable: this.executable(runtime), args, promptTransport: "argv", input: null, env };
|
|
138
167
|
},
|
|
139
168
|
|
|
@@ -187,8 +216,8 @@ export function ensureZcodeAvailable(options = {}) {
|
|
|
187
216
|
const env = options.env ?? process.env;
|
|
188
217
|
const pathDirs = options.pathDirs ?? (env.PATH ?? "").split(delimiter).filter(Boolean);
|
|
189
218
|
if (resolvesOnPath(pathDirs, ZCODE_BIN_NAME)) return;
|
|
190
|
-
const bundle = options.bundle ??
|
|
191
|
-
if (!existsSync(bundle.electron) || !existsSync(bundle.cli)) return;
|
|
219
|
+
const bundle = options.bundle ?? zcodeBundle(env);
|
|
220
|
+
if (!bundle || !existsSync(bundle.electron) || !existsSync(bundle.cli)) return;
|
|
192
221
|
const body = zcodeShim(bundle);
|
|
193
222
|
for (const dir of shimDirs(options.home ?? homedir())) {
|
|
194
223
|
if (!pathDirs.includes(dir)) continue;
|
|
@@ -200,6 +229,22 @@ export function ensureZcodeAvailable(options = {}) {
|
|
|
200
229
|
}
|
|
201
230
|
}
|
|
202
231
|
|
|
232
|
+
/**
|
|
233
|
+
* The bundle this host has, if any: an explicit `FABERUN_ZCODE_APP_DIR`
|
|
234
|
+
* override, the macOS app path on darwin, or the first probed Linux layout
|
|
235
|
+
* whose two paths both exist. A host with none gets null — the same "not
|
|
236
|
+
* installed" answer every probe below returns.
|
|
237
|
+
*
|
|
238
|
+
* @param {Record<string, string|undefined>} env
|
|
239
|
+
* @returns {{electron: string, cli: string}|null}
|
|
240
|
+
*/
|
|
241
|
+
function zcodeBundle(env) {
|
|
242
|
+
const appDir = env.FABERUN_ZCODE_APP_DIR;
|
|
243
|
+
if (appDir) return { electron: join(appDir, "zcode"), cli: join(appDir, "resources", "glm", "zcode.cjs") };
|
|
244
|
+
if (process.platform === "darwin") return ZCODE_MACOS_BUNDLE;
|
|
245
|
+
return ZCODE_LINUX_BUNDLES.find((bundle) => existsSync(bundle.electron) && existsSync(bundle.cli)) ?? null;
|
|
246
|
+
}
|
|
247
|
+
|
|
203
248
|
/**
|
|
204
249
|
* The shim body. It runs the bundle through the app's own Electron binary as
|
|
205
250
|
* node because the CLI mis-handles its response path under a system node
|
package/src/plan/pipeline.mjs
CHANGED
|
@@ -82,13 +82,17 @@ export const DEFAULT_NODE_BUDGET_MS = 600_000;
|
|
|
82
82
|
const APPROVE_BELOW_VALUES = new Set(["standard", "high", "none"]);
|
|
83
83
|
|
|
84
84
|
/**
|
|
85
|
-
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
|
|
85
|
+
* @param {{specPath: string, campaignId: string, phase: string, cwd?: string, reviewRounds?: number, approveBelow?: ApproveBelow, runtimeDefaults?: {worker?: string, judge?: string}, runtimes: Record<string, JsonObject>, verification?: VerificationSuites, packageMode?: import("./sizing.mjs").PackageMode, targetedFix?: boolean, launch: LaunchFn, wait: WaitFn, ask?: AskFn}} options
|
|
86
|
+
* `targetedFix` allows a plan with a single node. Sizing refuses one by
|
|
87
|
+
* default because a phase that decomposes into one node is usually a plan
|
|
88
|
+
* that was never decomposed; a targeted fix is the case where one node is
|
|
89
|
+
* the honest answer, and the operator says so.
|
|
86
90
|
* @returns {Promise<FrozenPipelineResult|ContestedPipelineResult>}
|
|
87
91
|
*/
|
|
88
92
|
export async function runPlanningPipeline(options) {
|
|
89
93
|
const {
|
|
90
94
|
specPath, campaignId, phase, runtimes, launch, wait,
|
|
91
|
-
reviewRounds = 2, runtimeDefaults = {}, verification = {},
|
|
95
|
+
reviewRounds = 2, runtimeDefaults = {}, verification = {}, targetedFix = false,
|
|
92
96
|
} = options;
|
|
93
97
|
const ask = options.ask ?? askPlanningRuntimes;
|
|
94
98
|
// Implementation work is sized by what it writes; exploratory work -- an
|
|
@@ -258,7 +262,7 @@ export async function runPlanningPipeline(options) {
|
|
|
258
262
|
stage = "sizing";
|
|
259
263
|
const sizing = applySizingRules(
|
|
260
264
|
{ nodes: currentPlan.nodes.map(toSizingNode), justification: currentPlan.justification },
|
|
261
|
-
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)) },
|
|
265
|
+
{ nodeBudgetMs: DEFAULT_NODE_BUDGET_MS, facts: repoFacts, minWriteFiles: MIN_WRITE_FILES, turnCeiling: DEFAULT_MAX_TURNS, packageMode, readVolume: (path) => fileLineCount(join(cwd, path)), targetedFix },
|
|
262
266
|
);
|
|
263
267
|
stage = "routing";
|
|
264
268
|
const routing = resolveRuntimes(sizing.plan.nodes, {
|