faberun 0.19.1 → 0.19.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/faberun/references/contract.md +17 -5
- package/src/campaign/index.mjs +56 -1
- package/src/campaign/watch.mjs +197 -0
- package/src/cli/campaign.mjs +32 -180
- package/src/cli/launch.mjs +8 -2
- package/src/cli/plan.mjs +173 -27
- package/src/cli/spec.mjs +17 -7
- package/src/cli.mjs +2 -0
- package/src/contract/definition-of-done.mjs +50 -0
- package/src/contract/final-verification.mjs +21 -0
- package/src/contract/index.mjs +20 -8
- package/src/contract/verification.mjs +75 -3
- package/src/engine/dispatch.mjs +2 -1
- package/src/engine/gate.mjs +24 -1
- package/src/engine/lifecycle.mjs +12 -1
- package/src/engine/process.mjs +21 -1
- package/src/engine/scheduler.mjs +2 -2
- package/src/plan/pipeline.mjs +32 -5
- package/src/plan/proof-run.mjs +121 -0
- package/src/plan/repo-facts.mjs +5 -39
- package/src/plan/sizing.mjs +72 -4
- package/src/plan/spec.mjs +25 -1
- package/src/plan/template.mjs +24 -2
- package/src/report/final.mjs +44 -7
- package/src/report/render.mjs +98 -139
- package/src/report/role-usage.mjs +145 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "faberun",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.3",
|
|
4
4
|
"description": "Faberun is a development orchestration system that turns intent into verified software: harness- and model-agnostic, it keeps the intent, coordinates the work, verifies the result and decides what happens next.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -46,8 +46,10 @@ typecheck`). Schema version is `3`.
|
|
|
46
46
|
```
|
|
47
47
|
|
|
48
48
|
Every `definitionOfDone` item declares `id`, `text`, and how it is proven:
|
|
49
|
-
`proof.kind` `command` (re-runs the command
|
|
50
|
-
|
|
49
|
+
`proof.kind` `command` (re-runs the command through a shell, bounded by the
|
|
50
|
+
node's own `timeoutSec`; quote a flag value containing spaces, which
|
|
51
|
+
`verification`'s argv does not need and a shell splits) or `path` (a file must
|
|
52
|
+
exist), or `judgment: true` for the judge. `proof: {
|
|
51
53
|
kind: "verification", ref: <index> }` reuses a `verification` entry's already
|
|
52
54
|
recorded result by position instead of re-running it — never by comparing argv
|
|
53
55
|
strings, since a joined argv loses shell semantics. A schema-1 string item is
|
|
@@ -95,8 +97,11 @@ discovery packet has empty `writeFiles`; with an empty `readFiles` it may
|
|
|
95
97
|
read the repository read-only to produce an execution packet — the one
|
|
96
98
|
exception to closed scope — otherwise it is closed to the listed files. Each
|
|
97
99
|
`verification` entry is `{argv, cwd?, timeoutSec? (default 120, max 600),
|
|
98
|
-
repeat? (default 1, max 8), env?}` — at most 32 commands, 64
|
|
99
|
-
KiB argv bytes per command. `
|
|
100
|
+
repeat? (default 1, max 8), env?, requirementId?}` — at most 32 commands, 64
|
|
101
|
+
argv items, 32 KiB argv bytes per command. `requirementId` names the spec
|
|
102
|
+
requirement this command proves, changing nothing about how it runs: it is
|
|
103
|
+
what lets `contract validate` report two copies of one proof that have stopped
|
|
104
|
+
agreeing. `env` declares variable *names* only; values
|
|
100
105
|
never travel in the packet. `prompt`/`promptFile` are
|
|
101
106
|
rejected; a node has `taskPacket` or `taskPacketFile`, never both. Measure a
|
|
102
107
|
candidate command's real duration before naming it in `verification` or a
|
|
@@ -289,7 +294,14 @@ only for a harness declaring `streamsOutput` (true for `codex`, `claude`,
|
|
|
289
294
|
others fall back to `timeoutSec` alone.
|
|
290
295
|
`timeoutSec` (default 2400s) caps one invocation and may be overridden per
|
|
291
296
|
node; a node is bounded by `(1 + maxRevisions) × 2 × timeoutSec`. Both clocks
|
|
292
|
-
are monotonic and pause with host suspend.
|
|
297
|
+
are monotonic and pause with host suspend.
|
|
298
|
+
`maxTurns` (default 150) bounds something else: the *provider requests* one
|
|
299
|
+
attempt may make, overridable per node. Reaching it ends the attempt with
|
|
300
|
+
`errorCode: turn_limit`, sealed then retried once, the spend already spent. It
|
|
301
|
+
bites the nodes that read much and write little — review, synthesis — so raise
|
|
302
|
+
it there; raising `timeoutSec` does not help. The controller says so once at
|
|
303
|
+
80%, and `usage.jsonl` records each invocation's `session.requests`.
|
|
304
|
+
`maxParallel` above 1 dispatches
|
|
293
305
|
every dependency-ready node concurrently, each into its own attempt
|
|
294
306
|
worktree; integration stays serialized. Nodes of one phase need no edge
|
|
295
307
|
between them: a continuation a live invocation already claims is never
|
package/src/campaign/index.mjs
CHANGED
|
@@ -132,9 +132,14 @@ export function closeCampaign(campaignPath, { at = new Date().toISOString(), eve
|
|
|
132
132
|
requireTimestamp(at, "at");
|
|
133
133
|
const campaign = readCampaign(campaignPath);
|
|
134
134
|
if (campaign.status === "closed") throw new Error(`campaign already closed: ${campaign.id}`);
|
|
135
|
-
|
|
135
|
+
const journal = readJournalForDedupe(campaignPath);
|
|
136
|
+
if (!journal.some((entry) => entry.type === "retrospective")) {
|
|
136
137
|
throw new Error(`campaign ${campaign.id} has no recorded retrospective; record one with note --kind retrospective before close`);
|
|
137
138
|
}
|
|
139
|
+
const unacknowledged = unacknowledgedAdvisories(campaignPath, campaign, journal);
|
|
140
|
+
if (unacknowledged.length) {
|
|
141
|
+
throw new Error(`campaign ${campaign.id} has judge findings no note has answered: ${unacknowledged.join("; ")}. Read them with \`faberun findings <run-dir>\`, then name the node in a note (\`campaign note ${campaign.id} --kind outcome --run-id <run-id> --text "...<node>..."\`) before close`);
|
|
142
|
+
}
|
|
138
143
|
const repoRoot = campaignRepoRoot(campaignPath);
|
|
139
144
|
const ledgerFiles = preserveCampaignLedger(campaignPath, repoRoot);
|
|
140
145
|
// The closure travels on the record itself, computed in one deterministic
|
|
@@ -147,6 +152,56 @@ export function closeCampaign(campaignPath, { at = new Date().toISOString(), eve
|
|
|
147
152
|
return { path: campaignPath, campaign: closed, ledgerFiles };
|
|
148
153
|
}
|
|
149
154
|
|
|
155
|
+
/**
|
|
156
|
+
* Judge findings on nodes the gate accepted, which no journal note names.
|
|
157
|
+
*
|
|
158
|
+
* A gate that accepts a node whose findings sit below `failOn` is correct and
|
|
159
|
+
* documented. What was wrong is that the campaign could then be closed with
|
|
160
|
+
* the finding never read by anyone: measured 2026-09-21, a synthesis node's
|
|
161
|
+
* `gate.verdict` was `fail` with a real finding, `STATUS` said `passed`, and
|
|
162
|
+
* the campaign's own retrospective recorded that every gate passed first
|
|
163
|
+
* time. A close is the last moment the claim can still be corrected.
|
|
164
|
+
*
|
|
165
|
+
* Acknowledgement is a note that names the node id. The node id is the
|
|
166
|
+
* identifier the run, the contract and the findings output all already use,
|
|
167
|
+
* so matching on it asks the operator for nothing new; matching on a
|
|
168
|
+
* finding's prose would be matching on text the judge wrote, which is not a
|
|
169
|
+
* stable name.
|
|
170
|
+
*
|
|
171
|
+
* @param {string} campaignPath
|
|
172
|
+
* @param {Campaign} campaign
|
|
173
|
+
* @param {{type: string, text?: unknown}[]} journal
|
|
174
|
+
* @returns {string[]} one `runId/nodeId (N findings, maxSeverity)` per unanswered node, sorted
|
|
175
|
+
*/
|
|
176
|
+
function unacknowledgedAdvisories(campaignPath, campaign, journal) {
|
|
177
|
+
const runsDir = resolve(campaignPath, "..", "..");
|
|
178
|
+
const noteText = journal
|
|
179
|
+
.filter((entry) => typeof entry.text === "string")
|
|
180
|
+
.map((entry) => /** @type {string} */ (entry.text))
|
|
181
|
+
.join("\n");
|
|
182
|
+
/** @type {string[]} */
|
|
183
|
+
const pending = [];
|
|
184
|
+
for (const runId of [...campaign.linkedRunIds].sort()) {
|
|
185
|
+
const contract = readRunJson(join(runsDir, runId, "contract.json"));
|
|
186
|
+
const nodes = contract !== null && Array.isArray(contract.nodes) ? /** @type {JsonObject[]} */ (contract.nodes) : [];
|
|
187
|
+
for (const node of nodes) {
|
|
188
|
+
const nodeId = typeof node.id === "string" ? node.id : "";
|
|
189
|
+
if (!nodeId) continue;
|
|
190
|
+
const snapshot = readRunJson(join(runsDir, runId, "nodes", `${nodeId}.json`));
|
|
191
|
+
if (snapshot === null) continue;
|
|
192
|
+
// Only an accepted node: on a rejected one the findings are the
|
|
193
|
+
// rejection itself, and the run already refuses to read as finished.
|
|
194
|
+
if (snapshot.status !== "done" && snapshot.status !== "no-op") continue;
|
|
195
|
+
const gate = /** @type {JsonObject|null|undefined} */ (snapshot.gate);
|
|
196
|
+
const findings = gate && Array.isArray(gate.findings) ? gate.findings : [];
|
|
197
|
+
if (findings.length === 0) continue;
|
|
198
|
+
if (noteText.includes(nodeId)) continue;
|
|
199
|
+
pending.push(`${runId}/${nodeId} (${findings.length} ${findings.length === 1 ? "finding" : "findings"}, ${typeof gate?.maxSeverity === "string" ? gate.maxSeverity : "unknown"})`);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return pending.sort();
|
|
203
|
+
}
|
|
204
|
+
|
|
150
205
|
/**
|
|
151
206
|
* The requirement closure a close records: one entry per requirement id the
|
|
152
207
|
* linked runs' contracts declared, correlated only by the identifiers the runs
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `campaign watch --wake`: the loop that turns a campaign's own records into
|
|
3
|
+
* the lines an operator is woken with, and the single-watcher lock that keeps
|
|
4
|
+
* two of them from doubling every wake.
|
|
5
|
+
*
|
|
6
|
+
* Separate from `cli/campaign.mjs` because that file owns argv, dispatch and
|
|
7
|
+
* usage for the campaign verb, and this is neither: it is a long-running
|
|
8
|
+
* supervisor with a poll interval, an idle policy, a durable dedupe through
|
|
9
|
+
* the inbox, and a lock with its own staleness rule. `cli/campaign.mjs` keeps
|
|
10
|
+
* the thin `watch` operation that reads the flags and calls in here.
|
|
11
|
+
*/
|
|
12
|
+
import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
|
|
13
|
+
import { join } from "node:path";
|
|
14
|
+
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
15
|
+
import { readCampaign } from "./record.mjs";
|
|
16
|
+
import { notifyQueueFor } from "../engine/notify-queue.mjs";
|
|
17
|
+
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
18
|
+
import { errorCode, readJsonTolerant } from "../util.mjs";
|
|
19
|
+
|
|
20
|
+
/** How often the watcher re-reads the campaign when the operator named no interval. */
|
|
21
|
+
export const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
22
|
+
|
|
23
|
+
/** How long without a material event before the watcher reports the campaign idle. */
|
|
24
|
+
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
25
|
+
|
|
26
|
+
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
27
|
+
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The watcher loop. Each line is announced through `notify`, which by default
|
|
31
|
+
* records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
|
|
32
|
+
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
|
|
33
|
+
* line already recorded is never re-sent. The injectable seams exist so a
|
|
34
|
+
* test can drive the loop deterministically.
|
|
35
|
+
*
|
|
36
|
+
* @param {string} campaignPath
|
|
37
|
+
* @param {string} runsDir
|
|
38
|
+
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
39
|
+
* @returns {Promise<void>}
|
|
40
|
+
*/
|
|
41
|
+
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
42
|
+
const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
|
|
43
|
+
const now = options.now ?? (() => Date.now());
|
|
44
|
+
const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
|
|
45
|
+
const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
|
|
46
|
+
const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
|
|
47
|
+
type: "attention",
|
|
48
|
+
campaignId: event.campaignId,
|
|
49
|
+
dedupeKey: event.dedupeKey,
|
|
50
|
+
summary: event.summary,
|
|
51
|
+
runId: event.runId ?? null,
|
|
52
|
+
nodeId: event.nodeId ?? null,
|
|
53
|
+
status: event.status ?? null,
|
|
54
|
+
errorCode: event.errorCode ?? null,
|
|
55
|
+
}));
|
|
56
|
+
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
57
|
+
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
58
|
+
emit(wakeCapabilityNotice());
|
|
59
|
+
/** @type {Map<string, string>} */
|
|
60
|
+
const runSignatures = new Map();
|
|
61
|
+
let lastActiveAt = now();
|
|
62
|
+
let first = true;
|
|
63
|
+
try {
|
|
64
|
+
for (;;) {
|
|
65
|
+
const campaign = readCampaign(campaignPath);
|
|
66
|
+
if (campaign.status !== "active") {
|
|
67
|
+
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
72
|
+
* so a restart or a second watcher skips a line already recorded even
|
|
73
|
+
* when delivery is injected.
|
|
74
|
+
*
|
|
75
|
+
* @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
|
|
76
|
+
*/
|
|
77
|
+
const announce = async (dedupeKey, summary, extra = {}) => {
|
|
78
|
+
if (seen.has(dedupeKey)) return;
|
|
79
|
+
seen.add(dedupeKey);
|
|
80
|
+
const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
81
|
+
if (!appended.appended) return;
|
|
82
|
+
emit(summary);
|
|
83
|
+
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
84
|
+
};
|
|
85
|
+
let anyActive = false;
|
|
86
|
+
for (const runId of campaign.linkedRunIds) {
|
|
87
|
+
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
88
|
+
if (!status || !Array.isArray(status.nodes)) continue;
|
|
89
|
+
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
90
|
+
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
91
|
+
const previous = runSignatures.get(runId);
|
|
92
|
+
runSignatures.set(runId, signature);
|
|
93
|
+
if (!terminal) {
|
|
94
|
+
anyActive = true;
|
|
95
|
+
const runLock = readLock(join(runsDir, runId));
|
|
96
|
+
const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
|
|
97
|
+
if (stale && !first) {
|
|
98
|
+
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (!first && previous !== signature) {
|
|
102
|
+
for (const node of status.nodes) {
|
|
103
|
+
const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
|
|
104
|
+
|| (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
|
|
105
|
+
if (attention) {
|
|
106
|
+
const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
|
|
107
|
+
await announce(
|
|
108
|
+
key,
|
|
109
|
+
`campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
|
|
110
|
+
{ runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
if (terminal) {
|
|
116
|
+
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
const nowMs = now();
|
|
120
|
+
if (anyActive) lastActiveAt = nowMs;
|
|
121
|
+
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
122
|
+
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
123
|
+
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
124
|
+
}
|
|
125
|
+
first = false;
|
|
126
|
+
if (options.once === true) return;
|
|
127
|
+
await sleep(pollMs);
|
|
128
|
+
}
|
|
129
|
+
} finally {
|
|
130
|
+
lock.release();
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const WATCH_LOCK_FILE = "watch.lock";
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* A durable campaign-watch lock, one watcher per campaign across processes.
|
|
138
|
+
* A live holder is never taken over; a dead or recycled pid's lock is stale
|
|
139
|
+
* and is replaced, so a restart after a crash is not blocked. The same
|
|
140
|
+
* liveness rule as the controller lock: a pid is dead only when the probe
|
|
141
|
+
* proves it.
|
|
142
|
+
*
|
|
143
|
+
* @param {string} campaignPath
|
|
144
|
+
* @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
|
|
145
|
+
*/
|
|
146
|
+
export function acquireWatchLock(campaignPath) {
|
|
147
|
+
const path = join(campaignPath, WATCH_LOCK_FILE);
|
|
148
|
+
/** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
|
|
149
|
+
let occupant = {};
|
|
150
|
+
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
151
|
+
const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
|
|
152
|
+
try {
|
|
153
|
+
const fd = openSync(path, "wx", 0o600);
|
|
154
|
+
try {
|
|
155
|
+
writeSync(fd, JSON.stringify(record));
|
|
156
|
+
} finally {
|
|
157
|
+
closeSync(fd);
|
|
158
|
+
}
|
|
159
|
+
return {
|
|
160
|
+
...record,
|
|
161
|
+
release() {
|
|
162
|
+
try {
|
|
163
|
+
unlinkSync(path);
|
|
164
|
+
} catch (error) {
|
|
165
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
};
|
|
169
|
+
} catch (error) {
|
|
170
|
+
if (errorCode(error) !== "EEXIST") throw error;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
174
|
+
} catch {
|
|
175
|
+
occupant = {};
|
|
176
|
+
}
|
|
177
|
+
if (!watchLockStale(occupant)) {
|
|
178
|
+
throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
|
|
179
|
+
}
|
|
180
|
+
try {
|
|
181
|
+
unlinkSync(path);
|
|
182
|
+
} catch (error) {
|
|
183
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
throw new Error("campaign watch lock contention did not settle");
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* @param {{pid?: number, processStartToken?: string|null}} occupant
|
|
191
|
+
* @returns {boolean}
|
|
192
|
+
*/
|
|
193
|
+
function watchLockStale(occupant) {
|
|
194
|
+
if (typeof occupant.pid !== "number") return true;
|
|
195
|
+
if (!pidAlive(occupant.pid)) return true;
|
|
196
|
+
return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
|
|
197
|
+
}
|
package/src/cli/campaign.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import { readFileSync } from "node:fs";
|
|
3
3
|
import { join, resolve } from "node:path";
|
|
4
4
|
import { parseArgs as parseFlags } from "node:util";
|
|
5
5
|
import {
|
|
@@ -11,25 +11,19 @@ import {
|
|
|
11
11
|
resolveCampaign,
|
|
12
12
|
} from "../campaign/index.mjs";
|
|
13
13
|
import { addContract, replaceContract } from "./campaign-contract.mjs";
|
|
14
|
-
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
15
14
|
import { runsRoot } from "../run/paths.mjs";
|
|
16
15
|
import { syncAgentSignal } from "../repo/signal.mjs";
|
|
17
16
|
import { acknowledgeJournalEvent, appendJournal, appendSeatAllowanceEvent, readJournal, watchJournal } from "../campaign/journal.mjs";
|
|
18
17
|
import { driveCampaignChain } from "../campaign/chain.mjs";
|
|
19
18
|
import { unparkCampaign } from "../campaign/unpark.mjs";
|
|
20
19
|
import { readCampaign } from "../campaign/record.mjs";
|
|
21
|
-
import {
|
|
22
|
-
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
20
|
+
import { DEFAULT_WAKE_POLL_MS, watchCampaignWake } from "../campaign/watch.mjs";
|
|
23
21
|
import { allowanceEventFields, sampleAllowance } from "../seat/allowance.mjs";
|
|
24
22
|
import { detectOperatorHarness } from "../seat/harnesses.mjs";
|
|
25
23
|
import { detachArgv, detachSelf, waitForBootstrap } from "./launch.mjs";
|
|
26
|
-
import {
|
|
24
|
+
import { readJsonTolerant } from "../util.mjs";
|
|
27
25
|
|
|
28
26
|
const SYNC_OUTPUT_MAX_BYTES = 8000;
|
|
29
|
-
const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
30
|
-
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
31
|
-
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
32
|
-
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
33
27
|
|
|
34
28
|
const NOTE_KINDS = new Set([
|
|
35
29
|
"intent",
|
|
@@ -159,177 +153,6 @@ async function watch(campaignId, values) {
|
|
|
159
153
|
await watchCampaignWake(path, runsDir, { pollMs, once: values.once === true });
|
|
160
154
|
}
|
|
161
155
|
|
|
162
|
-
/**
|
|
163
|
-
* The watcher loop. Each line is announced through `notify`, which by default
|
|
164
|
-
* records it in `<runs-dir>/inbox.jsonl` and delivers it to the campaign's
|
|
165
|
-
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so a
|
|
166
|
-
* line already recorded is never re-sent. The injectable seams exist so a
|
|
167
|
-
* test can drive the loop deterministically.
|
|
168
|
-
*
|
|
169
|
-
* @param {string} campaignPath
|
|
170
|
-
* @param {string} runsDir
|
|
171
|
-
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
172
|
-
* @returns {Promise<void>}
|
|
173
|
-
*/
|
|
174
|
-
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
175
|
-
const pollMs = options.pollMs ?? DEFAULT_WAKE_POLL_MS;
|
|
176
|
-
const now = options.now ?? (() => Date.now());
|
|
177
|
-
const sleep = options.sleep ?? ((ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)));
|
|
178
|
-
const emit = options.emit ?? ((line) => process.stdout.write(`${line}\n`));
|
|
179
|
-
const notify = options.notify ?? ((event) => notifyQueueFor(campaignPath).enqueue({
|
|
180
|
-
type: "attention",
|
|
181
|
-
campaignId: event.campaignId,
|
|
182
|
-
dedupeKey: event.dedupeKey,
|
|
183
|
-
summary: event.summary,
|
|
184
|
-
runId: event.runId ?? null,
|
|
185
|
-
nodeId: event.nodeId ?? null,
|
|
186
|
-
status: event.status ?? null,
|
|
187
|
-
errorCode: event.errorCode ?? null,
|
|
188
|
-
}));
|
|
189
|
-
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
190
|
-
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
191
|
-
emit(wakeCapabilityNotice());
|
|
192
|
-
/** @type {Map<string, string>} */
|
|
193
|
-
const runSignatures = new Map();
|
|
194
|
-
let lastActiveAt = now();
|
|
195
|
-
let first = true;
|
|
196
|
-
try {
|
|
197
|
-
for (;;) {
|
|
198
|
-
const campaign = readCampaign(campaignPath);
|
|
199
|
-
if (campaign.status !== "active") {
|
|
200
|
-
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
201
|
-
return;
|
|
202
|
-
}
|
|
203
|
-
/**
|
|
204
|
-
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
205
|
-
* so a restart or a second watcher skips a line already recorded even
|
|
206
|
-
* when delivery is injected.
|
|
207
|
-
*
|
|
208
|
-
* @param {string} dedupeKey @param {string} summary @param {{runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} [extra]
|
|
209
|
-
*/
|
|
210
|
-
const announce = async (dedupeKey, summary, extra = {}) => {
|
|
211
|
-
if (seen.has(dedupeKey)) return;
|
|
212
|
-
seen.add(dedupeKey);
|
|
213
|
-
const appended = appendInbox(runsDir, { type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
214
|
-
if (!appended.appended) return;
|
|
215
|
-
emit(summary);
|
|
216
|
-
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
217
|
-
};
|
|
218
|
-
let anyActive = false;
|
|
219
|
-
for (const runId of campaign.linkedRunIds) {
|
|
220
|
-
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
221
|
-
if (!status || !Array.isArray(status.nodes)) continue;
|
|
222
|
-
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
223
|
-
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
224
|
-
const previous = runSignatures.get(runId);
|
|
225
|
-
runSignatures.set(runId, signature);
|
|
226
|
-
if (!terminal) {
|
|
227
|
-
anyActive = true;
|
|
228
|
-
const runLock = readLock(join(runsDir, runId));
|
|
229
|
-
const stale = !runLock || /** @type {{invalid?: true}} */ (runLock).invalid || lockStale(runLock);
|
|
230
|
-
if (stale && !first) {
|
|
231
|
-
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
if (!first && previous !== signature) {
|
|
235
|
-
for (const node of status.nodes) {
|
|
236
|
-
const attention = ATTENTION_NODE_STATUSES.has(String(node.status))
|
|
237
|
-
|| (node.status === "blocked" && !(Array.isArray(node.blockedBy) && node.blockedBy.length > 0));
|
|
238
|
-
if (attention) {
|
|
239
|
-
const key = `node:${runId}:${node.id}:${node.status}:${node.errorCode ?? ""}`;
|
|
240
|
-
await announce(
|
|
241
|
-
key,
|
|
242
|
-
`campaign-watch: ${runId} node ${node.id} ${node.status}${node.errorCode ? ` [${node.errorCode}]` : ""}${node.note ? ` ${node.note}` : ""}`,
|
|
243
|
-
{ runId, nodeId: String(node.id), status: String(node.status), errorCode: node.errorCode ?? null },
|
|
244
|
-
);
|
|
245
|
-
}
|
|
246
|
-
}
|
|
247
|
-
}
|
|
248
|
-
if (terminal) {
|
|
249
|
-
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
250
|
-
}
|
|
251
|
-
}
|
|
252
|
-
const nowMs = now();
|
|
253
|
-
if (anyActive) lastActiveAt = nowMs;
|
|
254
|
-
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
255
|
-
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
256
|
-
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
257
|
-
}
|
|
258
|
-
first = false;
|
|
259
|
-
if (options.once === true) return;
|
|
260
|
-
await sleep(pollMs);
|
|
261
|
-
}
|
|
262
|
-
} finally {
|
|
263
|
-
lock.release();
|
|
264
|
-
}
|
|
265
|
-
}
|
|
266
|
-
|
|
267
|
-
const WATCH_LOCK_FILE = "watch.lock";
|
|
268
|
-
|
|
269
|
-
/**
|
|
270
|
-
* A durable campaign-watch lock, one watcher per campaign across processes.
|
|
271
|
-
* A live holder is never taken over; a dead or recycled pid's lock is stale
|
|
272
|
-
* and is replaced, so a restart after a crash is not blocked. The same
|
|
273
|
-
* liveness rule as the controller lock: a pid is dead only when the probe
|
|
274
|
-
* proves it.
|
|
275
|
-
*
|
|
276
|
-
* @param {string} campaignPath
|
|
277
|
-
* @returns {{pid: number, processStartToken: string|null, startedAt: string, release: () => void}}
|
|
278
|
-
*/
|
|
279
|
-
export function acquireWatchLock(campaignPath) {
|
|
280
|
-
const path = join(campaignPath, WATCH_LOCK_FILE);
|
|
281
|
-
/** @type {{pid?: number, processStartToken?: string|null, startedAt?: string}} */
|
|
282
|
-
let occupant = {};
|
|
283
|
-
for (let attempt = 0; attempt < 20; attempt += 1) {
|
|
284
|
-
const record = { pid: process.pid, processStartToken: processStartToken(process.pid), startedAt: new Date().toISOString() };
|
|
285
|
-
try {
|
|
286
|
-
const fd = openSync(path, "wx", 0o600);
|
|
287
|
-
try {
|
|
288
|
-
writeSync(fd, JSON.stringify(record));
|
|
289
|
-
} finally {
|
|
290
|
-
closeSync(fd);
|
|
291
|
-
}
|
|
292
|
-
return {
|
|
293
|
-
...record,
|
|
294
|
-
release() {
|
|
295
|
-
try {
|
|
296
|
-
unlinkSync(path);
|
|
297
|
-
} catch (error) {
|
|
298
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
299
|
-
}
|
|
300
|
-
},
|
|
301
|
-
};
|
|
302
|
-
} catch (error) {
|
|
303
|
-
if (errorCode(error) !== "EEXIST") throw error;
|
|
304
|
-
}
|
|
305
|
-
try {
|
|
306
|
-
occupant = /** @type {{pid?: number, processStartToken?: string|null}} */ (JSON.parse(readFileSync(path, "utf8")));
|
|
307
|
-
} catch {
|
|
308
|
-
occupant = {};
|
|
309
|
-
}
|
|
310
|
-
if (!watchLockStale(occupant)) {
|
|
311
|
-
throw new Error(`campaign watch is already running (pid ${occupant.pid})`);
|
|
312
|
-
}
|
|
313
|
-
try {
|
|
314
|
-
unlinkSync(path);
|
|
315
|
-
} catch (error) {
|
|
316
|
-
if (errorCode(error) !== "ENOENT") throw error;
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
throw new Error("campaign watch lock contention did not settle");
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
/**
|
|
323
|
-
* @param {{pid?: number, processStartToken?: string|null}} occupant
|
|
324
|
-
* @returns {boolean}
|
|
325
|
-
*/
|
|
326
|
-
function watchLockStale(occupant) {
|
|
327
|
-
if (typeof occupant.pid !== "number") return true;
|
|
328
|
-
if (!pidAlive(occupant.pid)) return true;
|
|
329
|
-
return Boolean(occupant.processStartToken) && processStartToken(occupant.pid) !== occupant.processStartToken;
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
|
|
333
156
|
/**
|
|
334
157
|
* @param {string} campaignId
|
|
335
158
|
* @param {CliValues} values
|
|
@@ -467,9 +290,38 @@ function close(campaignId, values) {
|
|
|
467
290
|
renderHandoff(path, runsDir);
|
|
468
291
|
process.stdout.write(`[campaign] ${closed.campaign.id} closed\n`);
|
|
469
292
|
process.stdout.write(`[campaign] ledger · docs/campaigns/${closed.campaign.id}/ledger · ${closed.ledgerFiles.length} files\n`);
|
|
293
|
+
reportRequirementClosure(closed.campaign.requirements ?? []);
|
|
470
294
|
if (syncAgentSignal(runsDir)) process.stdout.write(`[campaign] AGENTS.md signal updated\n`);
|
|
471
295
|
}
|
|
472
296
|
|
|
297
|
+
/**
|
|
298
|
+
* Say out loud what the close just wrote, and what it measured.
|
|
299
|
+
*
|
|
300
|
+
* The record is a correlation between the requirement ids a linked run's
|
|
301
|
+
* contract declared and the node snapshots that reached `done` carrying them.
|
|
302
|
+
* It reads the run directory and never the branch, so a requirement the
|
|
303
|
+
* operator delivered by hand -- because its node blocked, which happens --
|
|
304
|
+
* lands in the record as `open` while the work is on main. Measured
|
|
305
|
+
* 2026-09-22 on `availability-is-verified-not-assumed`: R4 read `open` with
|
|
306
|
+
* an empty `nodes` array and was merged in `a6a5d43`.
|
|
307
|
+
*
|
|
308
|
+
* Nothing printed this before, so the only way to meet the closure was to
|
|
309
|
+
* open the JSON, where `open` reads as "not delivered" instead of "no node of
|
|
310
|
+
* this campaign proved it". Naming the measurement is the whole fix: a close
|
|
311
|
+
* cannot know what it did not run.
|
|
312
|
+
*
|
|
313
|
+
* @param {import("../campaign/index.mjs").RequirementClosure[]} requirements
|
|
314
|
+
*/
|
|
315
|
+
function reportRequirementClosure(requirements) {
|
|
316
|
+
if (requirements.length === 0) return;
|
|
317
|
+
const open = requirements.filter((entry) => entry.status !== "covered");
|
|
318
|
+
const covered = requirements.length - open.length;
|
|
319
|
+
process.stdout.write(`[campaign] requirements · ${covered}/${requirements.length} carried by a node that reached done\n`);
|
|
320
|
+
if (open.length === 0) return;
|
|
321
|
+
process.stdout.write(`[campaign] no done node carried ${open.map((entry) => entry.requirementId).join(", ")}\n`);
|
|
322
|
+
process.stdout.write("[campaign] closure reads node snapshots, never the branch: a requirement delivered outside a node reads open here\n");
|
|
323
|
+
}
|
|
324
|
+
|
|
473
325
|
/**
|
|
474
326
|
* `campaign supervise <id>` (also spelled `supervise campaign <id>`): the
|
|
475
327
|
* idempotent re-invocation that drives the manifest. It takes the campaign's
|
package/src/cli/launch.mjs
CHANGED
|
@@ -55,8 +55,14 @@ export function detachSelf(command, target, extraArgs = []) {
|
|
|
55
55
|
* exactly as for every detached controller: a foreground launcher is the only
|
|
56
56
|
* moment an operator is present.
|
|
57
57
|
*
|
|
58
|
+
* `stdio` overrides the discarded default for a caller that has somewhere
|
|
59
|
+
* durable to put the child's output. A detached *run* wants the default: its
|
|
60
|
+
* bootstrap record is the channel, and a controller's real output belongs in
|
|
61
|
+
* the run directory. A detached *plan* has no bootstrap record, so discarding
|
|
62
|
+
* its stdio discarded the only account of why it died.
|
|
63
|
+
*
|
|
58
64
|
* @param {string[]} argv
|
|
59
|
-
* @param {{nonce?: string, env?: NodeJS.ProcessEnv}} [options]
|
|
65
|
+
* @param {{nonce?: string, env?: NodeJS.ProcessEnv, stdio?: import("node:child_process").StdioOptions}} [options]
|
|
60
66
|
* @returns {DetachedChild}
|
|
61
67
|
*/
|
|
62
68
|
export function detachArgv(argv, options = {}) {
|
|
@@ -74,7 +80,7 @@ export function detachArgv(argv, options = {}) {
|
|
|
74
80
|
// pending forever. There it also means DETACHED_PROCESS: no console
|
|
75
81
|
// window, which is what "ignore" stdio already implies.
|
|
76
82
|
detached: true,
|
|
77
|
-
stdio: "ignore",
|
|
83
|
+
stdio: options.stdio ?? "ignore",
|
|
78
84
|
}));
|
|
79
85
|
child.unref();
|
|
80
86
|
child.bootstrapNonce = nonce;
|