faberun 0.29.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +71 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +55 -15
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +120 -38
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +431 -55
- package/src/plan/resolve.mjs +13 -4
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +209 -50
- package/src/plan/sizing.mjs +53 -5
- package/src/plan/template.mjs +297 -49
- package/src/repo/integrate.mjs +47 -5
- package/src/repo/scope-closure.mjs +22 -0
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +27 -0
- package/src/report/locale.mjs +20 -0
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
package/src/campaign/watch.mjs
CHANGED
|
@@ -8,13 +8,37 @@
|
|
|
8
8
|
* supervisor with a poll interval, an idle policy, a durable dedupe through
|
|
9
9
|
* the inbox, and a lock with its own staleness rule. `cli/campaign.mjs` keeps
|
|
10
10
|
* the thin `watch` operation that reads the flags and calls in here.
|
|
11
|
+
*
|
|
12
|
+
* The idle episode is durable too, in `watch-idle.json` beside the lock: the
|
|
13
|
+
* idle alert's key carries the episode, and an episode kept only in memory
|
|
14
|
+
* re-anchors on every restart, which either re-sends a window the operator
|
|
15
|
+
* already saw or stays silent through the next one.
|
|
16
|
+
*
|
|
17
|
+
* The campaign alerts -- phase completed, recovery exhausted, decision
|
|
18
|
+
* needed, closure -- are derived from the same projection the page and the
|
|
19
|
+
* CLI read (`buildCampaignProgress`), never from a second scan of raw run
|
|
20
|
+
* files, so every surface answers the campaign from one set of numbers, and
|
|
21
|
+
* are deduplicated by campaign and episode through the inbox
|
|
22
|
+
* (`alert:<id>:<campaign>:<episode>`, the keys the ALERTS catalogue in
|
|
23
|
+
* `report/locale.mjs` declares). The automatic progress update is rendered
|
|
24
|
+
* from that projection alone -- never a model -- and is an explicit opt-in
|
|
25
|
+
* through `FABERUN_NOTIFY_EVENTS`; when the transport answers with a receipt
|
|
26
|
+
* naming a message id, the anchor in `watch-progress.json` records it, and
|
|
27
|
+
* the next update inside the fifteen-minute window travels as an edit of
|
|
28
|
+
* that message. The anchor is written only after the receipt returns, so an
|
|
29
|
+
* editable message is claimed only on evidence, and it carries the state
|
|
30
|
+
* signature it announced, so an unchanged campaign never re-sends. An
|
|
31
|
+
* attention event is always a new message: alerts never consult the anchor.
|
|
11
32
|
*/
|
|
12
33
|
import { closeSync, openSync, readFileSync, unlinkSync, writeSync } from "node:fs";
|
|
13
34
|
import { join } from "node:path";
|
|
14
35
|
import { lockStale, pidAlive, processStartToken, readLock } from "../run/lock.mjs";
|
|
36
|
+
import { writeJsonAtomic } from "../run/store.mjs";
|
|
15
37
|
import { readCampaign } from "./record.mjs";
|
|
16
38
|
import { notifyQueueFor } from "../engine/notify-queue.mjs";
|
|
17
|
-
import { appendInbox, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
39
|
+
import { appendInbox, deliverNotification, deliverableEventTypes, progressEditTarget, readInbox, wakeCapabilityNotice } from "../notify/index.mjs";
|
|
40
|
+
import { buildCampaignProgress } from "../report/progress.mjs";
|
|
41
|
+
import { ALERTS, chooseLanguage, labelsFor } from "../report/locale.mjs";
|
|
18
42
|
import { errorCode, readJsonTolerant } from "../util.mjs";
|
|
19
43
|
|
|
20
44
|
/** How often the watcher re-reads the campaign when the operator named no interval. */
|
|
@@ -23,19 +47,129 @@ export const DEFAULT_WAKE_POLL_MS = 30_000;
|
|
|
23
47
|
/** How long without a material event before the watcher reports the campaign idle. */
|
|
24
48
|
const WAKE_IDLE_AFTER_MS = 20 * 60_000;
|
|
25
49
|
|
|
50
|
+
/**
|
|
51
|
+
* Where the watcher records the idle episode it is in. The idle alert's dedupe
|
|
52
|
+
* key embeds the instant the episode began, so that anchor has to outlive the
|
|
53
|
+
* process; the file lives beside `watch.lock` because the one campaign watcher
|
|
54
|
+
* owns both.
|
|
55
|
+
*/
|
|
56
|
+
const WATCH_IDLE_FILE = "watch-idle.json";
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Where the watcher records the progress message a transport acknowledged:
|
|
60
|
+
* the message id from the transport's receipt, the instant of that receipt,
|
|
61
|
+
* and the state signature the message announced. Beside `watch.lock`, whose
|
|
62
|
+
* single watcher owns both; the anchor outlives the process, so a restart
|
|
63
|
+
* keeps editing the message the operator can still see and never re-sends a
|
|
64
|
+
* state that was already announced.
|
|
65
|
+
*/
|
|
66
|
+
const WATCH_PROGRESS_FILE = "watch-progress.json";
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* The dedupe key for one alert occurrence: the catalogue's two keys, campaign
|
|
70
|
+
* and episode (`report/locale.mjs`'s ALERTS declares the same pair), joined
|
|
71
|
+
* under the alert's id, so the same occurrence sends once through the inbox
|
|
72
|
+
* no matter how many polls or restarts re-derive it.
|
|
73
|
+
*
|
|
74
|
+
* @param {string} alertId
|
|
75
|
+
* @param {string} campaignId
|
|
76
|
+
* @param {string} episode
|
|
77
|
+
* @returns {string}
|
|
78
|
+
*/
|
|
79
|
+
function alertKey(alertId, campaignId, episode) {
|
|
80
|
+
return `alert:${alertId}:${campaignId}:${episode}`;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* The fields of the shared projection (`buildCampaignProgress`) the watcher
|
|
85
|
+
* reads. The projection's own module types its return as
|
|
86
|
+
* `Record<string, unknown>`, so every consumer narrows the fields it needs
|
|
87
|
+
* its own way; this is the watcher's narrowing, and it names exactly the
|
|
88
|
+
* fields the alerts and the progress update are rendered from -- never the
|
|
89
|
+
* whole shape, which is the projection's to own.
|
|
90
|
+
*
|
|
91
|
+
* @typedef {object} CampaignProjection
|
|
92
|
+
* @property {{done: number, total: number}} counts
|
|
93
|
+
* @property {{nodeId: string}|null} activity
|
|
94
|
+
* @property {number|null} costTotalUsd
|
|
95
|
+
* @property {{contractId: string|null, counts: {done: number, total: number}, nodes: {id: string, status: string}[]}[]} phases
|
|
96
|
+
* @property {{contractId: string}|null} currentPhase
|
|
97
|
+
*/
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* The one line an automatic progress update says, rendered entirely from the
|
|
101
|
+
* projection's own numbers and the campaign's wording home: nodes done over
|
|
102
|
+
* declared, the node in flight, the recorded spend. The same persisted state
|
|
103
|
+
* yields the same bytes on every poll and every restart -- that is the whole
|
|
104
|
+
* guarantee, and the reason no model is asked to phrase it.
|
|
105
|
+
*
|
|
106
|
+
* @param {CampaignProjection} progress
|
|
107
|
+
* @param {(key: string) => string} labels
|
|
108
|
+
* @returns {string}
|
|
109
|
+
*/
|
|
110
|
+
function progressSummaryText(progress, labels) {
|
|
111
|
+
/** @type {string[]} */
|
|
112
|
+
const parts = [`${progress.counts.done}/${progress.counts.total} ${labels("done")}`];
|
|
113
|
+
if (progress.activity) parts.push(`${labels("running")} ${progress.activity.nodeId}`);
|
|
114
|
+
if (typeof progress.costTotalUsd === "number") parts.push(`$${progress.costTotalUsd.toFixed(2)}`);
|
|
115
|
+
return parts.join(" · ");
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* The closure alert's detail: the campaign's final counts and its recorded
|
|
120
|
+
* spend.
|
|
121
|
+
*
|
|
122
|
+
* @param {CampaignProjection} progress
|
|
123
|
+
* @param {(key: string) => string} labels
|
|
124
|
+
* @param {string} campaignId
|
|
125
|
+
* @returns {string}
|
|
126
|
+
*/
|
|
127
|
+
function closureDetail(progress, labels, campaignId) {
|
|
128
|
+
const cost = typeof progress.costTotalUsd === "number" ? ` · $${progress.costTotalUsd.toFixed(2)}` : "";
|
|
129
|
+
return `${campaignId} · ${progress.counts.done}/${progress.counts.total} ${labels("done")}${cost}`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The automatic progress update's delivery: an explicit opt-in through
|
|
134
|
+
* `FABERUN_NOTIFY_EVENTS` (`progress` is not in the default set -- it is the
|
|
135
|
+
* one recurring message, and exactly the flood the default set exists to
|
|
136
|
+
* keep off the operator's phone). The summary arrives already rendered from
|
|
137
|
+
* the projection; `editOfMessageId`, when the caller anchors one, asks the
|
|
138
|
+
* transport to edit that message instead of sending a new one. Resolves
|
|
139
|
+
* `null` when nothing was attempted, so the caller's anchor stays untouched.
|
|
140
|
+
*
|
|
141
|
+
* @param {string} campaignId
|
|
142
|
+
* @param {string} summary
|
|
143
|
+
* @param {string|null} editOfMessageId
|
|
144
|
+
* @returns {Promise<import("../notify/index.mjs").DeliveryResult|null>}
|
|
145
|
+
*/
|
|
146
|
+
async function deliverProgressUpdate(campaignId, summary, editOfMessageId) {
|
|
147
|
+
if (!deliverableEventTypes().has("progress")) return null;
|
|
148
|
+
return deliverNotification({
|
|
149
|
+
type: "progress",
|
|
150
|
+
campaignId,
|
|
151
|
+
summary,
|
|
152
|
+
...(editOfMessageId ? { editOfMessageId } : {}),
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
|
|
26
156
|
const TERMINAL_NODE_STATUSES = new Set(["done", "no-op", "blocked", "failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
27
|
-
const ATTENTION_NODE_STATUSES = new Set(["failed", "exhausted", "stalled", "canceled", "cancelled"]);
|
|
28
157
|
|
|
29
158
|
/**
|
|
30
|
-
* The watcher loop.
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
159
|
+
* The watcher loop. The campaign alerts are derived from the projection and
|
|
160
|
+
* announced through `notify`, which by default records them in
|
|
161
|
+
* `<runs-dir>/inbox.jsonl` and delivers them to the campaign's
|
|
162
|
+
* `notify.jsonl`; the inbox is both the durable record and the dedupe, so an
|
|
163
|
+
* occurrence already recorded is never re-sent. The idle line is keyed by
|
|
164
|
+
* campaign, by the durable idle episode and by the twenty-minute window
|
|
165
|
+
* inside it. The automatic progress update is not an alert: it is delivered
|
|
166
|
+
* through `sendProgress`, which returns the transport's receipt, and the
|
|
167
|
+
* receipt -- only the receipt -- writes the `watch-progress.json` anchor.
|
|
168
|
+
* The injectable seams exist so a test can drive the loop deterministically.
|
|
35
169
|
*
|
|
36
170
|
* @param {string} campaignPath
|
|
37
171
|
* @param {string} runsDir
|
|
38
|
-
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, lock?: {release: () => void}}} [options]
|
|
172
|
+
* @param {{pollMs?: number, once?: boolean, now?: () => number, sleep?: (ms: number) => Promise<void>, emit?: (line: string) => void, notify?: (event: {type: string, campaignId: string, dedupeKey: string, summary: string, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}) => Promise<void>|void, sendProgress?: (campaignId: string, summary: string, editOfMessageId: string|null) => Promise<{ok: boolean, messageId?: string}|null>, lock?: {release: () => void}}} [options]
|
|
39
173
|
* @returns {Promise<void>}
|
|
40
174
|
*/
|
|
41
175
|
export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
@@ -53,20 +187,17 @@ export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
|
53
187
|
status: event.status ?? null,
|
|
54
188
|
errorCode: event.errorCode ?? null,
|
|
55
189
|
}));
|
|
190
|
+
const sendProgress = options.sendProgress ?? deliverProgressUpdate;
|
|
56
191
|
const seen = new Set(readInbox(runsDir).map((entry) => entry.dedupeKey));
|
|
57
192
|
const lock = options.lock ?? acquireWatchLock(campaignPath);
|
|
58
193
|
emit(wakeCapabilityNotice());
|
|
59
|
-
/** @type {Map<string, string>} */
|
|
60
|
-
const runSignatures = new Map();
|
|
61
194
|
let lastActiveAt = now();
|
|
195
|
+
/** @type {number|null} */
|
|
196
|
+
let idleSince = null;
|
|
62
197
|
let first = true;
|
|
63
198
|
try {
|
|
64
199
|
for (;;) {
|
|
65
200
|
const campaign = readCampaign(campaignPath);
|
|
66
|
-
if (campaign.status !== "active") {
|
|
67
|
-
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
68
|
-
return;
|
|
69
|
-
}
|
|
70
201
|
/**
|
|
71
202
|
* Persist first, deliver second: the inbox entry is the durable dedupe,
|
|
72
203
|
* so a restart or a second watcher skips a line already recorded even
|
|
@@ -82,14 +213,64 @@ export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
|
82
213
|
emit(summary);
|
|
83
214
|
await notify({ type: "attention", campaignId: campaign.id, dedupeKey, summary, ...extra });
|
|
84
215
|
};
|
|
216
|
+
const language = chooseLanguage(process.env, [campaign.goal]);
|
|
217
|
+
const labels = labelsFor(language);
|
|
218
|
+
const nowMs = now();
|
|
219
|
+
/** @type {CampaignProjection|null} */
|
|
220
|
+
let progress = null;
|
|
221
|
+
try {
|
|
222
|
+
progress = /** @type {CampaignProjection} */ (buildCampaignProgress(runsDir, campaign.id, nowMs));
|
|
223
|
+
} catch {
|
|
224
|
+
// A projection that cannot be built (a torn record, an unreadable
|
|
225
|
+
// journal) yields no alerts and no progress update this poll; the
|
|
226
|
+
// status loop and the idle clock below still run, and the next poll
|
|
227
|
+
// re-reads everything.
|
|
228
|
+
progress = null;
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* One alert occurrence, worded by the catalogue and deduped by campaign
|
|
232
|
+
* and episode through the inbox. Alerts are always new messages: none
|
|
233
|
+
* of them ever consults the progress anchor.
|
|
234
|
+
*
|
|
235
|
+
* @param {string} alertId @param {string} episode @param {string} detail
|
|
236
|
+
*/
|
|
237
|
+
const announceAlert = async (alertId, episode, detail) => {
|
|
238
|
+
const spec = ALERTS.find((entry) => entry.id === alertId);
|
|
239
|
+
if (!spec) throw new Error(`campaign watch: no catalogue entry for alert ${alertId}`);
|
|
240
|
+
await announce(alertKey(alertId, campaign.id, episode), `campaign-watch: ${spec.text[language]}: ${detail}`);
|
|
241
|
+
};
|
|
242
|
+
if (campaign.status !== "active") {
|
|
243
|
+
if (progress) await announceAlert("closure", campaign.id, closureDetail(progress, labels, campaign.id));
|
|
244
|
+
emit(`campaign-watch: ${campaign.id} is ${campaign.status}; stopping`);
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
if (progress) {
|
|
248
|
+
for (const phase of progress.phases) {
|
|
249
|
+
if (phase.contractId === null) continue;
|
|
250
|
+
if (phase.counts.total > 0 && phase.counts.done === phase.counts.total) {
|
|
251
|
+
await announceAlert("phase-completed", phase.contractId, `${phase.contractId} · ${phase.counts.done}/${phase.counts.total} ${labels("done")}`);
|
|
252
|
+
}
|
|
253
|
+
for (const node of phase.nodes) {
|
|
254
|
+
if (node.status === "exhausted") {
|
|
255
|
+
await announceAlert("recovery-exhausted", `${phase.contractId}:${node.id}`, `${phase.contractId} · ${node.id}`);
|
|
256
|
+
} else if (node.status === "blocked") {
|
|
257
|
+
await announceAlert("decision-needed", `${phase.contractId}:${node.id}`, `${phase.contractId} · ${node.id}`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
// A restart resumes the episode this campaign was already in. The key
|
|
263
|
+
// carries the episode's anchor, so resuming it is what keeps a window
|
|
264
|
+
// already announced quiet and the next window audible.
|
|
265
|
+
if (first) {
|
|
266
|
+
const storedSince = readIdleEpisode(campaignPath, campaign.id);
|
|
267
|
+
if (storedSince !== null) idleSince = storedSince;
|
|
268
|
+
}
|
|
85
269
|
let anyActive = false;
|
|
86
270
|
for (const runId of campaign.linkedRunIds) {
|
|
87
271
|
const status = /** @type {Record<string, any>|null} */ (readJsonTolerant(join(runsDir, runId, "status.json")));
|
|
88
272
|
if (!status || !Array.isArray(status.nodes)) continue;
|
|
89
273
|
const terminal = status.nodes.every((/** @type {any} */ node) => TERMINAL_NODE_STATUSES.has(String(node.status)));
|
|
90
|
-
const signature = status.nodes.map((/** @type {any} */ node) => `${node.id}:${node.status}:${node.errorCode ?? ""}`).join("|");
|
|
91
|
-
const previous = runSignatures.get(runId);
|
|
92
|
-
runSignatures.set(runId, signature);
|
|
93
274
|
if (!terminal) {
|
|
94
275
|
anyActive = true;
|
|
95
276
|
const runLock = readLock(join(runsDir, runId));
|
|
@@ -98,29 +279,41 @@ export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
|
98
279
|
await announce(`stale:${runId}`, `campaign-watch: ${runId} has non-terminal nodes but no live controller; resume it`, { runId });
|
|
99
280
|
}
|
|
100
281
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
282
|
+
}
|
|
283
|
+
if (anyActive) {
|
|
284
|
+
lastActiveAt = nowMs;
|
|
285
|
+
if (idleSince !== null) {
|
|
286
|
+
idleSince = null;
|
|
287
|
+
clearIdleEpisode(campaignPath);
|
|
288
|
+
}
|
|
289
|
+
} else if (idleSince === null) {
|
|
290
|
+
idleSince = lastActiveAt;
|
|
291
|
+
writeIdleEpisode(campaignPath, campaign.id, idleSince);
|
|
292
|
+
}
|
|
293
|
+
if (idleSince !== null && nowMs - idleSince >= WAKE_IDLE_AFTER_MS) {
|
|
294
|
+
const key = `idle:${campaign.id}:${new Date(idleSince).toISOString()}:${Math.floor((nowMs - idleSince) / WAKE_IDLE_AFTER_MS)}`;
|
|
295
|
+
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - idleSince) / 60_000)} min; dispatch the next step`);
|
|
296
|
+
}
|
|
297
|
+
// The automatic progress update. The anchor is the record of what was
|
|
298
|
+
// already announced: the same state signature is never delivered
|
|
299
|
+
// twice, whatever restarted in between. When there is something new to
|
|
300
|
+
// say, a still-editable message is edited in place, and the anchor is
|
|
301
|
+
// claimed only from the transport's receipt -- a delivery without a
|
|
302
|
+
// receipt claims nothing, and the next poll tries again.
|
|
303
|
+
if (progress) {
|
|
304
|
+
const signature = `${progress.counts.done}/${progress.counts.total}:${progress.activity?.nodeId ?? "-"}:${progress.currentPhase?.contractId ?? "-"}`;
|
|
305
|
+
const anchor = readProgressAnchor(campaignPath, campaign.id);
|
|
306
|
+
if (!anchor || anchor.signature !== signature) {
|
|
307
|
+
const editOfMessageId = progressEditTarget(anchor, nowMs);
|
|
308
|
+
const receipt = await sendProgress(campaign.id, progressSummaryText(progress, labels), editOfMessageId);
|
|
309
|
+
if (receipt) {
|
|
310
|
+
if (receipt.ok && typeof receipt.messageId === "string" && receipt.messageId) {
|
|
311
|
+
writeProgressAnchor(campaignPath, campaign.id, { messageId: receipt.messageId, at: nowMs, signature });
|
|
312
|
+
} else {
|
|
313
|
+
clearProgressAnchor(campaignPath);
|
|
112
314
|
}
|
|
113
315
|
}
|
|
114
316
|
}
|
|
115
|
-
if (terminal) {
|
|
116
|
-
await announce(`terminal:${runId}`, `campaign-watch: ${runId} terminal · ${status.summary ?? ""}`, { runId });
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
const nowMs = now();
|
|
120
|
-
if (anyActive) lastActiveAt = nowMs;
|
|
121
|
-
else if (!first && nowMs - lastActiveAt >= WAKE_IDLE_AFTER_MS) {
|
|
122
|
-
const key = `idle:${Math.floor((nowMs - lastActiveAt) / WAKE_IDLE_AFTER_MS)}`;
|
|
123
|
-
await announce(key, `campaign-watch: ${campaign.id} active but no run has been active for ${Math.round((nowMs - lastActiveAt) / 60_000)} min; dispatch the next step`);
|
|
124
317
|
}
|
|
125
318
|
first = false;
|
|
126
319
|
if (options.once === true) return;
|
|
@@ -131,6 +324,90 @@ export async function watchCampaignWake(campaignPath, runsDir, options = {}) {
|
|
|
131
324
|
}
|
|
132
325
|
}
|
|
133
326
|
|
|
327
|
+
/**
|
|
328
|
+
* The idle anchor a restart resumes, or null when no episode is in progress. A
|
|
329
|
+
* record naming another campaign, or whose instant does not parse, is no
|
|
330
|
+
* anchor: a campaign directory is just a path, and a stale file must never seed
|
|
331
|
+
* the key with an identity nothing can check.
|
|
332
|
+
*
|
|
333
|
+
* @param {string} campaignPath
|
|
334
|
+
* @param {string} campaignId
|
|
335
|
+
* @returns {number|null} epoch milliseconds the episode began
|
|
336
|
+
*/
|
|
337
|
+
function readIdleEpisode(campaignPath, campaignId) {
|
|
338
|
+
const record = /** @type {{campaignId?: unknown, idleSince?: unknown}|null} */ (readJsonTolerant(join(campaignPath, WATCH_IDLE_FILE)));
|
|
339
|
+
if (!record || record.campaignId !== campaignId || typeof record.idleSince !== "string") return null;
|
|
340
|
+
const started = Date.parse(record.idleSince);
|
|
341
|
+
return Number.isFinite(started) ? started : null;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* @param {string} campaignPath
|
|
346
|
+
* @param {string} campaignId
|
|
347
|
+
* @param {number} idleSince epoch milliseconds
|
|
348
|
+
*/
|
|
349
|
+
function writeIdleEpisode(campaignPath, campaignId, idleSince) {
|
|
350
|
+
writeJsonAtomic(join(campaignPath, WATCH_IDLE_FILE), { campaignId, idleSince: new Date(idleSince).toISOString() });
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
/**
|
|
354
|
+
* @param {string} campaignPath
|
|
355
|
+
*/
|
|
356
|
+
function clearIdleEpisode(campaignPath) {
|
|
357
|
+
try {
|
|
358
|
+
unlinkSync(join(campaignPath, WATCH_IDLE_FILE));
|
|
359
|
+
} catch (error) {
|
|
360
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* The progress anchor a restart resumes, or null when none stands. A record
|
|
366
|
+
* naming another campaign, a message id it does not carry, or an instant or
|
|
367
|
+
* state signature that does not parse is no anchor: the anchor claims a
|
|
368
|
+
* message the transport acknowledged, and a claim without all of that is
|
|
369
|
+
* exactly the claim the receipt rule forbids.
|
|
370
|
+
*
|
|
371
|
+
* @param {string} campaignPath
|
|
372
|
+
* @param {string} campaignId
|
|
373
|
+
* @returns {{messageId: string, at: number, signature: string}|null}
|
|
374
|
+
*/
|
|
375
|
+
function readProgressAnchor(campaignPath, campaignId) {
|
|
376
|
+
const record = /** @type {{campaignId?: unknown, messageId?: unknown, at?: unknown, signature?: unknown}|null} */ (readJsonTolerant(join(campaignPath, WATCH_PROGRESS_FILE)));
|
|
377
|
+
if (!record
|
|
378
|
+
|| record.campaignId !== campaignId
|
|
379
|
+
|| typeof record.messageId !== "string" || !record.messageId
|
|
380
|
+
|| typeof record.at !== "string"
|
|
381
|
+
|| typeof record.signature !== "string") return null;
|
|
382
|
+
const at = Date.parse(record.at);
|
|
383
|
+
return Number.isFinite(at) ? { messageId: record.messageId, at, signature: record.signature } : null;
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* @param {string} campaignPath
|
|
388
|
+
* @param {string} campaignId
|
|
389
|
+
* @param {{messageId: string, at: number, signature: string}} anchor
|
|
390
|
+
*/
|
|
391
|
+
function writeProgressAnchor(campaignPath, campaignId, anchor) {
|
|
392
|
+
writeJsonAtomic(join(campaignPath, WATCH_PROGRESS_FILE), {
|
|
393
|
+
campaignId,
|
|
394
|
+
messageId: anchor.messageId,
|
|
395
|
+
at: new Date(anchor.at).toISOString(),
|
|
396
|
+
signature: anchor.signature,
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* @param {string} campaignPath
|
|
402
|
+
*/
|
|
403
|
+
function clearProgressAnchor(campaignPath) {
|
|
404
|
+
try {
|
|
405
|
+
unlinkSync(join(campaignPath, WATCH_PROGRESS_FILE));
|
|
406
|
+
} catch (error) {
|
|
407
|
+
if (errorCode(error) !== "ENOENT") throw error;
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
|
|
134
411
|
const WATCH_LOCK_FILE = "watch.lock";
|
|
135
412
|
|
|
136
413
|
/**
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The judge
|
|
3
|
-
*
|
|
2
|
+
* The judge envelope: the limits `parseJudge` enforces, the reasons it rejects
|
|
3
|
+
* by, and the byte budget a judge prompt itself must fit.
|
|
4
4
|
*
|
|
5
5
|
* They live apart from the parser because the judge prompt must render the same
|
|
6
6
|
* numbers. A judge that is never told the limit can only be discarded by it: a
|
|
@@ -23,3 +23,18 @@ export const JUDGE_ENVELOPE_REASON = "judge result exceeds limits";
|
|
|
23
23
|
|
|
24
24
|
/** The reason `parseJudge` throws when a single finding overshoots the envelope. */
|
|
25
25
|
export const JUDGE_FINDING_ENVELOPE_REASON = "judge finding exceeds limits";
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The byte budget a judge prompt must fit.
|
|
29
|
+
*
|
|
30
|
+
* The prompt carries the Definition of Done, the evidence and the re-ask
|
|
31
|
+
* instruction, so it is the one judge artefact that must never be cut to fit.
|
|
32
|
+
* `dispatch.mjs` enforces this budget on the prompt it assembles, and the
|
|
33
|
+
* rotation in `engine/phase-session.mjs` refuses with it rather than bounding
|
|
34
|
+
* the caller's prompt down, which would send a judge to arbitrate less than the
|
|
35
|
+
* node declared. One number, two call sites.
|
|
36
|
+
*/
|
|
37
|
+
export const JUDGE_PROMPT_BYTES = 64 * 1024;
|
|
38
|
+
|
|
39
|
+
/** The reason an oversized judge prompt is refused, byte-for-byte in both call sites. */
|
|
40
|
+
export const JUDGE_PROMPT_REASON = `judge prompt exceeds ${JUDGE_PROMPT_BYTES} bytes`;
|
|
@@ -142,10 +142,11 @@ export function reauthorWriteConflicts(nodes, nodeId, addedWriteFiles) {
|
|
|
142
142
|
* packet: import("./task-packet.mjs").TaskPacket|null|undefined,
|
|
143
143
|
* findings: string[]|null|undefined,
|
|
144
144
|
* history: {round: number, accepted: boolean, additions?: ReauthorAdditions, findings?: string[]}[],
|
|
145
|
+
* failureCause?: import("../engine/failure-cause.mjs").FailureCause|null,
|
|
145
146
|
* }} entry
|
|
146
147
|
* @returns {Record<string, unknown>}
|
|
147
148
|
*/
|
|
148
|
-
export function reauthorProposalRecord({ nodeId, outcome, riskTier, budget, roundsUsed, additions, packet, findings, history }) {
|
|
149
|
+
export function reauthorProposalRecord({ nodeId, outcome, riskTier, budget, roundsUsed, additions, packet, findings, history, failureCause }) {
|
|
149
150
|
/** @param {string[]|undefined} paths @returns {string[]} */
|
|
150
151
|
const bounded = (paths) => [...new Set(paths ?? [])].slice(0, MAX_REAUTHOR_PATHS);
|
|
151
152
|
return {
|
|
@@ -153,6 +154,7 @@ export function reauthorProposalRecord({ nodeId, outcome, riskTier, budget, roun
|
|
|
153
154
|
node: nodeId,
|
|
154
155
|
outcome,
|
|
155
156
|
riskTier,
|
|
157
|
+
failureCause: failureCause ?? null,
|
|
156
158
|
reauthorProposal: {
|
|
157
159
|
packet: packet ?? null,
|
|
158
160
|
addedReadFiles: bounded(additions?.readFiles),
|
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The closed task packet: the schema, validation and prompt rendering for the
|
|
3
|
+
* durable scope one node runs under. Separate from the contract that carries
|
|
4
|
+
* it (`index.mjs`) because a packet is authored and validated on its own too —
|
|
5
|
+
* a discovery node's `artifacts[0]` is one — and from the planning templates
|
|
6
|
+
* (`plan/template.mjs`) because those only assemble the JSON this module
|
|
7
|
+
* refuses or renders. Three modes, one boundary: an execution packet is closed
|
|
8
|
+
* to named read and write files, a discovery packet is read-only, and an
|
|
9
|
+
* autonomous packet bounds writes to roots while leaving inspection free.
|
|
10
|
+
* Every prompt states the whole worker-result shape and the reserved owner
|
|
11
|
+
* decisions, and none may exceed the byte budget below: a packet's mandatory
|
|
12
|
+
* content — its declared read files among it — is never truncated to fit, so
|
|
13
|
+
* a packet without room for it is refused by name instead.
|
|
14
|
+
*/
|
|
1
15
|
import { existsSync, lstatSync, readFileSync, realpathSync, statSync } from "node:fs";
|
|
2
16
|
import { Buffer } from "node:buffer";
|
|
3
17
|
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
|
@@ -20,6 +34,29 @@ const FIELDS = new Set([
|
|
|
20
34
|
"verification",
|
|
21
35
|
]);
|
|
22
36
|
const PROMPT_MAX_BYTES = 64 * 1024;
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The one refusal for a packet whose mandatory content does not fit the
|
|
40
|
+
* prompt budget (R4, campaign-efficiency phase 4). Naming the node, the mode
|
|
41
|
+
* and the overage is the point: a packet is never truncated to fit — a
|
|
42
|
+
* dropped read file is a fact the worker was promised — so whoever authored
|
|
43
|
+
* the packet must be told which node to shrink and by how much, not left
|
|
44
|
+
* with an anonymous throw that kills the node at dispatch.
|
|
45
|
+
*
|
|
46
|
+
* The prompt-budget sentence is kept verbatim inside the message: it is the
|
|
47
|
+
* wording earlier packets and tests were written against.
|
|
48
|
+
*
|
|
49
|
+
* @param {string} prompt
|
|
50
|
+
* @param {string} nodeId
|
|
51
|
+
* @param {TaskPacket["mode"]} mode
|
|
52
|
+
* @returns {never}
|
|
53
|
+
*/
|
|
54
|
+
function refusePromptOverBudget(prompt, nodeId, mode) {
|
|
55
|
+
const total = Buffer.byteLength(prompt, "utf8");
|
|
56
|
+
throw new TypeError(
|
|
57
|
+
`worker_prompt_over_budget: node ${nodeId} (${mode}) prompt is ${total - PROMPT_MAX_BYTES} bytes over the ${PROMPT_MAX_BYTES}-byte budget at ${total} total — the worker prompt exceeds ${PROMPT_MAX_BYTES} bytes, and a packet is never truncated to fit: shrink the readFiles selection or the instructions, or split the node`,
|
|
58
|
+
);
|
|
59
|
+
}
|
|
23
60
|
/**
|
|
24
61
|
* The `## Verification` preamble every worker prompt carries. The controller's
|
|
25
62
|
* recorded run is the proof; a self-run is optional and must be a quick,
|
|
@@ -166,9 +203,7 @@ export function renderWorkerPrompt(packet, nodeId) {
|
|
|
166
203
|
REQUIRED_OUTPUT_SCHEMA,
|
|
167
204
|
];
|
|
168
205
|
const prompt = `${lines.join("\n")}\n`;
|
|
169
|
-
if (Buffer.byteLength(prompt, "utf8") > PROMPT_MAX_BYTES)
|
|
170
|
-
throw new TypeError(`worker prompt exceeds ${PROMPT_MAX_BYTES} bytes`);
|
|
171
|
-
}
|
|
206
|
+
if (Buffer.byteLength(prompt, "utf8") > PROMPT_MAX_BYTES) refusePromptOverBudget(prompt, nodeId, "execution");
|
|
172
207
|
return prompt;
|
|
173
208
|
}
|
|
174
209
|
|
|
@@ -513,7 +548,7 @@ function renderDiscoveryPrompt(packet, nodeId) {
|
|
|
513
548
|
...packet.verification.map((command) => `- ${command.argv.join(" ")}`),
|
|
514
549
|
];
|
|
515
550
|
const prompt = `${lines.join("\n")}\n`;
|
|
516
|
-
if (Buffer.byteLength(prompt, "utf8") > PROMPT_MAX_BYTES)
|
|
551
|
+
if (Buffer.byteLength(prompt, "utf8") > PROMPT_MAX_BYTES) refusePromptOverBudget(prompt, nodeId, "discovery");
|
|
517
552
|
return prompt;
|
|
518
553
|
}
|
|
519
554
|
|
|
@@ -560,7 +595,7 @@ function renderAutonomousPrompt(packet, nodeId) {
|
|
|
560
595
|
REQUIRED_OUTPUT_SCHEMA,
|
|
561
596
|
];
|
|
562
597
|
const prompt = `${lines.join("\n")}\n`;
|
|
563
|
-
if (Buffer.byteLength(prompt, "utf8") > PROMPT_MAX_BYTES)
|
|
598
|
+
if (Buffer.byteLength(prompt, "utf8") > PROMPT_MAX_BYTES) refusePromptOverBudget(prompt, nodeId, "autonomous");
|
|
564
599
|
return prompt;
|
|
565
600
|
}
|
|
566
601
|
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Buffer } from "node:buffer";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
2
3
|
import { realpathSync, statSync } from "node:fs";
|
|
3
4
|
import { isAbsolute, resolve } from "node:path";
|
|
4
5
|
import { fail, isContained, tailText } from "../util.mjs";
|
|
@@ -88,7 +89,10 @@ export const MUTATION_TIERS = Object.freeze({
|
|
|
88
89
|
*/
|
|
89
90
|
|
|
90
91
|
/**
|
|
91
|
-
* Callbacks and options for {@link runVerification}.
|
|
92
|
+
* Callbacks and options for {@link runVerification}. `logDir` is both the
|
|
93
|
+
* per-command evidence log and the checkpoint store a resumed pass serves
|
|
94
|
+
* from; the serving is `engine/run-command.mjs`'s, the record's shape is
|
|
95
|
+
* validated here.
|
|
92
96
|
*
|
|
93
97
|
* @typedef {{signal?: AbortSignal, logDir?: string, writeFiles?: string[], onAttemptStart?: (attempt: VerificationAttempt) => void, onAttemptSpawn?: (attempt: VerificationAttempt) => void, onAttemptComplete?: (attempt: VerificationAttempt) => void}} VerificationOptions
|
|
94
98
|
*/
|
|
@@ -225,6 +229,68 @@ export function compactVerification(result) {
|
|
|
225
229
|
return { passed: Boolean(result?.passed), commands };
|
|
226
230
|
}
|
|
227
231
|
|
|
232
|
+
/**
|
|
233
|
+
* A per-command checkpoint: the identity hash of everything the recorded proof
|
|
234
|
+
* depends on, plus the recorded result bounded exactly like persisted state.
|
|
235
|
+
* It lives beside the pass's evidence log (`checkpoint-<ordinal>.json`) and is
|
|
236
|
+
* what a resumed verification pass serves instead of re-running a command.
|
|
237
|
+
*
|
|
238
|
+
* @typedef {{identity: string, result: VerificationCommandResult}} CommandCheckpoint
|
|
239
|
+
*/
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* The identity of one command's proof: a hash over the workspace tree — the
|
|
243
|
+
* dirty fingerprint *and* the HEAD tree id, because the fingerprint hashes
|
|
244
|
+
* only the difference from HEAD and two clean trees at different commits share
|
|
245
|
+
* one (review 2026-10-02: a stale passing result was served across a commit) —
|
|
246
|
+
* plus the validated command, the exact child environment and the
|
|
247
|
+
* dependency-inputs hash. The tree, the command, the environment and the
|
|
248
|
+
* dependencies all participate — never the argv alone, which is the proof
|
|
249
|
+
* cache phase 5 refuses. Key order is fixed by construction (`env` arrives in
|
|
250
|
+
* `VERIFICATION_ENV_BASE_NAMES`-then-declared order and the canonical object
|
|
251
|
+
* is one literal), so the hash is stable across the process restart a resume
|
|
252
|
+
* is.
|
|
253
|
+
*
|
|
254
|
+
* @param {{treeFingerprint: string|null, headTree: string|null, command: VerificationCommand, env: Record<string, string>, dependencyHash: string}} parts
|
|
255
|
+
* @returns {string}
|
|
256
|
+
*/
|
|
257
|
+
export function commandCheckpointIdentity({ treeFingerprint, headTree, command, env, dependencyHash }) {
|
|
258
|
+
const canonical = { dependencyHash, command, env, headTree, treeFingerprint };
|
|
259
|
+
return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/**
|
|
263
|
+
* Read back a written checkpoint, failing closed: anything malformed throws
|
|
264
|
+
* and the caller re-runs the command rather than serving or crashing. The
|
|
265
|
+
* recorded result is re-compacted on read, so a hand-edited or oversized
|
|
266
|
+
* record is either bounded to exactly what persisted state may carry or
|
|
267
|
+
* rejected.
|
|
268
|
+
*
|
|
269
|
+
* @param {unknown} value
|
|
270
|
+
* @param {string} [label]
|
|
271
|
+
* @returns {CommandCheckpoint}
|
|
272
|
+
*/
|
|
273
|
+
export function parseCommandCheckpoint(value, label = "command checkpoint") {
|
|
274
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new TypeError(`${label} must be an object`);
|
|
275
|
+
const record = /** @type {Record<string, unknown>} */ (value);
|
|
276
|
+
for (const key of Object.keys(record)) {
|
|
277
|
+
if (key !== "identity" && key !== "result") throw new TypeError(`${label} has unexpected field ${key}`);
|
|
278
|
+
}
|
|
279
|
+
if (typeof record.identity !== "string" || !record.identity || Buffer.byteLength(record.identity, "utf8") > 128) {
|
|
280
|
+
throw new TypeError(`${label}.identity must be a hash of at most 128 bytes`);
|
|
281
|
+
}
|
|
282
|
+
const result = /** @type {Record<string, unknown>} */ (record.result);
|
|
283
|
+
if (!result || typeof result !== "object" || Array.isArray(result) || typeof result.passed !== "boolean") {
|
|
284
|
+
throw new TypeError(`${label}.result must be a single-command verification result`);
|
|
285
|
+
}
|
|
286
|
+
if (!Array.isArray(result.commands) || result.commands.length !== 1) {
|
|
287
|
+
throw new TypeError(`${label}.result must carry exactly one command`);
|
|
288
|
+
}
|
|
289
|
+
const compacted = compactVerification({ passed: result.passed, commands: /** @type {VerificationCommandResult[]} */ (result.commands) });
|
|
290
|
+
if (compacted.commands.length !== 1) throw new TypeError(`${label}.result exceeds the persisted command bounds`);
|
|
291
|
+
return { identity: record.identity, result: compacted.commands[0] };
|
|
292
|
+
}
|
|
293
|
+
|
|
228
294
|
|
|
229
295
|
/**
|
|
230
296
|
* One proof, one source. A command that declares `requirementId` says it is
|