faberun 0.29.0 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/campaign/campaign-brief.mjs +17 -6
- package/src/campaign/chain.mjs +124 -8
- package/src/campaign/ledger.mjs +19 -7
- package/src/campaign/metrics-command.mjs +68 -9
- package/src/campaign/metrics.mjs +75 -45
- package/src/campaign/record.mjs +8 -1
- package/src/campaign/reserve.mjs +340 -0
- package/src/campaign/spend.mjs +229 -0
- package/src/campaign/unpark.mjs +54 -1
- package/src/campaign/watch.mjs +314 -37
- package/src/contract/judge-envelope.mjs +17 -2
- package/src/contract/scope-findings.mjs +3 -1
- package/src/contract/task-packet.mjs +40 -5
- package/src/contract/verification.mjs +67 -1
- package/src/engine/aux-admission.mjs +124 -0
- package/src/engine/capacity.mjs +78 -0
- package/src/engine/dispatch.mjs +116 -23
- package/src/engine/failure-cause.mjs +59 -0
- package/src/engine/lifecycle.mjs +3 -2
- package/src/engine/phase-session.mjs +56 -10
- package/src/engine/prompts.mjs +57 -2
- package/src/engine/reserve-admission.mjs +99 -0
- package/src/engine/resume.mjs +3 -1
- package/src/engine/review.mjs +21 -0
- package/src/engine/run-command.mjs +166 -11
- package/src/engine/scheduler.mjs +71 -88
- package/src/engine/settle.mjs +15 -8
- package/src/engine/settlement-overlap.mjs +143 -0
- package/src/engine/verify.mjs +131 -7
- package/src/harnesses/session-metrics.mjs +112 -20
- package/src/notify/index.mjs +86 -10
- package/src/plan/freeze.mjs +55 -15
- package/src/plan/human-step.mjs +36 -0
- package/src/plan/pipeline.mjs +120 -38
- package/src/plan/preflight.mjs +81 -5
- package/src/plan/progress.mjs +101 -1
- package/src/plan/repo-facts.mjs +431 -55
- package/src/plan/resolve.mjs +13 -4
- package/src/plan/revise-context.mjs +108 -0
- package/src/plan/rounds.mjs +209 -50
- package/src/plan/sizing.mjs +53 -5
- package/src/plan/template.mjs +297 -49
- package/src/repo/integrate.mjs +47 -5
- package/src/repo/scope-closure.mjs +22 -0
- package/src/report/campaign-brief-estimate.mjs +47 -1
- package/src/report/campaign-brief.mjs +27 -0
- package/src/report/locale.mjs +20 -0
- package/src/report/metrics-report.mjs +38 -0
- package/src/report/progress.mjs +194 -10
- package/src/run/usage-windows.mjs +99 -4
- package/src/run/usage.mjs +69 -7
- package/src/web/api.mjs +179 -24
- package/src/web/app.mjs +27 -2
|
@@ -11,7 +11,14 @@
|
|
|
11
11
|
import { canonicalUsage, eventItem, extractJson } from "./protocol.mjs";
|
|
12
12
|
import { finite } from "../util.mjs";
|
|
13
13
|
|
|
14
|
-
/** @typedef {{turns: number, cacheReadInputTokens: number, toolCalls: number, completed: boolean, requests: number, contextFirst: number|null, contextMax: number|null, contextLast: number|null, contextSum: number}} SessionTotals */
|
|
14
|
+
/** @typedef {{turns: number, cacheReadInputTokens: number, cacheWriteInputTokens: number|null, outputTokens: number|null, toolCalls: number, completed: boolean, requests: number, contextFirst: number|null, contextMax: number|null, contextLast: number|null, contextSum: number}} SessionTotals */
|
|
15
|
+
/**
|
|
16
|
+
* The five measured counters one invocation proves, mapped from the
|
|
17
|
+
* harness-reported session usage. `bytes` is never stream-derived -- the
|
|
18
|
+
* ledger measures the dispatched prompt file -- so it is always null here;
|
|
19
|
+
* a counter the harness never reported stays null, never a plausible zero.
|
|
20
|
+
* @typedef {{bytes: number|null, inputTokens: number|null, cacheReadInputTokens: number|null, cacheWriteInputTokens: number|null, outputTokens: number|null}} UsageCounters
|
|
21
|
+
*/
|
|
15
22
|
/**
|
|
16
23
|
* The per-request ledger one transcript proves: provider requests that
|
|
17
24
|
* reported usage, and the context each re-sent -- uncached input (cache
|
|
@@ -155,6 +162,34 @@ function sumRequestUsage(usages) {
|
|
|
155
162
|
return inputTokens === null ? { inputTokens: null, cacheReadInputTokens: null } : { inputTokens, cacheReadInputTokens };
|
|
156
163
|
}
|
|
157
164
|
|
|
165
|
+
/** Sum two counters of which either may be unmeasured; null only when neither measured. @param {number|null} left @param {number|null} right @returns {number|null} */
|
|
166
|
+
export function addMeasured(left, right) {
|
|
167
|
+
return left === null && right === null ? null : (left ?? 0) + (right ?? 0);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* The five usage counters one bounded transcript proves: uncached input and
|
|
172
|
+
* cache reads from the observation the live meter reads, cache writes and
|
|
173
|
+
* output from the session totals the parser folds.
|
|
174
|
+
*
|
|
175
|
+
* @param {string} harness
|
|
176
|
+
* @param {string} stdout bounded transcript tail
|
|
177
|
+
* @returns {UsageCounters}
|
|
178
|
+
*/
|
|
179
|
+
export function liveUsageCounters(harness, stdout) {
|
|
180
|
+
const parser = new SessionMetricsParser(harness);
|
|
181
|
+
parser.push(String(stdout));
|
|
182
|
+
parser.flush();
|
|
183
|
+
const usage = liveUsage(harness, stdout);
|
|
184
|
+
return {
|
|
185
|
+
bytes: null,
|
|
186
|
+
inputTokens: usage.inputTokens,
|
|
187
|
+
cacheReadInputTokens: usage.cacheReadInputTokens,
|
|
188
|
+
cacheWriteInputTokens: parser.totals.cacheWriteInputTokens,
|
|
189
|
+
outputTokens: parser.totals.outputTokens,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
158
193
|
/**
|
|
159
194
|
* Budgeted live meter: one number, with cache reads weighted by the campaign
|
|
160
195
|
* policy so it is comparable with the persisted ledger. A provider that only
|
|
@@ -221,6 +256,9 @@ const TOOL_USE_NEEDLE = Buffer.from('"type":"tool_use"', "utf8");
|
|
|
221
256
|
/** Cache-read evidence spellings across harness streams. */
|
|
222
257
|
const CACHE_READ_PATTERN = /"(?:cache_read_input_tokens|cached_input_tokens|cacheReadInputTokens)":(\d+)/gu;
|
|
223
258
|
|
|
259
|
+
/** Cache-write evidence spellings across harness streams. */
|
|
260
|
+
const CACHE_WRITE_PATTERN = /"(?:cache_creation_input_tokens|cacheWriteInputTokens|cache_write_input_tokens|cache_creation_tokens|cache_write_tokens)":(\d+)/gu;
|
|
261
|
+
|
|
224
262
|
/**
|
|
225
263
|
* Bounded incremental session-metrics parser: fold fixed-size chunks into
|
|
226
264
|
* running rotation totals without ever holding a buffer that scales with the
|
|
@@ -240,6 +278,8 @@ export class SessionMetricsParser {
|
|
|
240
278
|
this.totals = {
|
|
241
279
|
turns: previous.turns ?? 0,
|
|
242
280
|
cacheReadInputTokens: previous.cacheReadInputTokens ?? 0,
|
|
281
|
+
cacheWriteInputTokens: null,
|
|
282
|
+
outputTokens: null,
|
|
243
283
|
toolCalls: previous.toolCalls ?? 0,
|
|
244
284
|
completed: previous.completed === true,
|
|
245
285
|
requests: 0,
|
|
@@ -455,7 +495,11 @@ function foldRecord(harness, totals, record) {
|
|
|
455
495
|
if (harness === "codex") {
|
|
456
496
|
if (record.type === "turn.completed") {
|
|
457
497
|
totals.turns += 1;
|
|
458
|
-
|
|
498
|
+
const usage = canonicalUsage(record.usage);
|
|
499
|
+
totals.cacheReadInputTokens = Math.max(totals.cacheReadInputTokens, usage.cacheReadInputTokens ?? 0);
|
|
500
|
+
// Codex counts input and output cumulatively per turn; it reports no
|
|
501
|
+
// cache-write counter at all, so the write total stays unmeasured.
|
|
502
|
+
if (usage.outputTokens !== null) totals.outputTokens = Math.max(totals.outputTokens ?? 0, usage.outputTokens);
|
|
459
503
|
} else if (record.type === "item.completed" && CODEX_TOOL_ITEM_TYPES.has(String(eventItem(record)?.type))) {
|
|
460
504
|
totals.toolCalls += 1;
|
|
461
505
|
}
|
|
@@ -465,14 +509,16 @@ function foldRecord(harness, totals, record) {
|
|
|
465
509
|
if (record.type === "assistant") {
|
|
466
510
|
const message = /** @type {Record<string, unknown>} */ (record.message ?? {});
|
|
467
511
|
totals.turns += 1;
|
|
468
|
-
|
|
469
|
-
|
|
512
|
+
const usage = canonicalUsage(message.usage);
|
|
513
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
514
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(message.usage));
|
|
515
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
516
|
+
foldRequestUsage(totals, usage);
|
|
470
517
|
totals.toolCalls += Array.isArray(message.content)
|
|
471
518
|
? message.content.filter((/** @type {{type?: unknown}} */ block) => block?.type === "tool_use").length
|
|
472
519
|
: 0;
|
|
473
520
|
} else if (record.type === "result") {
|
|
474
|
-
|
|
475
|
-
if (sessionTotal !== null) totals.cacheReadInputTokens = sessionTotal;
|
|
521
|
+
foldSessionTotal(totals, record.usage);
|
|
476
522
|
}
|
|
477
523
|
return;
|
|
478
524
|
}
|
|
@@ -487,13 +533,15 @@ function foldRecord(harness, totals, record) {
|
|
|
487
533
|
// activity -- three kills on 2026-09-16 at 903s against a 900s limit.
|
|
488
534
|
if (record.type === transcript.usage) {
|
|
489
535
|
totals.turns += 1;
|
|
490
|
-
|
|
491
|
-
|
|
536
|
+
const usage = canonicalUsage(record.usage);
|
|
537
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
538
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(record.usage));
|
|
539
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
540
|
+
foldRequestUsage(totals, usage);
|
|
492
541
|
} else if (record.type === transcript.tool) {
|
|
493
542
|
totals.toolCalls += 1;
|
|
494
543
|
} else if (record.type === transcript.completed || record.type === transcript.failed) {
|
|
495
|
-
|
|
496
|
-
if (sessionTotal !== null) totals.cacheReadInputTokens = sessionTotal;
|
|
544
|
+
foldSessionTotal(totals, record.usage);
|
|
497
545
|
}
|
|
498
546
|
return;
|
|
499
547
|
}
|
|
@@ -507,32 +555,57 @@ function foldRecord(harness, totals, record) {
|
|
|
507
555
|
if (step) {
|
|
508
556
|
if (step.step_type === "agent_response" && step.state === "DONE") {
|
|
509
557
|
totals.turns += 1;
|
|
510
|
-
|
|
511
|
-
|
|
558
|
+
const usage = canonicalUsage(step.usage);
|
|
559
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
560
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(step.usage));
|
|
561
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
562
|
+
foldRequestUsage(totals, usage);
|
|
512
563
|
} else if (step.step_type === "tool" && (step.state === "DONE" || step.state === "ERROR")) {
|
|
513
564
|
totals.toolCalls += 1;
|
|
514
565
|
}
|
|
515
566
|
} else if (record.event === "result") {
|
|
516
|
-
|
|
517
|
-
if (sessionTotal !== null) totals.cacheReadInputTokens = sessionTotal;
|
|
567
|
+
foldSessionTotal(totals, agyResult(record)?.usage);
|
|
518
568
|
}
|
|
519
569
|
return;
|
|
520
570
|
}
|
|
521
571
|
if (harness === "exec-jsonl" && record.type === "run.completed") {
|
|
522
572
|
// The protocol carries no tool events; only a completed run proves a turn.
|
|
523
573
|
totals.turns += 1;
|
|
524
|
-
|
|
525
|
-
|
|
574
|
+
const usage = canonicalUsage(record.usage);
|
|
575
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
576
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(record.usage));
|
|
577
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
578
|
+
foldRequestUsage(totals, usage);
|
|
526
579
|
}
|
|
527
580
|
if (harness === "replay" && typeof record.status === "string") {
|
|
528
581
|
// A replayed envelope is the whole invocation: one completed turn, no
|
|
529
582
|
// tool events, usage only in the terminal record.
|
|
530
583
|
totals.turns += 1;
|
|
531
|
-
|
|
532
|
-
|
|
584
|
+
const usage = canonicalUsage(record.usage);
|
|
585
|
+
totals.cacheReadInputTokens += usage.cacheReadInputTokens ?? 0;
|
|
586
|
+
totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWriteTokensOf(record.usage));
|
|
587
|
+
totals.outputTokens = addMeasured(totals.outputTokens, usage.outputTokens);
|
|
588
|
+
foldRequestUsage(totals, usage);
|
|
533
589
|
}
|
|
534
590
|
}
|
|
535
591
|
|
|
592
|
+
/** Fold a terminal record's session totals over the mid-run sums; an unreported total leaves the running sum alone. @param {SessionTotals} totals @param {unknown} raw */
|
|
593
|
+
function foldSessionTotal(totals, raw) {
|
|
594
|
+
const usage = canonicalUsage(raw);
|
|
595
|
+
if (usage.cacheReadInputTokens !== null) totals.cacheReadInputTokens = usage.cacheReadInputTokens;
|
|
596
|
+
const writeTotal = cacheWriteTokensOf(raw);
|
|
597
|
+
if (writeTotal !== null) totals.cacheWriteInputTokens = writeTotal;
|
|
598
|
+
if (usage.outputTokens !== null) totals.outputTokens = usage.outputTokens;
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
/** The cache-write tokens one raw usage object reports, across the harness spellings; null when none. @param {unknown} raw @returns {number|null} */
|
|
602
|
+
function cacheWriteTokensOf(raw) {
|
|
603
|
+
if (!raw || typeof raw !== "object") return null;
|
|
604
|
+
const usage = /** @type {Record<string, unknown>} */ (raw);
|
|
605
|
+
return finite(usage.cache_creation_input_tokens ?? usage.cacheWriteInputTokens
|
|
606
|
+
?? usage.cache_write_input_tokens ?? usage.cache_creation_tokens ?? usage.cache_write_tokens);
|
|
607
|
+
}
|
|
608
|
+
|
|
536
609
|
/**
|
|
537
610
|
* Fold one provider request's usage into the per-request ledger. A request
|
|
538
611
|
* with no measured input is not one the ledger can count.
|
|
@@ -575,15 +648,21 @@ function agyResult(record) {
|
|
|
575
648
|
*/
|
|
576
649
|
function foldFragmentRecord(harness, totals, fragments) {
|
|
577
650
|
const text = `${fragments.head}\n${fragments.tail}`;
|
|
651
|
+
// Output tokens have no fragment scanner: a record that outgrew the
|
|
652
|
+
// retention bound leaves the output total unmeasured rather than guessed.
|
|
578
653
|
if (harness === "claude") {
|
|
579
654
|
if (text.includes('"type":"assistant"')) {
|
|
580
655
|
totals.turns += 1;
|
|
581
656
|
totals.toolCalls += fragments.toolUse;
|
|
582
657
|
const cacheRead = lastCacheRead(text);
|
|
583
658
|
if (cacheRead !== null) totals.cacheReadInputTokens += cacheRead;
|
|
659
|
+
const cacheWrite = lastCacheWrite(text);
|
|
660
|
+
if (cacheWrite !== null) totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWrite);
|
|
584
661
|
} else if (text.includes('"type":"result"')) {
|
|
585
|
-
const
|
|
586
|
-
if (
|
|
662
|
+
const cacheRead = lastCacheRead(text);
|
|
663
|
+
if (cacheRead !== null) totals.cacheReadInputTokens = cacheRead;
|
|
664
|
+
const cacheWrite = lastCacheWrite(text);
|
|
665
|
+
if (cacheWrite !== null) totals.cacheWriteInputTokens = cacheWrite;
|
|
587
666
|
totals.completed = true;
|
|
588
667
|
}
|
|
589
668
|
return;
|
|
@@ -607,6 +686,8 @@ function foldFragmentRecord(harness, totals, fragments) {
|
|
|
607
686
|
totals.turns += 1;
|
|
608
687
|
const cacheRead = lastCacheRead(text);
|
|
609
688
|
if (cacheRead !== null) totals.cacheReadInputTokens += cacheRead;
|
|
689
|
+
const cacheWrite = lastCacheWrite(text);
|
|
690
|
+
if (cacheWrite !== null) totals.cacheWriteInputTokens = addMeasured(totals.cacheWriteInputTokens, cacheWrite);
|
|
610
691
|
totals.completed = true;
|
|
611
692
|
}
|
|
612
693
|
}
|
|
@@ -697,3 +778,14 @@ function lastCacheRead(text) {
|
|
|
697
778
|
const matches = [...text.matchAll(CACHE_READ_PATTERN)];
|
|
698
779
|
return matches.length > 0 ? Number(matches.at(-1)?.[1]) : null;
|
|
699
780
|
}
|
|
781
|
+
|
|
782
|
+
/**
|
|
783
|
+
* The last cache-write number in a fragment text, or null.
|
|
784
|
+
*
|
|
785
|
+
* @param {string} text
|
|
786
|
+
* @returns {number|null}
|
|
787
|
+
*/
|
|
788
|
+
function lastCacheWrite(text) {
|
|
789
|
+
const matches = [...text.matchAll(CACHE_WRITE_PATTERN)];
|
|
790
|
+
return matches.length > 0 ? Number(matches.at(-1)?.[1]) : null;
|
|
791
|
+
}
|
package/src/notify/index.mjs
CHANGED
|
@@ -17,6 +17,18 @@
|
|
|
17
17
|
* fallback. The macOS notifier is reachable only by setting
|
|
18
18
|
* `FABERUN_NOTIFY_BIN=os-macos`, an explicit opt-in, never a default.
|
|
19
19
|
*
|
|
20
|
+
* A transport may also answer with a receipt: the last line it prints, as
|
|
21
|
+
* JSON carrying the `messageId` it created or edited. The campaign watcher's
|
|
22
|
+
* automatic progress update is the consumer -- its text is rendered from the
|
|
23
|
+
* shared projection (`buildCampaignProgress`), never a model, and when a
|
|
24
|
+
* previous progress message of the same campaign is still inside the
|
|
25
|
+
* fifteen-minute window (`progressEditTarget`) the update travels as an edit
|
|
26
|
+
* of that message. The anchor naming the editable message is written only
|
|
27
|
+
* after the transport's receipt returns, so a delivery is claimed only on
|
|
28
|
+
* evidence; a transport that prints no receipt simply always gets a new
|
|
29
|
+
* message. An attention event never consults the anchor: it is always a new
|
|
30
|
+
* message.
|
|
31
|
+
*
|
|
20
32
|
* Measured 2026-09-16: `test/cli/cli.test.mjs`'s two notifier fixtures were
|
|
21
33
|
* instrumented with `{at, phase}` timelines (spawned, stdin-end, exit) and run
|
|
22
34
|
* over 80 times (targeted loops, four-way parallel full-file bursts, and a
|
|
@@ -92,7 +104,7 @@ export const NOTIFY_LANG_ENV = "FABERUN_NOTIFY_LANG";
|
|
|
92
104
|
export const NOTIFY_ENV_NAMES = Object.freeze([NOTIFY_BIN_ENV, NOTIFY_SESSION_ENV, NOTIFY_EVENTS_ENV, NOTIFY_LANG_ENV]);
|
|
93
105
|
|
|
94
106
|
/** Every event type the dispatcher can be asked to deliver. */
|
|
95
|
-
export const NOTIFY_EVENT_TYPES = Object.freeze(["node.terminal", "run.terminal", "attention", "advisory"]);
|
|
107
|
+
export const NOTIFY_EVENT_TYPES = Object.freeze(["node.terminal", "run.terminal", "attention", "advisory", "progress"]);
|
|
96
108
|
|
|
97
109
|
/**
|
|
98
110
|
* What leaves the controller when `FABERUN_NOTIFY_EVENTS` is unset: a phase
|
|
@@ -107,6 +119,30 @@ export const NOTIFY_EVENT_TYPES = Object.freeze(["node.terminal", "run.terminal"
|
|
|
107
119
|
*/
|
|
108
120
|
export const DEFAULT_NOTIFY_EVENTS = Object.freeze(["run.terminal", "attention", "advisory"]);
|
|
109
121
|
|
|
122
|
+
/**
|
|
123
|
+
* How long a delivered progress message stays editable, from the phase spec's
|
|
124
|
+
* constraint: a progress message of the same campaign inside this window is
|
|
125
|
+
* edited in place, an event that demands attention is always a new message.
|
|
126
|
+
*/
|
|
127
|
+
export const PROGRESS_EDIT_WINDOW_MS = 15 * 60_000;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* The message id an automatic progress update may edit, from the receipt-
|
|
131
|
+
* backed anchor of the last one: an anchor inside the window names its
|
|
132
|
+
* message; an expired, foreign, malformed or absent one edits nothing and the
|
|
133
|
+
* next update is a new message. An attention event never consults this --
|
|
134
|
+
* attention is always a new message.
|
|
135
|
+
*
|
|
136
|
+
* @param {{messageId: string, at: number}|null} anchor the anchor the last receipt wrote
|
|
137
|
+
* @param {number} nowMs epoch milliseconds
|
|
138
|
+
* @returns {string|null}
|
|
139
|
+
*/
|
|
140
|
+
export function progressEditTarget(anchor, nowMs) {
|
|
141
|
+
if (!anchor || typeof anchor.messageId !== "string" || !anchor.messageId) return null;
|
|
142
|
+
if (typeof anchor.at !== "number" || !Number.isFinite(anchor.at)) return null;
|
|
143
|
+
return nowMs - anchor.at < PROGRESS_EDIT_WINDOW_MS ? anchor.messageId : null;
|
|
144
|
+
}
|
|
145
|
+
|
|
110
146
|
/** The two languages the message renders its wording in; `FABERUN_NOTIFY_LANG` may name either. */
|
|
111
147
|
export const NOTIFY_LANGUAGES = Object.freeze(["en", "pt"]);
|
|
112
148
|
|
|
@@ -209,9 +245,9 @@ export const NOTIFY_NO_TRANSPORT_WARNING = "no human notification transport is c
|
|
|
209
245
|
/** @typedef {Record<string, unknown>} JsonObject */
|
|
210
246
|
/** @typedef {{schemaVersion: number, eventId: string, at: string, type: string, campaignId: string|null, runId: string|null, nodeId: string|null, status: string|null, errorCode: string|null, dedupeKey: string, summary: string}} InboxEntry */
|
|
211
247
|
/** @typedef {{type: string, dedupeKey: string, summary: string, at?: string, campaignId?: string|null, runId?: string|null, nodeId?: string|null, status?: string|null, errorCode?: string|null}} InboxEvent */
|
|
212
|
-
/** @typedef {{type: "node.terminal"|"run.terminal"|"attention"|"advisory", runId: string|null, campaignId?: string|null, nodeId?: string|null, status?: string|null, attempt?: number|null, errorCode?: string|null, done?: number|null, total?: number|null, dedupeKey?: string|null, runDir?: string|null, costUsd?: number|null, summary?: string|null, eventId?: string}} NotifyEvent */
|
|
213
|
-
/** @typedef {{id: string, ok: boolean, error?: string}} TransportOutcome one transport's own outcome, named so the receipt says which took the message */
|
|
214
|
-
/** @typedef {{ok: boolean, error?: string, noTransport?: boolean, transports?: TransportOutcome[]}} DeliveryResult */
|
|
248
|
+
/** @typedef {{type: "node.terminal"|"run.terminal"|"attention"|"advisory"|"progress", runId: string|null, campaignId?: string|null, nodeId?: string|null, status?: string|null, attempt?: number|null, errorCode?: string|null, done?: number|null, total?: number|null, dedupeKey?: string|null, runDir?: string|null, costUsd?: number|null, summary?: string|null, eventId?: string, editOfMessageId?: string|null}} NotifyEvent */
|
|
249
|
+
/** @typedef {{id: string, ok: boolean, error?: string, messageId?: string}} TransportOutcome one transport's own outcome, named so the receipt says which took the message; `messageId` is the id a transport receipt named for the message it created or edited */
|
|
250
|
+
/** @typedef {{ok: boolean, error?: string, noTransport?: boolean, messageId?: string, transports?: TransportOutcome[]}} DeliveryResult */
|
|
215
251
|
|
|
216
252
|
/**
|
|
217
253
|
* Render the message for an event. For `node.terminal`, `run.terminal` and
|
|
@@ -219,7 +255,10 @@ export const NOTIFY_NO_TRANSPORT_WARNING = "no human notification transport is c
|
|
|
219
255
|
* own persisted state (its contract, its node snapshots, its usage) from
|
|
220
256
|
* `event.runDir` -- so whatever calls this function once gets exactly the
|
|
221
257
|
* string every audience of that event sees: the inbox entry's `summary` and
|
|
222
|
-
* the transport's stdin are never rendered separately and never drift.
|
|
258
|
+
* the transport's stdin are never rendered separately and never drift. A
|
|
259
|
+
* `progress` event is never rendered here: the campaign watcher renders it
|
|
260
|
+
* from the shared projection and passes it pre-rendered, so a progress event
|
|
261
|
+
* without a summary is a caller bug and throws.
|
|
223
262
|
*
|
|
224
263
|
* `runDir` is absent, or names a directory with no readable run scaffold
|
|
225
264
|
* (no `contract.json`, no `run.json`, a node snapshot that fails validation),
|
|
@@ -438,13 +477,16 @@ export function appendInbox(runsDir, event) {
|
|
|
438
477
|
* text. The result is `ok` when any one of them took the message, carries
|
|
439
478
|
* every transport's own outcome for the receipt, and is
|
|
440
479
|
* `{ok: false, noTransport: true}` when nothing at all is bound -- without
|
|
441
|
-
* spawning or connecting anything.
|
|
480
|
+
* spawning or connecting anything. When a transport answered with a receipt
|
|
481
|
+
* naming the message it created or edited, that id is surfaced as `messageId`
|
|
482
|
+
* so the caller can claim exactly what the transport acknowledged -- never
|
|
483
|
+
* more.
|
|
442
484
|
*
|
|
443
485
|
* @param {{type: string, summary: string, campaignId?: string|null, [key: string]: unknown}} event
|
|
444
486
|
* @param {{bin?: string, env?: NodeJS.ProcessEnv, spawn?: typeof defaultSpawn, timeoutMs?: number}} [options]
|
|
445
487
|
* @returns {Promise<DeliveryResult>}
|
|
446
488
|
*/
|
|
447
|
-
async function deliverNotification(event, options = {}) {
|
|
489
|
+
export async function deliverNotification(event, options = {}) {
|
|
448
490
|
const env = options.env ?? process.env;
|
|
449
491
|
const bin = options.bin ?? env[NOTIFY_BIN_ENV];
|
|
450
492
|
const targets = resolveSessionTargets(env);
|
|
@@ -461,20 +503,50 @@ async function deliverNotification(event, options = {}) {
|
|
|
461
503
|
if (!attempts.length && !targets.length) return { ok: false, noTransport: true, transports: [] };
|
|
462
504
|
const transports = [...(await Promise.all(attempts)), ...(await sessions)];
|
|
463
505
|
const failures = transports.filter((outcome) => !outcome.ok).map((outcome) => `${outcome.id}: ${outcome.error ?? "failed"}`);
|
|
506
|
+
const messageId = transports.map((outcome) => outcome.messageId).find((id) => typeof id === "string" && id !== "");
|
|
464
507
|
return {
|
|
465
508
|
ok: transports.some((outcome) => outcome.ok),
|
|
509
|
+
...(messageId ? { messageId } : {}),
|
|
466
510
|
...(failures.length ? { error: failures.join("; ") } : {}),
|
|
467
511
|
transports,
|
|
468
512
|
};
|
|
469
513
|
}
|
|
470
514
|
|
|
515
|
+
/**
|
|
516
|
+
* The transport's delivery receipt, read from the last non-empty line it
|
|
517
|
+
* printed: a JSON object whose `messageId` names the message it created or
|
|
518
|
+
* edited. A transport that prints nothing else, prints non-JSON, or exits
|
|
519
|
+
* non-zero yields no receipt, and the caller claims no editable message from
|
|
520
|
+
* it.
|
|
521
|
+
*
|
|
522
|
+
* @param {string} stdout
|
|
523
|
+
* @returns {{messageId?: string}}
|
|
524
|
+
*/
|
|
525
|
+
function transportReceipt(stdout) {
|
|
526
|
+
const lines = stdout.split("\n");
|
|
527
|
+
for (let index = lines.length - 1; index >= 0; index -= 1) {
|
|
528
|
+
const trimmed = lines[index].trim();
|
|
529
|
+
if (!trimmed) continue;
|
|
530
|
+
try {
|
|
531
|
+
const parsed = JSON.parse(trimmed);
|
|
532
|
+
const messageId = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed.messageId : undefined;
|
|
533
|
+
return typeof messageId === "string" && messageId ? { messageId } : {};
|
|
534
|
+
} catch {
|
|
535
|
+
return {};
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
return {};
|
|
539
|
+
}
|
|
540
|
+
|
|
471
541
|
/**
|
|
472
542
|
* Resolves on the child's own `exit`, not `close`: `close` waits for every
|
|
473
543
|
* stdio stream to finish closing, and a piped `stderr` fd can be held open
|
|
474
544
|
* by a lingering grandchild or be slow to flush under load well after the
|
|
475
545
|
* notifier process itself has terminated. Gating delivery on that stream
|
|
476
546
|
* closing (as the previous implementation did) could stall a healthy,
|
|
477
|
-
* already-exited delivery until the timeout fired.
|
|
547
|
+
* already-exited delivery until the timeout fired. A zero exit is the
|
|
548
|
+
* delivery; whatever the transport printed on stdout is its receipt, read
|
|
549
|
+
* by `transportReceipt`.
|
|
478
550
|
*
|
|
479
551
|
* @param {string} bin
|
|
480
552
|
* @param {JsonObject} event
|
|
@@ -490,7 +562,7 @@ function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs = notificati
|
|
|
490
562
|
// names its interpreter. Either spawns as EFTYPE unhandled, which is a
|
|
491
563
|
// notification that silently never arrives.
|
|
492
564
|
const invocation = spawnInvocation(bin, []);
|
|
493
|
-
child = spawn(invocation.command, invocation.args, { stdio: ["pipe", "
|
|
565
|
+
child = spawn(invocation.command, invocation.args, { stdio: ["pipe", "pipe", "pipe"], env: process.env, ...invocation.options });
|
|
494
566
|
} catch (error) {
|
|
495
567
|
resolveDelivery({ ok: false, error: errorMessage(error) });
|
|
496
568
|
return;
|
|
@@ -504,11 +576,15 @@ function spawnDeliver(bin, event, { spawn = defaultSpawn, timeoutMs = notificati
|
|
|
504
576
|
resolveDelivery(result);
|
|
505
577
|
};
|
|
506
578
|
let stderr = "";
|
|
579
|
+
let stdout = "";
|
|
507
580
|
child.stderr?.on("data", (chunk) => {
|
|
508
581
|
stderr = `${stderr}${chunk}`.slice(-1024);
|
|
509
582
|
});
|
|
583
|
+
child.stdout?.on("data", (chunk) => {
|
|
584
|
+
stdout = `${stdout}${chunk}`.slice(-2048);
|
|
585
|
+
});
|
|
510
586
|
child.once("error", (error) => finish({ ok: false, error: errorMessage(error) }));
|
|
511
|
-
child.once("exit", (code) => finish(code === 0 ? { ok: true } : { ok: false, error: stderr || `notification exited ${code}` }));
|
|
587
|
+
child.once("exit", (code) => finish(code === 0 ? { ok: true, ...transportReceipt(stdout) } : { ok: false, error: stderr || `notification exited ${code}` }));
|
|
512
588
|
const timer = setTimeout(() => {
|
|
513
589
|
try {
|
|
514
590
|
child.kill("SIGTERM");
|
package/src/plan/freeze.mjs
CHANGED
|
@@ -35,7 +35,8 @@ import { NODE_VALUE_OPTIONS, assertFilteredProofsNameTheirTest } from "./proof-s
|
|
|
35
35
|
/** @typedef {{path: string, digest: string}} PlanSpecIdentity */
|
|
36
36
|
/** @typedef {{formatVersion: number, contractDigest: string, spec?: PlanSpecIdentity, phases?: PlanPhase[], humanSteps?: PlanHumanStep[], provenance: PlanProvenance}} FrozenPlan */
|
|
37
37
|
/** @typedef {{ok: boolean, digest: string, expectedDigest: string}} FrozenPlanVerdict */
|
|
38
|
-
/** @typedef {{
|
|
38
|
+
/** @typedef {{argv: string[], measuredMs: number|null, eligible: boolean}} MeasuredCandidate */
|
|
39
|
+
/** @typedef {{scripts?: Record<string, string>, verificationCandidates: MeasuredCandidate[]}} MeasuredFacts */
|
|
39
40
|
|
|
40
41
|
const PLAN_FORMAT_VERSION = 1;
|
|
41
42
|
|
|
@@ -74,13 +75,17 @@ function includedDirectories(arg, directories) {
|
|
|
74
75
|
* the sum of the `node --test <dir>` candidates it includes, `npm test`
|
|
75
76
|
* resolved through the `test` script. A lower bound when the command also
|
|
76
77
|
* runs files no candidate measured; null when it includes nothing measured.
|
|
78
|
+
* Only an eligible candidate with a numeric duration counts, exact or
|
|
79
|
+
* aggregated: a candidate killed at its timeout persists measuredMs null
|
|
80
|
+
* with eligible false, so neither an exact read nor a directory sum may
|
|
81
|
+
* plan from it.
|
|
77
82
|
*
|
|
78
83
|
* @param {string[]} argv
|
|
79
84
|
* @param {MeasuredFacts} facts
|
|
80
85
|
* @returns {number|null}
|
|
81
86
|
*/
|
|
82
87
|
function measuredMsFor(argv, facts) {
|
|
83
|
-
const candidates = facts.verificationCandidates;
|
|
88
|
+
const candidates = facts.verificationCandidates.filter((candidate) => candidate.eligible === true && typeof candidate.measuredMs === "number");
|
|
84
89
|
const exact = candidates.find((candidate) => candidate.argv.join(" ") === argv.join(" "));
|
|
85
90
|
if (exact) return exact.measuredMs;
|
|
86
91
|
const npmTest = argv[0] === "npm" && ["test", "run test"].includes(argv.slice(1).join(" "));
|
|
@@ -135,6 +140,33 @@ export function assertTimeoutsCoverMeasured(contract, facts) {
|
|
|
135
140
|
if (problems.size) throw new TypeError(`verification timeouts do not cover their measured durations: ${[...problems].join("; ")}`);
|
|
136
141
|
}
|
|
137
142
|
|
|
143
|
+
/**
|
|
144
|
+
* Raise whichever of a command list's timeouts sit under
|
|
145
|
+
* `MEASURED_TIMEOUT_MARGIN` times their measured duration, recording each
|
|
146
|
+
* raise under `label`. Returns the list unchanged (by identity) when nothing
|
|
147
|
+
* needed raising, so the caller can tell whether to re-list.
|
|
148
|
+
*
|
|
149
|
+
* @param {import("../contract/index.mjs").VerificationCommand[]} commands
|
|
150
|
+
* @param {MeasuredFacts} facts
|
|
151
|
+
* @param {string} label
|
|
152
|
+
* @param {string[]} raised
|
|
153
|
+
* @returns {import("../contract/index.mjs").VerificationCommand[]}
|
|
154
|
+
*/
|
|
155
|
+
function raiseCommandTimeouts(commands, facts, label, raised) {
|
|
156
|
+
let changed = false;
|
|
157
|
+
const next = commands.map((command) => {
|
|
158
|
+
const measuredMs = measuredMsFor(command.argv, facts);
|
|
159
|
+
if (measuredMs === null) return command;
|
|
160
|
+
const requiredSec = Math.ceil((measuredMs * MEASURED_TIMEOUT_MARGIN) / 1_000);
|
|
161
|
+
const timeoutSec = command.timeoutSec ?? 120;
|
|
162
|
+
if (timeoutSec >= requiredSec || requiredSec > VERIFICATION_LIMITS.maxTimeoutSec) return command;
|
|
163
|
+
raised.push(`${label}: ${command.argv.join(" ")} ${timeoutSec}s -> ${requiredSec}s`);
|
|
164
|
+
changed = true;
|
|
165
|
+
return { ...command, timeoutSec: requiredSec };
|
|
166
|
+
});
|
|
167
|
+
return changed ? next : commands;
|
|
168
|
+
}
|
|
169
|
+
|
|
138
170
|
/**
|
|
139
171
|
* Raise every plan node's verification timeout that sits under
|
|
140
172
|
* `MEASURED_TIMEOUT_MARGIN` times its measured duration to exactly that
|
|
@@ -145,6 +177,11 @@ export function assertTimeoutsCoverMeasured(contract, facts) {
|
|
|
145
177
|
* plan with no critical from review, contested only because three nodes gave
|
|
146
178
|
* `node --test test/harnesses` 120s against a measured 84.4s (bound 127s).
|
|
147
179
|
*
|
|
180
|
+
* The plan's own contract-level suites (RM-107) are raised here too, over the
|
|
181
|
+
* same measurement: `assertTimeoutsCoverMeasured` reads them, so a repair that
|
|
182
|
+
* skipped them would leave the check naming a repair this function does not
|
|
183
|
+
* make. Its label is the suite key, since a contract suite belongs to no node.
|
|
184
|
+
*
|
|
148
185
|
* @param {import("./template.mjs").PlanOutput} plan
|
|
149
186
|
* @param {MeasuredFacts} facts
|
|
150
187
|
* @returns {{plan: import("./template.mjs").PlanOutput, raised: string[]}}
|
|
@@ -152,19 +189,22 @@ export function assertTimeoutsCoverMeasured(contract, facts) {
|
|
|
152
189
|
export function raiseTimeoutsToMeasured(plan, facts) {
|
|
153
190
|
/** @type {string[]} */
|
|
154
191
|
const raised = [];
|
|
155
|
-
const nodes = plan.nodes.map((node) =>
|
|
156
|
-
|
|
157
|
-
verification
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
192
|
+
const nodes = plan.nodes.map((node) => {
|
|
193
|
+
const verification = raiseCommandTimeouts(node.verification ?? [], facts, node.id, raised);
|
|
194
|
+
return verification === node.verification ? node : { ...node, verification };
|
|
195
|
+
});
|
|
196
|
+
const sharedVerification = raiseCommandTimeouts(plan.sharedVerification ?? [], facts, "sharedVerification", raised);
|
|
197
|
+
const finalVerification = raiseCommandTimeouts(plan.finalVerification ?? [], facts, "finalVerification", raised);
|
|
198
|
+
if (raised.length === 0) return { plan, raised };
|
|
199
|
+
return {
|
|
200
|
+
plan: {
|
|
201
|
+
...plan,
|
|
202
|
+
nodes,
|
|
203
|
+
...(plan.sharedVerification === undefined ? {} : { sharedVerification }),
|
|
204
|
+
...(plan.finalVerification === undefined ? {} : { finalVerification }),
|
|
205
|
+
},
|
|
206
|
+
raised,
|
|
207
|
+
};
|
|
168
208
|
}
|
|
169
209
|
|
|
170
210
|
/** @returns {string} the installed package's own version, read once per call so a freeze always names the toolchain that produced it */
|
package/src/plan/human-step.mjs
CHANGED
|
@@ -5,6 +5,16 @@
|
|
|
5
5
|
* freeze.mjs can carry it on the frozen plan as an explicit stop, and the
|
|
6
6
|
* Campaign Brief can list it among the human decisions. Pure text matching:
|
|
7
7
|
* no file I/O, no git, no model.
|
|
8
|
+
*
|
|
9
|
+
* The same module owns the other thing only an operator can do, because it
|
|
10
|
+
* is the same boundary at a different source: a packet refusal the failure
|
|
11
|
+
* classifier named `ambiguous_requirement` (R5, campaign-efficiency phase 4)
|
|
12
|
+
* is not repairable by widening the packet — the requirement itself can be
|
|
13
|
+
* read more than one way — so it becomes a durable question
|
|
14
|
+
* (`durableQuestion`) the run carries until `resume --answer <node>` closes
|
|
15
|
+
* it, instead of another widening attempt. The source is the run's failure
|
|
16
|
+
* journal, not the spec's constraints; the discipline is unchanged: pure
|
|
17
|
+
* record shaping, no file I/O, no git, no model.
|
|
8
18
|
*/
|
|
9
19
|
|
|
10
20
|
/** @typedef {{requirementId: string, step: string, command: string}} HumanStep */
|
|
@@ -36,3 +46,29 @@ export function collectHumanSteps(requirements) {
|
|
|
36
46
|
.map((requirement) => detectHumanStep(requirement))
|
|
37
47
|
.filter((step) => step !== null);
|
|
38
48
|
}
|
|
49
|
+
|
|
50
|
+
/** @typedef {{node: string, artifactVersion: string, question: string}} DurableQuestion */
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The durable question an ambiguous-requirement refusal becomes. Null for
|
|
54
|
+
* every other cause: a read or write gap is the packet's own defect and a
|
|
55
|
+
* widening round can repair it, but an ambiguity is a property of the
|
|
56
|
+
* requirement the packet was written from, and no widening of read or write
|
|
57
|
+
* scope can answer it — another attempt would re-read the same prose.
|
|
58
|
+
*
|
|
59
|
+
* The question names the node and the packet version it refuses so the
|
|
60
|
+
* operator's answer is bound to the same key the failure journal uses: a
|
|
61
|
+
* widened packet changes the version, and with it the question.
|
|
62
|
+
*
|
|
63
|
+
* @param {{cause: string, node: string, artifactVersion: string}} record a
|
|
64
|
+
* failureCause record as the reauthor journal writes it
|
|
65
|
+
* @returns {DurableQuestion|null}
|
|
66
|
+
*/
|
|
67
|
+
export function durableQuestion(record) {
|
|
68
|
+
if (record.cause !== "ambiguous_requirement") return null;
|
|
69
|
+
return {
|
|
70
|
+
node: record.node,
|
|
71
|
+
artifactVersion: record.artifactVersion,
|
|
72
|
+
question: `Node ${record.node} refused its packet at ${record.artifactVersion} with cause ambiguous_requirement: the requirement can be read more than one way, and no widening of the packet can answer it. Decide what the requirement means, then resume with --answer ${record.node}; no reauthor round and no further attempt runs on this packet version until then.`,
|
|
73
|
+
};
|
|
74
|
+
}
|