omp-conductor 0.19.7 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +10 -1
- package/agents/to-spec.md +76 -9
- package/package.json +1 -1
- package/schema/config.schema.json +4 -0
- package/src/admission.ts +58 -14
- package/src/arm-challenge.ts +255 -85
- package/src/ask.ts +130 -615
- package/src/board.ts +7 -1
- package/src/brief-upgrade.ts +24 -0
- package/src/briefs/console.md +258 -0
- package/src/briefs/correction.md +203 -0
- package/src/briefs/orchestrator.md +167 -97
- package/src/briefs/policy.md +19 -16
- package/src/briefs/to-spec.md +76 -9
- package/src/briefs/worker.md +50 -16
- package/src/cli.ts +4 -0
- package/src/command-manifest.ts +54 -8
- package/src/commands/arm.ts +115 -49
- package/src/commands/console.ts +70 -0
- package/src/commands/context.ts +2 -0
- package/src/commands/epic.ts +132 -0
- package/src/commands/extend.ts +9 -1
- package/src/commands/intake.ts +44 -14
- package/src/commands/stats.ts +19 -4
- package/src/commands/worker.ts +9 -1
- package/src/config-schema.ts +13 -0
- package/src/config.ts +27 -0
- package/src/daemon/ack.ts +159 -0
- package/src/daemon/admission-pass.ts +135 -0
- package/src/daemon/brief.ts +461 -0
- package/src/daemon/deps.ts +539 -0
- package/src/daemon/dispatch.ts +1779 -0
- package/src/daemon/drain.ts +185 -0
- package/src/daemon/groom-pass.ts +422 -0
- package/src/daemon/http.ts +417 -0
- package/src/daemon/integrity.ts +108 -0
- package/src/daemon/panes.ts +180 -0
- package/src/daemon/review.ts +1888 -0
- package/src/daemon/runtime.ts +788 -0
- package/src/daemon/settle-pass.ts +606 -0
- package/src/daemon/supervision.ts +438 -0
- package/src/daemon/tick.ts +968 -0
- package/src/daemon/views.ts +751 -0
- package/src/daemon.ts +105 -7923
- package/src/dashboard/app.js +58 -0
- package/src/dashboard/controls.ts +22 -3
- package/src/dashboard/server.ts +4 -0
- package/src/diff-flags.ts +135 -9
- package/src/doctor.ts +2 -2
- package/src/failure-class.ts +257 -2
- package/src/fleet.ts +295 -176
- package/src/groom.ts +461 -0
- package/src/http-token.ts +142 -0
- package/src/knowledge.ts +229 -0
- package/src/mining.ts +316 -0
- package/src/orchestrator-tick.ts +689 -1670
- package/src/ready-gate.ts +267 -0
- package/src/settlement.ts +107 -11
- package/src/setup-host.ts +32 -9
- package/src/setup-wizard.ts +55 -7
- package/src/setup.ts +229 -3
- package/src/stats.ts +257 -2
- package/src/status-render.ts +169 -14
- package/src/store.ts +618 -28
- package/src/to-spec.ts +426 -44
- package/src/tracker/github.ts +50 -0
- package/src/types.ts +434 -18
- package/src/verbs/protocol.ts +28 -0
- package/src/verbs/server.ts +330 -39
- package/src/wake.ts +19 -2
- package/src/worker.ts +570 -1
package/src/dashboard/app.js
CHANGED
|
@@ -574,6 +574,62 @@ function orDash(value, format) {
|
|
|
574
574
|
return value === null || value === undefined ? "—" : format(value);
|
|
575
575
|
}
|
|
576
576
|
|
|
577
|
+
/**
|
|
578
|
+
* The attribution block (phase 4): where the spend went, as opposed to what it
|
|
579
|
+
* bought. Rendered in the empty case too — a dry queue is exactly when the
|
|
580
|
+
* daemon's own grooming sessions run, and that spend bought no outcome, which
|
|
581
|
+
* is the number an operator most wants to see.
|
|
582
|
+
*/
|
|
583
|
+
function renderAttribution(out, report) {
|
|
584
|
+
const models = report.models ?? [];
|
|
585
|
+
if (models.length > 0) {
|
|
586
|
+
const box = el("div", "chart");
|
|
587
|
+
box.appendChild(el("h4", undefined, "Per model (grouped on the model actually billed)"));
|
|
588
|
+
const rows = el("div", "summary");
|
|
589
|
+
for (const m of models) {
|
|
590
|
+
// Nothing metered at all is stated as unknown: "$0.00 metered" is true and
|
|
591
|
+
// still reads as free, which is the misreading this panel exists to
|
|
592
|
+
// prevent — the same rule the CLI's human form follows.
|
|
593
|
+
const cost =
|
|
594
|
+
m.spendUsd === 0 && m.unmeteredRuns === m.runs
|
|
595
|
+
? "cost unknown — no run on this model metered"
|
|
596
|
+
: `${usd(m.spendUsd)} metered · ${orDash(m.spendPerMerge, usd)} / merge` +
|
|
597
|
+
(m.unmeteredRuns > 0 ? ` · ${m.unmeteredRuns} unmetered` : "");
|
|
598
|
+
rows.appendChild(statLine(m.model, `${m.runs} run(s) · ${m.merges} merge(s) · ${cost}`));
|
|
599
|
+
}
|
|
600
|
+
box.appendChild(rows);
|
|
601
|
+
out.appendChild(box);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
const ran = (report.sessions ?? []).filter((s) => s.sessions > 0);
|
|
605
|
+
const unmeteredRoles = report.unmeteredRoles ?? [];
|
|
606
|
+
// Silence rather than a row of zeros: "no grooming ran" and "grooming cost
|
|
607
|
+
// nothing" are different claims, and only the first would be true.
|
|
608
|
+
if (ran.length === 0 && unmeteredRoles.length === 0) return;
|
|
609
|
+
const box = el("div", "chart");
|
|
610
|
+
box.appendChild(el("h4", undefined, "Daemon-owned sessions (not worker runs)"));
|
|
611
|
+
const rows = el("div", "summary");
|
|
612
|
+
for (const s of ran) {
|
|
613
|
+
// An unmetered session's cost is unknown, so it is stated as a count and
|
|
614
|
+
// never folded into the dollar figure as a zero.
|
|
615
|
+
const cost =
|
|
616
|
+
s.spendUsd === 0 && s.unmeteredSessions === s.sessions
|
|
617
|
+
? "cost unknown — no session metered"
|
|
618
|
+
: `${usd(s.spendUsd)} metered` +
|
|
619
|
+
(s.unmeteredSessions > 0
|
|
620
|
+
? ` · ${s.unmeteredSessions} unmetered (cost unknown, not $0.00)`
|
|
621
|
+
: "");
|
|
622
|
+
rows.appendChild(statLine(s.role, `${s.sessions} session(s) · ${s.turns} turn(s) · ${cost}`));
|
|
623
|
+
}
|
|
624
|
+
for (const role of unmeteredRoles) {
|
|
625
|
+
rows.appendChild(
|
|
626
|
+
statLine(role, "unmetered — no usage events observable, so the cost is unknown"),
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
box.appendChild(rows);
|
|
630
|
+
out.appendChild(box);
|
|
631
|
+
}
|
|
632
|
+
|
|
577
633
|
function hours(ms) {
|
|
578
634
|
return `${(ms / 3_600_000).toFixed(1)}h`;
|
|
579
635
|
}
|
|
@@ -594,6 +650,7 @@ async function refreshStats() {
|
|
|
594
650
|
),
|
|
595
651
|
);
|
|
596
652
|
out.appendChild(statLine("GitHub API calls", String(report.ghCalls)));
|
|
653
|
+
renderAttribution(out, report);
|
|
597
654
|
return;
|
|
598
655
|
}
|
|
599
656
|
|
|
@@ -657,6 +714,7 @@ async function refreshStats() {
|
|
|
657
714
|
(v) => String(v),
|
|
658
715
|
),
|
|
659
716
|
);
|
|
717
|
+
renderAttribution(out, report);
|
|
660
718
|
}
|
|
661
719
|
|
|
662
720
|
statsWindowBar.addEventListener("click", (event) => {
|
|
@@ -47,6 +47,7 @@ import { makeTracker } from "../tracker/github.ts";
|
|
|
47
47
|
import { dbPath, openStore } from "../store.ts";
|
|
48
48
|
import { loadConfig, resolveCaps } from "../config.ts";
|
|
49
49
|
import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
|
|
50
|
+
import { httpAuthHeader, missingHttpTokenMessage } from "../http-token.ts";
|
|
50
51
|
import type { ProjectConfig } from "../types.ts";
|
|
51
52
|
|
|
52
53
|
/** The source string every dashboard mutation attributes itself with. */
|
|
@@ -265,8 +266,24 @@ export function dashboardAnswerDecision(id: string, body: unknown, d: ControlDep
|
|
|
265
266
|
|
|
266
267
|
// ------------------------------------------------------- production wiring --
|
|
267
268
|
|
|
268
|
-
/**
|
|
269
|
-
|
|
269
|
+
/**
|
|
270
|
+
* Forward to the living daemon's own HTTP surface, or report there is none.
|
|
271
|
+
*
|
|
272
|
+
* Every route this forwards to is mutating, so the request carries the
|
|
273
|
+
* daemon's bearer token (Phase 4). Note this is deliberately *not* the
|
|
274
|
+
* dashboard's own token: the browser authenticated to the dashboard, and the
|
|
275
|
+
* dashboard authenticates to the daemon as a second, separately-scoped hop.
|
|
276
|
+
* A missing daemon token is its own answer rather than the 502 "no living
|
|
277
|
+
* daemon" — the daemon may be perfectly alive; what is missing is the
|
|
278
|
+
* credential file, and telling the operator "start the daemon" when it is
|
|
279
|
+
* already running would send them chasing the wrong thing.
|
|
280
|
+
*
|
|
281
|
+
* Exported so the suite can drive the *production* forwarder — the one that
|
|
282
|
+
* actually builds the request — against a stub daemon. Every other test injects
|
|
283
|
+
* `ControlDeps.proxy`, which by construction cannot prove this function sends
|
|
284
|
+
* the credential.
|
|
285
|
+
*/
|
|
286
|
+
export async function defaultProxy(
|
|
270
287
|
project: ProjectConfig,
|
|
271
288
|
path: string,
|
|
272
289
|
body: unknown,
|
|
@@ -277,13 +294,15 @@ async function defaultProxy(
|
|
|
277
294
|
// would apply an operator's action to the wrong fleet.
|
|
278
295
|
if (record === undefined) return undefined;
|
|
279
296
|
if (record.project !== undefined && record.project !== project.name) return undefined;
|
|
297
|
+
const auth = httpAuthHeader();
|
|
298
|
+
if (auth === undefined) return { status: 503, body: { error: missingHttpTokenMessage() } };
|
|
280
299
|
// Cheap liveness first, so an unreachable daemon is a 502 rather than a
|
|
281
300
|
// mutation attempt that hangs the browser.
|
|
282
301
|
if (!(await healthCheck(record.port)).ok) return undefined;
|
|
283
302
|
try {
|
|
284
303
|
const res = await fetch(`http://127.0.0.1:${record.port}${path}`, {
|
|
285
304
|
method: "PUT",
|
|
286
|
-
headers: { "content-type": "application/json" },
|
|
305
|
+
headers: { "content-type": "application/json", ...auth },
|
|
287
306
|
body: JSON.stringify(body),
|
|
288
307
|
signal: AbortSignal.timeout(10_000),
|
|
289
308
|
});
|
package/src/dashboard/server.ts
CHANGED
|
@@ -331,6 +331,10 @@ async function statsProducer(name: string, since: string): Promise<StatsReport |
|
|
|
331
331
|
window,
|
|
332
332
|
ghCalls: store.ghCallsBetween(window.sinceDay, window.untilDay),
|
|
333
333
|
runs: store.statsRuns(name, window.sinceEpochMs),
|
|
334
|
+
// Phase 4 attribution: daemon-owned session spend lives in its own table,
|
|
335
|
+
// deliberately outside `runs`, so it is a second read rather than more
|
|
336
|
+
// rows in the first one.
|
|
337
|
+
sessions: store.sessionSpendSince(name, window.sinceEpochMs),
|
|
334
338
|
});
|
|
335
339
|
} finally {
|
|
336
340
|
store.close();
|
package/src/diff-flags.ts
CHANGED
|
@@ -438,20 +438,125 @@ const ASSERTION =
|
|
|
438
438
|
/^(?:await\s+)?(?:expect|assert|assert_[a-z_]+|assertEquals?|assertTrue|assertFalse|assertThat|assertRaises|assertRaisesRegex|self\.assert[A-Za-z]*|should|chai\.|t\.(?:Error|Fatal)f?|require\.[A-Z][A-Za-z]*|Expect)\s*[.(]/;
|
|
439
439
|
|
|
440
440
|
/**
|
|
441
|
-
* A named timeout and its value
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
*
|
|
441
|
+
* A named timeout and its value, but only on the test runner's own timeout
|
|
442
|
+
* surface ({@link onRunnerSurface}) — a matching key handed to the code under
|
|
443
|
+
* test is a domain parameter, not a runner deadline (#1062). Only a *raised*
|
|
444
|
+
* one is a finding — a brand-new timeout on a new test is not a weakening — so
|
|
445
|
+
* the value is compared against the same key on the pre-image side and silence
|
|
446
|
+
* is the answer whenever the key appears on only one side.
|
|
445
447
|
*/
|
|
446
448
|
const TIMEOUT =
|
|
447
|
-
/\b(timeout|timeoutMs|timeout_ms|timeoutSeconds|deadline|maxDuration|wallClock(?:Ms)?|setTimeout|jest\.setTimeout|retries)\b\s*[:=(]\s*(\d[\d_]*)/gi;
|
|
449
|
+
/\b(timeout|timeoutMs|timeout_ms|timeoutSeconds|deadline|maxDuration|wallClock(?:Ms)?|setTimeout|jest\.setTimeout|retries)\b\s*([:=(])\s*(\d[\d_]*)/gi;
|
|
450
|
+
|
|
451
|
+
/** Identifiers that name the test runner itself when a timeout-shaped key is
|
|
452
|
+
* called on or passed to them. `t` is the test context (vitest, node:test, a
|
|
453
|
+
* Go `*testing.T` helper); `pytest` names the Python runner module. */
|
|
454
|
+
const RUNNER_BINDINGS: Record<string, true> = {
|
|
455
|
+
test: true,
|
|
456
|
+
it: true,
|
|
457
|
+
describe: true,
|
|
458
|
+
context: true,
|
|
459
|
+
suite: true,
|
|
460
|
+
bench: true,
|
|
461
|
+
jest: true,
|
|
462
|
+
t: true,
|
|
463
|
+
pytest: true,
|
|
464
|
+
};
|
|
465
|
+
|
|
466
|
+
/**
|
|
467
|
+
* Whether a {@link TIMEOUT} match sits on the runner's own timeout surface of
|
|
468
|
+
* its line, as opposed to an argument handed to the code under test.
|
|
469
|
+
*
|
|
470
|
+
* The audit sees hunks, not files, so the judgement is structural on the
|
|
471
|
+
* changed line alone, and anything it cannot vouch for stays silent: a flag is
|
|
472
|
+
* advisory, and a false positive costs the trust in every flag after it
|
|
473
|
+
* (measured: #614/#896 read a raised `timeoutMs` passed to `arm(...)` — the
|
|
474
|
+
* challenge window of the code under test — as a weakened test).
|
|
475
|
+
*
|
|
476
|
+
* Two positions qualify, both on the runner's own call:
|
|
477
|
+
*
|
|
478
|
+
* - the key is called on the runner — `jest.setTimeout(...)`,
|
|
479
|
+
* `t.timeout(...)`, `test.setTimeout(...)`, `pytest.mark.timeout(...)` —
|
|
480
|
+
* the dotted receiver before the key starts with a runner binding;
|
|
481
|
+
* - the key is a property of an object literal that is a direct argument of a
|
|
482
|
+
* runner call — the trailing per-test configuration: `test("x", fn, {
|
|
483
|
+
* timeout: 45_000 })`, `test.use({ retries: 3 })`,
|
|
484
|
+
* `describe.configure({ retries: 3 })`.
|
|
485
|
+
*
|
|
486
|
+
* Everything else is the code under test: `await arm({ timeoutMs: 60_000 })`,
|
|
487
|
+
* `const opts = { timeout: 45000 }`, `server.setTimeout(30_000)`. A shape
|
|
488
|
+
* whose decisive frame is on another line — `}, { timeout: 45000 });` after a
|
|
489
|
+
* multi-line `test(` — cannot be vouched for from the line alone and also
|
|
490
|
+
* stays silent.
|
|
491
|
+
*/
|
|
492
|
+
function onRunnerSurface(code: string, match: RegExpMatchArray): boolean {
|
|
493
|
+
const key = match[1]?.toLowerCase() ?? "";
|
|
494
|
+
if (RUNNER_BINDINGS[key.split(".")[0] ?? ""] === true) return true;
|
|
495
|
+
|
|
496
|
+
const before = code.slice(0, match.index);
|
|
497
|
+
// The dotted receiver the key is called on: `t.timeout(5000)` reads `t.`,
|
|
498
|
+
// `pytest.mark.timeout(500)` reads `pytest.mark.`. No receiver — a bare
|
|
499
|
+
// `timeout: 5000` property — falls through to the config-object rule.
|
|
500
|
+
let at = before.length - 1;
|
|
501
|
+
while (at >= 0 && /[A-Za-z0-9_$.]/.test(before[at] ?? "")) at--;
|
|
502
|
+
const receiver = before.slice(at + 1).replace(/\.$/, "").split(".")[0];
|
|
503
|
+
if (RUNNER_BINDINGS[receiver ?? ""] === true) return true;
|
|
504
|
+
|
|
505
|
+
// A property key (`key: value`) needs the object around it to be the
|
|
506
|
+
// runner's own configuration; a `key = value` assignment or a bare
|
|
507
|
+
// `key(5000)` call on the line is never that.
|
|
508
|
+
if (match[2] !== ":") return false;
|
|
509
|
+
return inRunnerConfigObject(before);
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
type RunnerFrame = { kind: "call"; base: string | undefined } | { kind: "obj" };
|
|
513
|
+
|
|
514
|
+
/** Whether the key sits in an object literal that is a direct argument of a
|
|
515
|
+
* call on a runner binding — the trailing per-test configuration:
|
|
516
|
+
* `test("x", fn, { timeout: 45_000 })`, `test.use({ retries: 3 })`,
|
|
517
|
+
* `describe.configure({ retries: 3 })`. The frames are walked over the line
|
|
518
|
+
* prefix only, with strings already stripped by {@link splitCode}, so the
|
|
519
|
+
* object's nesting inside the call — a direct argument versus a property of
|
|
520
|
+
* a nested object or of the callback's own body — decides the verdict. */
|
|
521
|
+
function inRunnerConfigObject(before: string): boolean {
|
|
522
|
+
const frames: RunnerFrame[] = [];
|
|
523
|
+
for (let at = 0; at < before.length; at++) {
|
|
524
|
+
const ch = before[at];
|
|
525
|
+
if (ch === "(") {
|
|
526
|
+
// The dotted name the call was opened on, if any: `test.use(` reads
|
|
527
|
+
// `test.use`. No name — an arrow's parameter list, or a call on a
|
|
528
|
+
// previous call's result — means the object below it is not a runner
|
|
529
|
+
// surface.
|
|
530
|
+
let end = at;
|
|
531
|
+
while (end > 0 && /[A-Za-z0-9_$.]/.test(before[end - 1] ?? "")) end--;
|
|
532
|
+
const chain = before.slice(end, at);
|
|
533
|
+
frames.push({
|
|
534
|
+
kind: "call",
|
|
535
|
+
base: chain.length > 0 && !chain.endsWith(".") ? chain : undefined,
|
|
536
|
+
});
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
if (ch === "{" || ch === "[") {
|
|
540
|
+
frames.push({ kind: "obj" });
|
|
541
|
+
continue;
|
|
542
|
+
}
|
|
543
|
+
if (ch === ")" || ch === "}" || ch === "]") frames.pop();
|
|
544
|
+
}
|
|
545
|
+
const object = frames.at(-1);
|
|
546
|
+
if (object?.kind !== "obj") return false;
|
|
547
|
+
const enclosing = frames.at(-2);
|
|
548
|
+
if (enclosing?.kind !== "call") return false;
|
|
549
|
+
const firstSegment = enclosing.base?.split(".")[0];
|
|
550
|
+
return firstSegment !== undefined && RUNNER_BINDINGS[firstSegment] === true;
|
|
551
|
+
}
|
|
448
552
|
|
|
449
553
|
function timeouts(text: string): { key: string; value: number }[] {
|
|
450
554
|
const found: { key: string; value: number }[] = [];
|
|
451
555
|
for (const match of text.matchAll(TIMEOUT)) {
|
|
452
556
|
const key = match[1]?.toLowerCase();
|
|
453
|
-
const raw = match[
|
|
557
|
+
const raw = match[3]?.replaceAll("_", "");
|
|
454
558
|
if (key === undefined || raw === undefined) continue;
|
|
559
|
+
if (!onRunnerSurface(text, match)) continue;
|
|
455
560
|
const value = Number(raw);
|
|
456
561
|
if (Number.isSafeInteger(value)) found.push({ key, value });
|
|
457
562
|
}
|
|
@@ -813,14 +918,35 @@ function claimRegion(body: string): string {
|
|
|
813
918
|
return next === null ? rest : rest.slice(0, marker[0].length + next.index);
|
|
814
919
|
}
|
|
815
920
|
|
|
921
|
+
/**
|
|
922
|
+
* An executable token: the shape a claimed command's first word must have to
|
|
923
|
+
* be a program somebody could have run. Either an ordinary command name — an
|
|
924
|
+
* ASCII letter or `_` first, then letters, digits, `_`, `.`, `+`, `-` (`bun`,
|
|
925
|
+
* `bash`, `cd`, `export`, `omp-conductor`, `python3.12`) — or a `./`-relative
|
|
926
|
+
* executable path built from those same characters plus `/`
|
|
927
|
+
* (`./scripts/deploy.sh`).
|
|
928
|
+
*
|
|
929
|
+
* Nothing else is a command, and that exclusion is the whole point (#1039):
|
|
930
|
+
* this predicate used to accept any first word that merely lacked a `/`, so
|
|
931
|
+
* the malformed-JSON fixture `{not json` described in a PR's verified prose
|
|
932
|
+
* became a "claimed" command with no transcript match, and a green PR settled
|
|
933
|
+
* with a false `claimed-proof-missing`. The grammar therefore refuses
|
|
934
|
+
* punctuation-led fragments (`{not`, `{"kind":`, `[1,`), quote-led values
|
|
935
|
+
* (`"some value"`), tokens carrying `:` or `=` (config expressions like
|
|
936
|
+
* `retry.modelFallback: true`, env assignments), digit-led data literals
|
|
937
|
+
* (timestamps like `2026-08-24 09:14Z`) and flag-led spans (`--json`). An
|
|
938
|
+
* absolute path stays excluded as it always was: it carries `/` without the
|
|
939
|
+
* leading `./`.
|
|
940
|
+
*/
|
|
941
|
+
const EXECUTABLE_TOKEN = /^(?:[A-Za-z_][A-Za-z0-9_.+-]*|\.\/[A-Za-z0-9_.+\-/]+)$/;
|
|
942
|
+
|
|
816
943
|
/** One backticked span that reads as a whole command: at least two shell words
|
|
817
|
-
* and a first word that is an
|
|
944
|
+
* and a first word that is an {@link EXECUTABLE_TOKEN}.
|
|
818
945
|
* A span that names a single file or a config key is not a command claim. */
|
|
819
946
|
function claimedCommand(span: string): boolean {
|
|
820
947
|
const words = span.split(/\s+/).filter((word) => word.length > 0);
|
|
821
948
|
if (words.length < 2) return false;
|
|
822
|
-
|
|
823
|
-
return first.startsWith("./") || !first.includes("/");
|
|
949
|
+
return EXECUTABLE_TOKEN.test(words[0] ?? "");
|
|
824
950
|
}
|
|
825
951
|
|
|
826
952
|
/**
|
package/src/doctor.ts
CHANGED
|
@@ -1345,8 +1345,8 @@ function armAckProbe(probes: Probes, p: ProjectConfig): Finding {
|
|
|
1345
1345
|
if (sighting.expiresAt !== undefined && now >= sighting.expiresAt) {
|
|
1346
1346
|
return warnFinding(
|
|
1347
1347
|
"arm-ack",
|
|
1348
|
-
`[${p.name}] an arming challenge from ${window} expired without being settled — it is inert (replies past expiry are refused)
|
|
1349
|
-
`
|
|
1348
|
+
`[${p.name}] an arming challenge from ${window} expired without being settled — it is inert (replies past expiry are refused); the fleet session notifies the operator and clears it once its notice goes out, or the next arm replaces it`,
|
|
1349
|
+
`the code is dead — no reply to it can arm. Re-run the ceremony: \`omp-conductor arm --project ${p.name}\``,
|
|
1350
1350
|
);
|
|
1351
1351
|
}
|
|
1352
1352
|
if (sighting.acknowledgedAt !== undefined) {
|
package/src/failure-class.ts
CHANGED
|
@@ -34,6 +34,14 @@ export interface ClassifyFacts {
|
|
|
34
34
|
* reachable. Lets the table tell an infrastructure outage (#177) from a
|
|
35
35
|
* deterministic test failure by the log's own words. */
|
|
36
36
|
failingLog?: string;
|
|
37
|
+
/** The PR's changed-file list, derived at the settle call site from the
|
|
38
|
+
* diff the settlement already fetches (#1059). Lets the table tell a
|
|
39
|
+
* Compose dependency-startup failure no PR could have caused from one a
|
|
40
|
+
* PR's own diff did — a PR that touches no container configuration cannot
|
|
41
|
+
* have broken `docker compose up`. Absent means the diff could not be
|
|
42
|
+
* read (or was cut short), which is "could not tell": the conditional
|
|
43
|
+
* signature below then stays silent and the attempt is charged. */
|
|
44
|
+
changedFiles?: string[];
|
|
37
45
|
}
|
|
38
46
|
|
|
39
47
|
export interface Classification {
|
|
@@ -62,6 +70,16 @@ const INFRA_CHECK_STATES: Record<string, true> = {
|
|
|
62
70
|
/** States that mean "this check has a verdict and it is good". */
|
|
63
71
|
const SUCCESS_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
|
|
64
72
|
|
|
73
|
+
/**
|
|
74
|
+
* The runner's own sentence for "the agent under this job went away": written
|
|
75
|
+
* when the runner service is stopped, auto-updates, or is torn down under a
|
|
76
|
+
* live job. Named once because two different accountings quote it — the
|
|
77
|
+
* implementation-attempt waiver below ({@link INFRA_LOG_SIGNATURES}, #177) and
|
|
78
|
+
* the review-correction waiver ({@link runnerInfraFailure}) — and a second
|
|
79
|
+
* spelling of it would waive one while charging the other.
|
|
80
|
+
*/
|
|
81
|
+
const RUNNER_SHUTDOWN_SIGNAL = "the runner has received a shutdown signal";
|
|
82
|
+
|
|
65
83
|
/**
|
|
66
84
|
* Substrings in a failed check's log that prove the failure was infrastructure,
|
|
67
85
|
* not the diff (#177). Each is a registry/docker/runner fault a worker cannot
|
|
@@ -87,7 +105,7 @@ const INFRA_LOG_SIGNATURES = [
|
|
|
87
105
|
// the runner's, and must not be waived (#637, #639).
|
|
88
106
|
"response status code does not indicate success: 429 (too many requests)",
|
|
89
107
|
"failed to resolve source metadata for",
|
|
90
|
-
|
|
108
|
+
RUNNER_SHUTDOWN_SIGNAL,
|
|
91
109
|
"could not resolve host",
|
|
92
110
|
];
|
|
93
111
|
|
|
@@ -103,6 +121,70 @@ export function infraLogSignature(log: string): string | undefined {
|
|
|
103
121
|
return INFRA_LOG_SIGNATURES.find((signature) => lower.includes(signature));
|
|
104
122
|
}
|
|
105
123
|
|
|
124
|
+
/**
|
|
125
|
+
* Runner-infrastructure wordings: a red check whose own text says the *runner*
|
|
126
|
+
* failed, not the diff (Phase 3, #1043 lane). Matched lowercased as substrings
|
|
127
|
+
* against the failing job's log — the same surface {@link infraLogSignature}
|
|
128
|
+
* reads — and each entry is the real GitHub Actions sentence, verified against
|
|
129
|
+
* the runner's own reports rather than invented:
|
|
130
|
+
*
|
|
131
|
+
* Kept as its own list rather than folded into {@link INFRA_LOG_SIGNATURES}
|
|
132
|
+
* for two reasons. The consumers differ: that list waives an *implementation
|
|
133
|
+
* attempt* for a settled run (#177), this one waives a *correction round* for
|
|
134
|
+
* a live pull request's red check. And {@link infraSignatureVersion} is a
|
|
135
|
+
* persisted cursor fingerprint — growing that list restarts the historical
|
|
136
|
+
* reconciliation, which recognising a runner-lost red has no business doing.
|
|
137
|
+
* The one overlap, {@link RUNNER_SHUTDOWN_SIGNAL}, is shared by reference so
|
|
138
|
+
* the two accountings can never disagree about that wording.
|
|
139
|
+
*/
|
|
140
|
+
const RUNNER_INFRA_SIGNATURES = [
|
|
141
|
+
// "The self-hosted runner: <name> lost communication with the server." and
|
|
142
|
+
// its hosted sibling "The hosted runner: <name> lost communication with the
|
|
143
|
+
// server." — the runner process was killed, starved or cut off mid-job
|
|
144
|
+
// (actions/runner#3539, community#84877, community#173431). The name varies,
|
|
145
|
+
// so only the invariant tail is matched.
|
|
146
|
+
"lost communication with the server",
|
|
147
|
+
// The runner service was stopped or auto-updated under the job: an
|
|
148
|
+
// infrastructure cancellation, not a verdict on the diff.
|
|
149
|
+
RUNNER_SHUTDOWN_SIGNAL,
|
|
150
|
+
// "The job was not acquired by Runner of type hosted even after multiple
|
|
151
|
+
// attempts" — the job never reached a runner at all, so nothing in the diff
|
|
152
|
+
// was ever executed (community#186216, community#165287).
|
|
153
|
+
"was not acquired by runner",
|
|
154
|
+
// "The hosted runner encountered an error while running your job. (Error
|
|
155
|
+
// Type: Failure)." — GitHub's own statement that its runner broke
|
|
156
|
+
// (community#126539). The parenthesised error type varies; the sentence does
|
|
157
|
+
// not.
|
|
158
|
+
"the hosted runner encountered an error while running your job",
|
|
159
|
+
];
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* The runner-infrastructure wording a red CI check's text carries, or
|
|
163
|
+
* `undefined` when it carries none.
|
|
164
|
+
*
|
|
165
|
+
* Exists so the review-correction accounting can decline to charge a
|
|
166
|
+
* correction round for a red that the worker's diff did not cause: a lost
|
|
167
|
+
* runner, a runner shut down under the job, or a job no runner ever picked up.
|
|
168
|
+
* The return value is the matched wording, so whatever waives a round can say
|
|
169
|
+
* *which* sentence waived it instead of asserting "infra" unexplained.
|
|
170
|
+
*
|
|
171
|
+
* Fails closed in both directions, and deliberately narrower than it could be:
|
|
172
|
+
*
|
|
173
|
+
* - Unclassifiable, empty or unread text is `undefined` — an unexplained red
|
|
174
|
+
* still counts, and a log-fetch failure hands this function nothing to match,
|
|
175
|
+
* so it never waives.
|
|
176
|
+
* - "The operation was canceled." is *not* here: it is what a failed `needs:`
|
|
177
|
+
* dependency and an operator's own cancel both print, so matching it would
|
|
178
|
+
* waive rounds for ordinary red.
|
|
179
|
+
* - "…has exceeded the maximum execution time of N minutes" is *not* here
|
|
180
|
+
* either: a job that ran until the ceiling is usually a hanging test or an
|
|
181
|
+
* infinite loop, which is exactly the diff's problem to fix.
|
|
182
|
+
*/
|
|
183
|
+
export function runnerInfraFailure(log: string): string | undefined {
|
|
184
|
+
const lower = log.toLowerCase();
|
|
185
|
+
return RUNNER_INFRA_SIGNATURES.find((signature) => lower.includes(signature));
|
|
186
|
+
}
|
|
187
|
+
|
|
106
188
|
/**
|
|
107
189
|
* The cap kills that produced nothing: the worker reached a ceiling with no PR,
|
|
108
190
|
* no observed head and no salvage commit.
|
|
@@ -134,6 +216,62 @@ export function infraSignatureVersion(): string {
|
|
|
134
216
|
return INFRA_LOG_SIGNATURES.join("|");
|
|
135
217
|
}
|
|
136
218
|
|
|
219
|
+
/**
|
|
220
|
+
* Compose's own sentence for a dependency that failed its healthcheck during
|
|
221
|
+
* `docker compose up`: "dependency failed to start: container
|
|
222
|
+
* chad-postgres-1 is unhealthy" (#1059). Matched with its trailing context,
|
|
223
|
+
* never a bare `is unhealthy` — that phrase is common enough in application
|
|
224
|
+
* logs to be unsafe on its own. Shared with the settle call site so the diff
|
|
225
|
+
* fetch and the classifier can never disagree about which log sentence needs
|
|
226
|
+
* the changed-file list.
|
|
227
|
+
*/
|
|
228
|
+
export const COMPOSE_DEPENDENCY_STARTUP_SIGNATURE =
|
|
229
|
+
"dependency failed to start: container ";
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* A changed path that configures the containers Compose jobs run, or the
|
|
233
|
+
* workflow that invokes them: a compose file — `docker-compose*.ya?ml`,
|
|
234
|
+
* Compose v2's default `compose.ya?ml`, or any `*.ya?ml` directly inside a
|
|
235
|
+
* `compose/` directory (#1071) — anywhere in the tree, a `Dockerfile*`, or
|
|
236
|
+
* anything under `.github/workflows/`. A PR touching any of these can have
|
|
237
|
+
* broken `docker compose up` itself — that is what turns the signature below
|
|
238
|
+
* from infrastructure into a charged, deterministic failure. Matched
|
|
239
|
+
* case-insensitively: the price of missing a container file is waiving a
|
|
240
|
+
* genuine attempt, while a false positive (a path merely *named* like one)
|
|
241
|
+
* only charges one.
|
|
242
|
+
*/
|
|
243
|
+
const CONTAINER_CONFIG_PATH =
|
|
244
|
+
/(?:^|\/)(?:docker-compose[^/]*\.ya?ml|compose\.ya?ml|compose\/[^/]+\.ya?ml|Dockerfile[^/]*)$|(?:^|\/)\.github\/workflows\//i;
|
|
245
|
+
|
|
246
|
+
/**
|
|
247
|
+
* Evidence that a failed check's log names a Compose dependency-startup
|
|
248
|
+
* failure, or `undefined` when the log does not name one or the PR's own diff
|
|
249
|
+
* could have caused it (#1059).
|
|
250
|
+
*
|
|
251
|
+
* Deliberately conditional on the PR's changed-file list, and deliberately
|
|
252
|
+
* left out of {@link INFRA_LOG_SIGNATURES}: a job that runs `docker compose`
|
|
253
|
+
* can be broken by the diff itself — a bad `docker-compose.yml` in a PR
|
|
254
|
+
* produces this exact sentence — so the blanket signature would waive the
|
|
255
|
+
* implementation attempt for a real defect. Only when the diff touches no
|
|
256
|
+
* compose file, no Dockerfile and no workflow is the fault one the PR cannot
|
|
257
|
+
* have introduced, and the recovery (`rerun-checks`) is bounded by the strike
|
|
258
|
+
* cap either way.
|
|
259
|
+
*
|
|
260
|
+
* Fails closed: `undefined` changedFiles is "could not read the diff", never
|
|
261
|
+
* "clean", and stays silent — an unknown or truncated diff must not waive an
|
|
262
|
+
* attempt.
|
|
263
|
+
*/
|
|
264
|
+
export function composeDependencyStartup(
|
|
265
|
+
log: string,
|
|
266
|
+
changedFiles: string[] | undefined,
|
|
267
|
+
): string | undefined {
|
|
268
|
+
if (changedFiles === undefined) return undefined;
|
|
269
|
+
const lower = log.toLowerCase();
|
|
270
|
+
if (!lower.includes(COMPOSE_DEPENDENCY_STARTUP_SIGNATURE)) return undefined;
|
|
271
|
+
if (changedFiles.some((path) => CONTAINER_CONFIG_PATH.test(path))) return undefined;
|
|
272
|
+
return COMPOSE_DEPENDENCY_STARTUP_SIGNATURE.trim();
|
|
273
|
+
}
|
|
274
|
+
|
|
137
275
|
/** Lowercased check state — `gh pr checks` has emitted both `failure` and
|
|
138
276
|
* `FAILURE` across versions, and the classifier's callers must agree on one
|
|
139
277
|
* spelling so log selection and classification see the same set of checks. */
|
|
@@ -527,6 +665,69 @@ export function noVerdictExit(run: RunRecord): string | undefined {
|
|
|
527
665
|
return `the session ended without delivering a settlement verdict; last words: "${lastWords.slice(0, 80)}"`;
|
|
528
666
|
}
|
|
529
667
|
|
|
668
|
+
/** A reliable PR reference inside a blocker's prose: a `PR #N`/
|
|
669
|
+
* `pull request #N` mention or a GitHub `…/pull/N` URL. Named once so the
|
|
670
|
+
* evidence's "the next act is an observation of this PR" claim and the code
|
|
671
|
+
* that finds the PR stay one pattern (#1068). */
|
|
672
|
+
const PR_REFERENCE_PATTERN =
|
|
673
|
+
/(?:PR|pull request)\s+#?\s*\d+|github\.com\/[^\s/]+\/[^\s/]+\/pull\/\d+/i;
|
|
674
|
+
|
|
675
|
+
/**
|
|
676
|
+
* The `blockers:` list of a blocked settlement's stored report, in order
|
|
677
|
+
* (#1068). The worker's structured yield renders one item per blocker as
|
|
678
|
+
* ` - <item>` under a `blockers:` heading, and prose settlements that use the
|
|
679
|
+
* same heading parse the same way. `undefined` when the report carries no
|
|
680
|
+
* such section — a report that names no blockers and a missing report read
|
|
681
|
+
* identically, which is honest: the worker did not name a condition to
|
|
682
|
+
* observe.
|
|
683
|
+
*/
|
|
684
|
+
function blockedSettlementBlockers(report: string | undefined): string[] | undefined {
|
|
685
|
+
if (report === undefined) return undefined;
|
|
686
|
+
const blockers: string[] = [];
|
|
687
|
+
let inBlockers = false;
|
|
688
|
+
for (const line of report.split("\n")) {
|
|
689
|
+
if (inBlockers) {
|
|
690
|
+
const item = /^ {2,}-\s+(.*)$/.exec(line);
|
|
691
|
+
if (item === null) break;
|
|
692
|
+
blockers.push(item[1]!.trim());
|
|
693
|
+
continue;
|
|
694
|
+
}
|
|
695
|
+
if (/^blockers:\s*$/i.test(line)) inBlockers = true;
|
|
696
|
+
}
|
|
697
|
+
return blockers.length === 0 ? undefined : blockers;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/**
|
|
701
|
+
* An explicit question in the blocked row's own words, or `undefined` when the
|
|
702
|
+
* row names none (#1068). The worker's question travels in the pre-existing
|
|
703
|
+
* `lastError` slot, or as a `blockers:` item that ends interrogative; a
|
|
704
|
+
* blocker that names a condition to observe ("PR #1067 checks still
|
|
705
|
+
* pending…") is not a question. Quoted verbatim into the escalation evidence,
|
|
706
|
+
* because the orchestrator answers the worker's own words rather than a
|
|
707
|
+
* paraphrase.
|
|
708
|
+
*/
|
|
709
|
+
function blockedQuestion(run: RunRecord): string | undefined {
|
|
710
|
+
if (run.lastError !== undefined && run.lastError.trim() !== "") return run.lastError;
|
|
711
|
+
return blockedSettlementBlockers(run.report)?.find((b) => /[??]\s*$/.test(b));
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/**
|
|
715
|
+
* The report's first substantive line: skips the `status:`/`pr:`/`branch:`/
|
|
716
|
+
* `head:` headers and blank lines a structured settlement puts on top, and
|
|
717
|
+
* bounds the quote so a prose dump cannot balloon an evidence line. `undefined`
|
|
718
|
+
* when the report is empty or carries nothing but scaffold.
|
|
719
|
+
*/
|
|
720
|
+
function firstReportContentLine(report: string | undefined): string | undefined {
|
|
721
|
+
if (report === undefined) return undefined;
|
|
722
|
+
for (const raw of report.split("\n")) {
|
|
723
|
+
const line = raw.trim();
|
|
724
|
+
if (line === "") continue;
|
|
725
|
+
if (/^(?:status|pr|branch|head):\s*\S/.test(line)) continue;
|
|
726
|
+
return line.length > 200 ? `${line.slice(0, 200)}…` : line;
|
|
727
|
+
}
|
|
728
|
+
return undefined;
|
|
729
|
+
}
|
|
730
|
+
|
|
530
731
|
export function classifyRun(
|
|
531
732
|
run: RunRecord,
|
|
532
733
|
facts: ClassifyFacts,
|
|
@@ -670,10 +871,44 @@ export function classifyRun(
|
|
|
670
871
|
}
|
|
671
872
|
|
|
672
873
|
if (run.state === "blocked") {
|
|
874
|
+
// `blocked` is overloaded (#1068). A worker that finishes its work and
|
|
875
|
+
// stops because it is waiting on an observable condition — almost always
|
|
876
|
+
// its PR's checks — records that condition as `blockers:` in its
|
|
877
|
+
// settlement report; a worker that stops to ask records an actual
|
|
878
|
+
// question. Only the second shape is a human escalation: the first names
|
|
879
|
+
// the next act (observe, then the sweep settles the row the way it
|
|
880
|
+
// settles `settlement-stuck` when the PR resolves), so escalating it
|
|
881
|
+
// wakes the orchestrator for a question that does not exist and the act
|
|
882
|
+
// that *is* required appears nowhere. The evidence never claims the
|
|
883
|
+
// worker "left no report" — it quotes the question or the first blocker,
|
|
884
|
+
// and when the report genuinely is empty it says which fields are.
|
|
885
|
+
const question = blockedQuestion(run);
|
|
886
|
+
if (question !== undefined) {
|
|
887
|
+
return { cls: "question", recovery: "escalate", evidence: question };
|
|
888
|
+
}
|
|
889
|
+
const blockers = blockedSettlementBlockers(run.report);
|
|
890
|
+
if (blockers !== undefined) {
|
|
891
|
+
// Prefer the blocker that names a PR: the reader's next act is an
|
|
892
|
+
// observation of that PR and a merge, and the evidence must say so.
|
|
893
|
+
const named = blockers.find((b) => PR_REFERENCE_PATTERN.test(b)) ?? blockers[0]!;
|
|
894
|
+
return {
|
|
895
|
+
cls: "awaiting-observation",
|
|
896
|
+
recovery: "observe",
|
|
897
|
+
evidence: `waiting on ${named}`,
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
const reportEmpty = run.report === undefined || run.report.trim() === "";
|
|
901
|
+
const firstContent = firstReportContentLine(run.report);
|
|
673
902
|
return {
|
|
674
903
|
cls: "question",
|
|
675
904
|
recovery: "escalate",
|
|
676
|
-
evidence:
|
|
905
|
+
evidence: reportEmpty
|
|
906
|
+
? `the worker stopped without a run report (run.report is empty${
|
|
907
|
+
run.lastError === undefined || run.lastError.trim() === "" ? "; lastError is empty too" : ""
|
|
908
|
+
})`
|
|
909
|
+
: firstContent === undefined
|
|
910
|
+
? "the worker stopped without asking a question or naming a blocker (run.report is non-empty but has no readable content)"
|
|
911
|
+
: `the worker stopped without asking a question or naming a blocker — ${firstContent}`,
|
|
677
912
|
};
|
|
678
913
|
}
|
|
679
914
|
|
|
@@ -780,6 +1015,26 @@ export function classifyRun(
|
|
|
780
1015
|
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`,
|
|
781
1016
|
};
|
|
782
1017
|
}
|
|
1018
|
+
// A Compose dependency-startup failure is infra only when the PR's
|
|
1019
|
+
// diff could not have caused it (#1059): a job that runs `docker
|
|
1020
|
+
// compose` is broken by a PR that breaks its own compose file, and
|
|
1021
|
+
// that attempt is genuinely spent. The settlement hands the
|
|
1022
|
+
// classifier the PR's changed-file list, so "did this diff touch any
|
|
1023
|
+
// container configuration" is a fact, not a guess from the log alone.
|
|
1024
|
+
// Absent list (or a log without the sentence) stays silent and the
|
|
1025
|
+
// row reads `ci-deterministic`.
|
|
1026
|
+
const compose = composeDependencyStartup(facts.failingLog, facts.changedFiles);
|
|
1027
|
+
if (compose !== undefined) {
|
|
1028
|
+
const check = checks.find((c) => normalise(c.state) === "failure");
|
|
1029
|
+
return {
|
|
1030
|
+
cls: "ci-infra",
|
|
1031
|
+
recovery: "rerun-checks",
|
|
1032
|
+
// Both halves, so the evidence says *which* sentence matched and
|
|
1033
|
+
// *why* it was not the diff's fault — the changed-file list
|
|
1034
|
+
// showed no container configuration.
|
|
1035
|
+
evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${compose}" — the PR diff touches no container configuration, so the PR cannot have failed this check`,
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
783
1038
|
}
|
|
784
1039
|
const failing = unresolved.filter((c) => normalise(c.state) === "failure");
|
|
785
1040
|
if (failing.length > 0) {
|