@feigi/fleet-ctl 0.0.0 → 3.16.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/{fleet-implementer.agent.md → fleet-implementer-slow-high.agent.md} +23 -2
- package/agents/fleet-implementer-slow-medium.agent.md +231 -0
- package/agents/fleet-implementer-smol-high.agent.md +231 -0
- package/agents/{fleet-implementer-alt.agent.md → fleet-implementer-task-high.agent.md} +23 -2
- package/agents/fleet-implementer-task-max.agent.md +231 -0
- package/agents/fleet-review-comments.agent.md +4 -2
- package/agents/fleet-review-correctness.agent.md +2 -1
- package/agents/fleet-review-runner.agent.md +14 -10
- package/agents/fleet-review-silent-failure.agent.md +4 -3
- package/agents/fleet-review-simplify.agent.md +2 -1
- package/agents/fleet-review-snapshot.agent.md +2 -1
- package/agents/fleet-review-test-run.agent.md +29 -0
- package/agents/fleet-review-tests.agent.md +10 -5
- package/agents/fleet-review-types.agent.md +4 -2
- package/agents/fleet-review-verifier.agent.md +4 -3
- package/commands/review-and-fix.md +7 -6
- package/commands/run-merge-bot.md +14 -4
- package/package.json +4 -2
- package/scripts/agent-test.test.mjs +36 -32
- package/scripts/ambient-git-vars-mjs-prose.test.mjs +23 -13
- package/scripts/apply-ruleset.test.mjs +4 -3
- package/scripts/arg-header-probes-prose.test.mjs +1 -0
- package/scripts/arg.test.mjs +18 -18
- package/scripts/board-cli.test.mjs +18 -21
- package/scripts/board-identity.test.mjs +9 -9
- package/scripts/board-liveness-banner.test.mjs +6 -6
- package/scripts/board-prev-shape.test.mjs +12 -13
- package/scripts/board-tryparse-null.test.mjs +7 -7
- package/scripts/board.mjs +4 -7
- package/scripts/board.test.mjs +57 -61
- package/scripts/candidates.test.mjs +228 -17
- package/scripts/ci-completes-premise-prose.test.mjs +1 -1
- package/scripts/ci-state-prose.test.mjs +2 -2
- package/scripts/ci-state.mjs +389 -187
- package/scripts/ci-state.test.mjs +458 -165
- package/scripts/claim-lifecycle.test.mjs +2 -1
- package/scripts/claim-ticket.test.mjs +35 -34
- package/scripts/closing-keyword-prose.test.mjs +1 -1
- package/scripts/cockpit-tick-nonblocking.test.mjs +8 -8
- package/scripts/compute-board.mjs +13 -3
- package/scripts/compute-board.test.mjs +64 -1
- package/scripts/compute-spend.mjs +46 -8
- package/scripts/compute-spend.test.mjs +64 -10
- package/scripts/cross-repo-citation-prose.test.mjs +18 -19
- package/scripts/delete-merged-branch.sh +218 -0
- package/scripts/delete-merged-branch.test.mjs +576 -0
- package/scripts/derive-testcmd.sh +18 -7
- package/scripts/derive-testcmd.test.mjs +9 -8
- package/scripts/diff-stats.mjs +2 -2
- package/scripts/diff-stats.test.mjs +7 -11
- package/scripts/dispatch-block-golden-prose.test.mjs +111 -6
- package/scripts/dispatch-block-pins-prose.test.mjs +5 -5
- package/scripts/dispositions-check.mjs +507 -0
- package/scripts/dispositions-check.test.mjs +843 -0
- package/scripts/dispositions-gate-prose.test.mjs +114 -0
- package/scripts/drop-merged-label.test.mjs +3 -4
- package/scripts/exec-stub.mjs +93 -0
- package/scripts/exec-stub.test.mjs +69 -0
- package/scripts/filing-bar-seat-prose.test.mjs +1 -1
- package/scripts/finisher-test-verdict-prose.test.mjs +102 -0
- package/scripts/fix-applier-correction-rules-prose.test.mjs +11 -10
- package/scripts/fleet-bootstrap +5 -20
- package/scripts/fleet-bootstrap.test.mjs +75 -0
- package/scripts/fleet-heartbeat.mjs +2 -2
- package/scripts/fleet-provenance +15 -24
- package/scripts/fleet-provenance.test.mjs +86 -12
- package/scripts/fleet-run.test.mjs +5 -5
- package/scripts/fleet-state.mjs +33 -11
- package/scripts/fleet-state.test.mjs +90 -3
- package/scripts/fleet-tick.mjs +346 -70
- package/scripts/fleet-tick.test.mjs +566 -16
- package/scripts/force-with-lease-reason-prose.test.mjs +1 -1
- package/scripts/frontmatter-check.mjs +3 -2
- package/scripts/frontmatter-check.test.mjs +6 -6
- package/scripts/immutable-body-claim-prose.test.mjs +53 -262
- package/scripts/implementer-model-tier.test.mjs +98 -506
- package/scripts/inflight.test.mjs +18 -127
- package/scripts/is-cli.mjs +26 -0
- package/scripts/is-cli.test.mjs +99 -0
- package/scripts/json.test.mjs +3 -3
- package/scripts/ledger-dispatch.test.mjs +368 -7
- package/scripts/ledger-grammar.mjs +85 -1
- package/scripts/ledger-grammar.test.mjs +86 -1
- package/scripts/ledger-read-require-file.test.mjs +7 -7
- package/scripts/ledger-rotate.test.mjs +142 -0
- package/scripts/ledger.mjs +181 -17
- package/scripts/ledger.test.mjs +12 -130
- package/scripts/main-checkout.mjs +312 -0
- package/scripts/main-checkout.test.mjs +332 -0
- package/scripts/member-outcomes-header.test.mjs +3 -3
- package/scripts/member-outcomes.mjs +4 -5
- package/scripts/member-outcomes.test.mjs +92 -18
- package/scripts/member-prompt-prose.test.mjs +17 -13
- package/scripts/member-record.mjs +8 -3
- package/scripts/member-record.test.mjs +5 -5
- package/scripts/member-write-guard.mjs +5 -5
- package/scripts/member-write-guard.test.mjs +6 -3
- package/scripts/no-undo-audit.sh +24 -21
- package/scripts/no-undo-audit.test.mjs +9 -13
- package/scripts/pointer-target-prose.test.mjs +1 -1
- package/scripts/pr-overlap.test.mjs +6 -9
- package/scripts/prompt-renderer.mjs +10 -9
- package/scripts/prose-pin.mjs +10 -11
- package/scripts/prove-merge.test.mjs +3 -3
- package/scripts/quiet-payload-prose.test.mjs +1 -1
- package/scripts/reap.test.mjs +3 -4
- package/scripts/release-label-split-prose.test.mjs +17 -2
- package/scripts/release-ticket.test.mjs +35 -44
- package/scripts/review-core-cwd-isolation.test.mjs +7 -6
- package/scripts/review-core-reads.test.mjs +1 -1
- package/scripts/review-core-scratch-refusal.test.mjs +64 -0
- package/scripts/review-core-shared-test-run.test.mjs +238 -0
- package/scripts/review-core-snapshot-path.test.mjs +59 -12
- package/scripts/review-core-specialist-scratch.test.mjs +12 -10
- package/scripts/review-core-testcmd.test.mjs +76 -45
- package/scripts/review-core-unrun.test.mjs +154 -168
- package/scripts/review-core.mjs +250 -56
- package/scripts/review-eval.mjs +11 -4
- package/scripts/review-host-fixture.mjs +12 -3
- package/scripts/review-in-run-retry.test.mjs +27 -3
- package/scripts/review-runner-spawns.test.mjs +5 -3
- package/scripts/review-runner.test.mjs +14 -3
- package/scripts/review-tier-audit.test.mjs +1 -1
- package/scripts/run-merge-bot-prose.test.mjs +29 -6
- package/scripts/scratch-partition-invariant-prose.test.mjs +1 -1
- package/scripts/section-between-guard-prose.test.mjs +4 -3
- package/scripts/select-dimensions.test.mjs +1 -1
- package/scripts/shell-traps-prose.test.mjs +6 -9
- package/scripts/shortlist.mjs +4 -1
- package/scripts/shortlist.test.mjs +30 -16
- package/scripts/snapshot-repo.test.mjs +177 -4
- package/scripts/stash-prohibition-prose.test.mjs +1 -1
- package/scripts/temp-dir.mjs +47 -0
- package/scripts/tier-check.mjs +95 -28
- package/scripts/tier-check.test.mjs +131 -29
- package/scripts/tier-outcomes-header.test.mjs +11 -54
- package/scripts/tier-outcomes.mjs +34 -12
- package/scripts/tier-outcomes.test.mjs +157 -4
- package/scripts/tier-roles.mjs +118 -29
- package/scripts/tier-roles.test.mjs +226 -61
- package/scripts/tracker-block-copy-prose.test.mjs +3 -3
- package/scripts/verify-sha.test.mjs +4 -5
- package/scripts/watcher-degraded-block.test.mjs +5 -8
- package/scripts/within-run-pair-prose.test.mjs +43 -75
- package/scripts/worktree-audit.test.mjs +2 -1
- package/skills/next-ticket/SKILL.md +2 -2
- package/skills/run-team/SKILL.md +433 -303
- package/skills/run-team/references/ci-and-staleness.md +7 -6
- package/skills/run-team/references/correction-tickets.md +3 -3
- package/skills/run-team/references/isolation.md +1 -1
- package/skills/sizing-a-ticket/SKILL.md +8 -8
- package/scripts/tier-guard-gate-prose.test.mjs +0 -216
package/scripts/review-core.mjs
CHANGED
|
@@ -2,10 +2,10 @@
|
|
|
2
2
|
// on #1296). Read that ruling before touching this file.
|
|
3
3
|
//
|
|
4
4
|
// This file holds every host-independent declaration PR review needs: the
|
|
5
|
-
//
|
|
5
|
+
// JSON schemas, `usableDiff`, `readRules`, `resolveTestCmd`,
|
|
6
6
|
// `decodeArgs`, `selectDimensions`, `resolveDimensions`, `snapshotMissing`,
|
|
7
|
-
// `unrunReason`/`unrunEntries`/`unrunCrashed`, `
|
|
8
|
-
// and `runReview(host, args)` — the orchestration function itself.
|
|
7
|
+
// `unrunReason`/`unrunEntries`/`unrunCrashed`, `sharedRunNote`, `verdictFor`,
|
|
8
|
+
// `resumeFor`, and `runReview(host, args)` — the orchestration function itself.
|
|
9
9
|
//
|
|
10
10
|
// review-eval.mjs imports this file directly, a RELATIVE specifier: both
|
|
11
11
|
// ship together under the same Install root, so the Resolver is only needed
|
|
@@ -22,9 +22,10 @@
|
|
|
22
22
|
// `null`, which every guard here — `if (snap) {...}`, `unrunCrashed`,
|
|
23
23
|
// `verdictFor` — is written against).
|
|
24
24
|
//
|
|
25
|
-
//
|
|
25
|
+
// Two imports. arg.mjs's isDigits(), added by #878: this file is an ordinary
|
|
26
26
|
// module, so it consumes the repo's digits rule directly rather than
|
|
27
|
-
// declaring its own copy.
|
|
27
|
+
// declaring its own copy. node:path's `posix`, added by #2323, to normalise a
|
|
28
|
+
// scratch path before `runnerScratchRefusal` reads its last component.
|
|
28
29
|
//
|
|
29
30
|
// `.mjs`, not `.js` (#1763): this file was `review-core.js` until then, and
|
|
30
31
|
// nothing that ships declares a `type`, so Node below 20.19.0/22.7.0 —
|
|
@@ -32,6 +33,7 @@
|
|
|
32
33
|
// review-eval.mjs's import of it failed. node-floor-sweep.test.mjs keeps a
|
|
33
34
|
// shipped `.mjs` from importing a relative module under any other extension.
|
|
34
35
|
|
|
36
|
+
import { posix } from "node:path";
|
|
35
37
|
import { isDigits } from "./arg.mjs";
|
|
36
38
|
|
|
37
39
|
// --- Schemas ----------------------------------------------------------
|
|
@@ -51,10 +53,10 @@ export const FINDINGS_SCHEMA = {
|
|
|
51
53
|
additionalProperties: false,
|
|
52
54
|
required: ["command", "tests"],
|
|
53
55
|
description:
|
|
54
|
-
"The
|
|
56
|
+
"The review's ONE shared test run, copied from your prompt — you do not run the full suite yourself (#2315). Report it even when it failed or produced nothing.",
|
|
55
57
|
properties: {
|
|
56
|
-
command: { type: "string", description: "The command as
|
|
57
|
-
tests: { type: "integer", description: "
|
|
58
|
+
command: { type: "string", description: "The shared run's command, verbatim as your prompt states it." },
|
|
59
|
+
tests: { type: "integer", description: "The shared run's test count as your prompt states it; 0 when it states none." },
|
|
58
60
|
pass: { type: "integer" },
|
|
59
61
|
fail: { type: "integer" },
|
|
60
62
|
},
|
|
@@ -122,6 +124,27 @@ export const SNAPSHOT_SCHEMA = {
|
|
|
122
124
|
},
|
|
123
125
|
};
|
|
124
126
|
|
|
127
|
+
// #2315. What the review's one shared test run reports back. `command` and the
|
|
128
|
+
// log path are NOT asked for: the caller handed both out, so it already knows
|
|
129
|
+
// them, and an agent's echo of either could only disagree. Every count is
|
|
130
|
+
// optional and stays absent when the log does not state it — an absent
|
|
131
|
+
// `tests` is how `unrunReason` learns the run produced no counts at all
|
|
132
|
+
// (crash, deadline, no summary), which a typed-in 0 would misreport as a run
|
|
133
|
+
// that collected nothing.
|
|
134
|
+
export const TEST_RUN_SCHEMA = {
|
|
135
|
+
type: "object",
|
|
136
|
+
additionalProperties: false,
|
|
137
|
+
properties: {
|
|
138
|
+
exitCode: { type: "integer", description: "The TEST_RUN_EXIT value the command block printed." },
|
|
139
|
+
tests: { type: "integer" },
|
|
140
|
+
pass: { type: "integer" },
|
|
141
|
+
fail: { type: "integer" },
|
|
142
|
+
cancelled: { type: "integer" },
|
|
143
|
+
skipped: { type: "integer" },
|
|
144
|
+
error: { type: "string", description: "Only when no summary was printed: what happened instead." },
|
|
145
|
+
},
|
|
146
|
+
};
|
|
147
|
+
|
|
125
148
|
// --- Dimension catalog --------------------------------------------------
|
|
126
149
|
// #1349, per #1303's gap 3: no dimension carries a `model`/`effort` field —
|
|
127
150
|
// dispatch tier lives ONLY in each fleet-owned agent definition's own
|
|
@@ -166,16 +189,19 @@ export const DEFAULT_DIMENSIONS = [
|
|
|
166
189
|
|
|
167
190
|
// #2102: the omp `fleet-review-runner` agent definition's `spawns:`
|
|
168
191
|
// frontmatter must allow exactly the agent types dispatched below (the
|
|
169
|
-
// snapshot agent,
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
//
|
|
192
|
+
// snapshot agent, the shared test run's agent (#2315), every
|
|
193
|
+
// DEFAULT_DIMENSIONS entry, and the verifier) — omp denies every spawn by
|
|
194
|
+
// default, and the runner declared no `spawns` at all until this fix.
|
|
195
|
+
// Exporting the three literal-only types (snapshot/test-run/verifier have no
|
|
196
|
+
// DEFAULT_DIMENSIONS entry of their own) plus the derived full set lets
|
|
197
|
+
// review-runner-spawns.test.mjs pin the frontmatter against what this file
|
|
198
|
+
// actually dispatches, instead of a hand-copied list that can drift.
|
|
175
199
|
export const SNAPSHOT_AGENT_TYPE = "fleet-review-snapshot";
|
|
200
|
+
export const TEST_RUN_AGENT_TYPE = "fleet-review-test-run";
|
|
176
201
|
export const VERIFIER_AGENT_TYPE = "fleet-review-verifier";
|
|
177
202
|
export const SPAWNED_AGENT_TYPES = [
|
|
178
203
|
SNAPSHOT_AGENT_TYPE,
|
|
204
|
+
TEST_RUN_AGENT_TYPE,
|
|
179
205
|
...DEFAULT_DIMENSIONS.map((d) => d.agentType),
|
|
180
206
|
VERIFIER_AGENT_TYPE,
|
|
181
207
|
];
|
|
@@ -210,7 +236,7 @@ export function verifiersFor(A) {
|
|
|
210
236
|
export function usableDiff(snap) {
|
|
211
237
|
if (!snap.diffPath) return null;
|
|
212
238
|
if (!snap.diffLines) return null;
|
|
213
|
-
// No head check here:
|
|
239
|
+
// No head check here: the snap must already be admitted by `snapshotMissing`, which owns it.
|
|
214
240
|
return `${snap.runRoot}/pr.diff`;
|
|
215
241
|
}
|
|
216
242
|
|
|
@@ -419,28 +445,119 @@ validation of the tree: its counts are snapshot-measured, and a failure in it
|
|
|
419
445
|
cannot be told apart from a regression.`;
|
|
420
446
|
}
|
|
421
447
|
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
448
|
+
// #2315. Every run-quality verdict below reads the review's ONE shared test
|
|
449
|
+
// run — `{command, logPath, exitCode?, tests?, pass?, fail?, cancelled?,
|
|
450
|
+
// skipped?, error?}`, `command`/`logPath` the caller's own and the rest the
|
|
451
|
+
// test-run agent's report — never a specialist's own run: no specialist runs
|
|
452
|
+
// the full suite any more, so a verdict on one would judge a copy of this.
|
|
453
|
+
// `run` null is a caller holding no run at all — runReview always hands one
|
|
454
|
+
// over, a null or thrown test-run dispatch arriving as `{error}` — and `tests`
|
|
455
|
+
// absent is a run that returned without a count. Both are "no counts", and
|
|
456
|
+
// neither is ever a pass. Counts alone are not a pass either: the exit status
|
|
457
|
+
// is the command's own verdict, so a run that reports none, or exits non-zero
|
|
458
|
+
// with nothing failing or cancelled to account for it, is unusable — a
|
|
459
|
+
// coverage gate, a crash after the summary, or an agent that dropped the
|
|
460
|
+
// TEST_RUN_EXIT line would otherwise read as green.
|
|
461
|
+
function testRunReason(run) {
|
|
462
|
+
if (!run) return "the review's shared test run returned nothing — no counts exist, so no dimension's tests ran";
|
|
426
463
|
const cmd = run.command || "the test command";
|
|
464
|
+
if (typeof run.tests !== "number")
|
|
465
|
+
return `\`${cmd}\` produced no counts — the shared test run crashed, hit its deadline, or printed no summary${run.error ? ` (${run.error})` : ""} — a failed run, not a pass`;
|
|
427
466
|
if (!run.tests) return `\`${cmd}\` produced 0 tests — a failed run, not a pass`;
|
|
428
467
|
if (run.pass === 0 && !run.fail) return `\`${cmd}\` passed nothing and failed nothing — every test skipped, not a pass`;
|
|
429
468
|
const executed = run.pass + (run.fail ?? 0);
|
|
430
469
|
if (typeof run.pass === "number" && executed * 2 < run.tests)
|
|
431
470
|
return `\`${cmd}\` passed ${run.pass} and failed ${run.fail ?? 0} of the ${run.tests} tests it collected — most of what it collected never ran`;
|
|
432
|
-
if (run.
|
|
433
|
-
return `\`${cmd}\` reported
|
|
471
|
+
if (typeof run.exitCode !== "number")
|
|
472
|
+
return `\`${cmd}\` reported counts but no exit status — nothing shows the command succeeded, so its counts are not a pass`;
|
|
473
|
+
if (run.exitCode !== 0 && !failingOf(run))
|
|
474
|
+
return `\`${cmd}\` exited ${run.exitCode} but reported no failing or cancelled tests — its counts do not account for the failure, so they are not a pass`;
|
|
434
475
|
return null;
|
|
435
476
|
}
|
|
436
477
|
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
478
|
+
// A cancelled test is a failure the runner counts apart from `fail` — node's
|
|
479
|
+
// runner exits 1 on one with `fail 0` — so both are the review's to report.
|
|
480
|
+
function failingOf(run) {
|
|
481
|
+
return (run.fail || 0) + (run.cancelled || 0);
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
function failingDesc(run) {
|
|
485
|
+
return run.cancelled ? `${run.fail || 0} failing and ${run.cancelled} cancelled tests` : `${run.fail} failing tests`;
|
|
440
486
|
}
|
|
441
487
|
|
|
488
|
+
// Failing tests are owned by the review as a whole (#2315): `findings` is
|
|
489
|
+
// every finding that reached the payload, and ONE from any dimension
|
|
490
|
+
// satisfies the check for all — no dimension is asked to duplicate a
|
|
491
|
+
// sibling's, and none is marked unrun because a sibling filed it. A refuted
|
|
492
|
+
// finding does not count: the review rejected it, so an unrelated claim its
|
|
493
|
+
// verifiers threw out would otherwise stand in for failures nobody reported.
|
|
494
|
+
// A dimension whose verify stage died never reaches the payload, so its
|
|
495
|
+
// findings are not here to count either.
|
|
496
|
+
export function unrunReason(run, findings) {
|
|
497
|
+
const why = testRunReason(run);
|
|
498
|
+
if (why) return why;
|
|
499
|
+
if (failingOf(run) > 0 && !findings.some((f) => f && f.verdict !== "refuted"))
|
|
500
|
+
return `\`${run.command || "the test command"}\` reported ${failingDesc(run)} and no selected dimension filed a finding about them that the review kept`;
|
|
501
|
+
return null;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// One verdict, every key: the shared run is the same run for each of them.
|
|
505
|
+
export function unrunEntries(run, findings, keys) {
|
|
506
|
+
const why = unrunReason(run, findings);
|
|
507
|
+
return why ? keys.map((dimension) => ({ dimension, reason: why })) : [];
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
export const CRASHED_REASON = "the reviewer returned nothing — spend limit, timeout, or terminal error";
|
|
511
|
+
|
|
442
512
|
export function unrunCrashed(reviewed, dimensions) {
|
|
443
|
-
return reviewed.flatMap((r, i) => (r ? [] :
|
|
513
|
+
return reviewed.flatMap((r, i) => (r ? [] : [{ dimension: dimensions[i]?.key ?? `slot ${i}`, reason: CRASHED_REASON }]));
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
function countsOf(run) {
|
|
517
|
+
return ["tests", "pass", "fail", "cancelled", "skipped"]
|
|
518
|
+
.filter((k) => typeof run[k] === "number")
|
|
519
|
+
.map((k) => `${k} ${run[k]}`)
|
|
520
|
+
.join(", ");
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// #2315. The Tests paragraph of every specialist prompt: the shared run's
|
|
524
|
+
// command, counts, exit status and log path, in place of the instruction to
|
|
525
|
+
// run the full suite that each specialist used to follow — one review was up
|
|
526
|
+
// to six full sweeps of one immutable snapshot. `owner` is the one dimension
|
|
527
|
+
// told to file the finding about failing tests, so the others are told not
|
|
528
|
+
// to duplicate it rather than left to race each other to it.
|
|
529
|
+
export function sharedRunNote(run, owner, key) {
|
|
530
|
+
const lines = [
|
|
531
|
+
`Tests: this review ran its test command ONCE, from the snapshot's root, before
|
|
532
|
+
dispatching you. Do NOT run that full command yourself — every dimension reads
|
|
533
|
+
this one run. Targeted probes and mutations in your own copy of the snapshot are
|
|
534
|
+
still yours to run.
|
|
535
|
+
command: ${run.command}
|
|
536
|
+
counts: ${countsOf(run) || "none — the run produced no counts"}
|
|
537
|
+
exit: ${run.exitCode ?? "(not reported)"}
|
|
538
|
+
log: ${run.logPath}
|
|
539
|
+
Read the log for anything the counts do not say. Report this run in
|
|
540
|
+
\`test_run\` — its command verbatim and these counts, \`tests: 0\` when it states
|
|
541
|
+
none — never a run of your own.`,
|
|
542
|
+
];
|
|
543
|
+
const why = testRunReason(run);
|
|
544
|
+
if (why)
|
|
545
|
+
lines.push(`This run is NOT usable: ${why}. Every dimension of this review is reported
|
|
546
|
+
unrun for it — do not run the full command yourself to replace it.`);
|
|
547
|
+
else if (failingOf(run) > 0)
|
|
548
|
+
lines.push(
|
|
549
|
+
key === owner
|
|
550
|
+
? `It has ${failingDesc(run)}, and filing them is YOUR job in this review:
|
|
551
|
+
file at least one finding about them, naming each failing test from the log. A
|
|
552
|
+
failure you cannot separate from the environment is still filed — say so in the
|
|
553
|
+
finding, so it is reproduced in the worktree before anyone acts on it. If none is
|
|
554
|
+
filed, or this review's refuters reject every finding filed, every dimension of
|
|
555
|
+
this review is reported unrun.`
|
|
556
|
+
: `It has ${failingDesc(run)}, and the ${owner} dimension files the finding
|
|
557
|
+
about them. Do not file a duplicate — cite a failure as evidence only where it
|
|
558
|
+
bears on your own lens.`,
|
|
559
|
+
);
|
|
560
|
+
return lines.join("\n");
|
|
444
561
|
}
|
|
445
562
|
|
|
446
563
|
// #1433 gap: the CWD-AUDIT line the Review dispatch below asks a specialist
|
|
@@ -528,6 +645,30 @@ export function runnerPrRefusal(pr) {
|
|
|
528
645
|
return isDigits(pr) ? null : `args.pr must be a PR number, got ${JSON.stringify(pr)}`;
|
|
529
646
|
}
|
|
530
647
|
|
|
648
|
+
// #2323. `runReview` owns the review side's `pr<N>/` partition — it appends
|
|
649
|
+
// `/pr${pr}` to the scratch it is given — so a scratch whose last component
|
|
650
|
+
// is already `pr` plus digits is a caller that partitioned too, and its run
|
|
651
|
+
// roots would nest at `<root>/pr<N>/pr<N>/run-*`, below the snapshot prompt's
|
|
652
|
+
// `find <root>/pr<N> -maxdepth 1` prune. Refused whatever the digits, never
|
|
653
|
+
// coerced: stripping the suffix would guess which directory the caller meant.
|
|
654
|
+
//
|
|
655
|
+
// "Last component" is the one the filesystem resolves, not the last text in
|
|
656
|
+
// the string: `.` segments and repeated or trailing slashes are collapsed
|
|
657
|
+
// first, so `/x/pr42/.` is refused like `/x/pr42`. A `..` segment is refused
|
|
658
|
+
// outright rather than collapsed — the kernel resolves it against a symlink's
|
|
659
|
+
// target and `posix.normalize` against the text, so `/x/link/..` can name a
|
|
660
|
+
// `pr<N>` directory the lexical answer says it does not. Shared with
|
|
661
|
+
// `runReviewToFile` for the reason the digits rule above is. Null when
|
|
662
|
+
// `scratch` is acceptable.
|
|
663
|
+
export function runnerScratchRefusal(scratch) {
|
|
664
|
+
if (typeof scratch !== "string") return null;
|
|
665
|
+
if (scratch.split("/").includes("..")) {
|
|
666
|
+
return `args.scratch ${JSON.stringify(scratch)} has a ".." segment — pass the scratch root as a path without one`;
|
|
667
|
+
}
|
|
668
|
+
if (!/(?:^|\/)pr\d+\/*$/.test(posix.normalize(scratch))) return null;
|
|
669
|
+
return `args.scratch ${JSON.stringify(scratch)} already ends in a pr<N> directory — pass the scratch root; the review creates <root>/pr<N>/ itself`;
|
|
670
|
+
}
|
|
671
|
+
|
|
531
672
|
// --- Orchestration ----------------------------------------------------
|
|
532
673
|
// `host` supplies `agent(prompt, opts)` (must resolve to PARSED DATA — a
|
|
533
674
|
// rejection or an unresolvable dispatch must resolve to `null`, the
|
|
@@ -600,6 +741,9 @@ export async function runReview(host, args) {
|
|
|
600
741
|
// into it: an absent `pr` is owed "required", not a complaint about digits.
|
|
601
742
|
if (!isDigits(pr)) throw new Error(`review-pr: args.pr must be a PR number, got ${JSON.stringify(pr)}`);
|
|
602
743
|
|
|
744
|
+
const scratchRefusal = runnerScratchRefusal(scratch);
|
|
745
|
+
if (scratchRefusal) throw new Error(`review-pr: ${scratchRefusal}`);
|
|
746
|
+
|
|
603
747
|
const explicitDimensions = resolveDimensions(A.dimensions, DEFAULT_DIMENSIONS);
|
|
604
748
|
|
|
605
749
|
phase("Snapshot");
|
|
@@ -611,7 +755,7 @@ export async function runReview(host, args) {
|
|
|
611
755
|
mkdir -p "${runRootParent}" || { echo SNAPSHOT_RUNROOT_FAILED; exit 1; }
|
|
612
756
|
RUN=$(mktemp -d "${runRootPrefix}XXXXXXXX") || { echo SNAPSHOT_RUNROOT_FAILED; exit 1; }
|
|
613
757
|
echo SNAPSHOT_RUN_ROOT="$RUN"
|
|
614
|
-
find "${runRootParent}" -maxdepth 1 -type d -name 'run-*' -mtime +7 -exec rm -rf {} + || echo SNAPSHOT_PRUNE_FAILED
|
|
758
|
+
find "${runRootParent}" -maxdepth 1 -type d -name 'run-*' -mtime +7 -exec sh -c 'rc=0; for d; do chmod -R u+rwx "$d" 2>/dev/null; if command -v chflags >/dev/null 2>&1; then chflags -R nouchg "$d" 2>/dev/null; chmod -R u+rwx "$d" 2>/dev/null; fi; rm -rf "$d" || rc=1; done; exit $rc' sh {} + || echo SNAPSHOT_PRUNE_FAILED
|
|
615
759
|
SHA=$(git -C ${worktree} rev-parse --short HEAD) || { echo SNAPSHOT_REVPARSE_FAILED; exit 1; }
|
|
616
760
|
SNAP="$RUN/snapshot-$SHA"
|
|
617
761
|
echo SNAPSHOT_DEST="$SNAP"
|
|
@@ -633,7 +777,7 @@ Report \`repoVerified\` = true ONLY if the last line above printed
|
|
|
633
777
|
SNAPSHOT_TREE_MATCH; otherwise report it false and put the line it printed
|
|
634
778
|
instead — SNAPSHOT_INIT_FAILED, or the whole SNAPSHOT_TREE_MISMATCH= value with
|
|
635
779
|
both hashes — in \`repoError\`. The init is what makes the snapshot MEASURABLE:
|
|
636
|
-
|
|
780
|
+
the review runs this repository's own suite in there, and a suite with tests
|
|
637
781
|
that need a working tree reports fewer passes and more failures in a bare
|
|
638
782
|
extraction than in a checkout at the same commit, with nothing in the payload
|
|
639
783
|
saying the measurement happened somewhere else (#1056). The tree-hash compare is
|
|
@@ -750,6 +894,65 @@ a false repoVerified.`,
|
|
|
750
894
|
);
|
|
751
895
|
log(`agents dispatched ${dimensions.map((d) => `${d.key}=${d.agentType}`).join(" ")}`);
|
|
752
896
|
|
|
897
|
+
// #2315. The review's ONE run of the test command, before any specialist is
|
|
898
|
+
// dispatched: every dimension used to run the full suite itself, so one
|
|
899
|
+
// review was up to six full sweeps of one immutable snapshot. NOT wrapped in
|
|
900
|
+
// `retryCrashed`: a crashed dispatch may already have launched the command,
|
|
901
|
+
// so a retry would be a second launch — the load this step removes. A null
|
|
902
|
+
// or thrown answer is a run with no counts, which `unrunReason` reports for
|
|
903
|
+
// every dimension, never a pass.
|
|
904
|
+
const logPath = `${snap.runRoot}/test-run.log`;
|
|
905
|
+
phase("Test run");
|
|
906
|
+
const ran = await agent(
|
|
907
|
+
`Run this repository's test command ONCE, for the review of PR #${pr}, and
|
|
908
|
+
report what it printed. Every review specialist reads your counts instead of
|
|
909
|
+
running the suite itself, so this is the review's only full run.
|
|
910
|
+
|
|
911
|
+
Your shell starts in a directory you must not write to: this dispatch carries
|
|
912
|
+
no working directory of its own, so you begin wherever the controller's own
|
|
913
|
+
review cell is standing, and a relative path in any command lands THERE. The
|
|
914
|
+
block below is absolute and cd-chained on purpose — run it exactly as written,
|
|
915
|
+
and add nothing relative to it.
|
|
916
|
+
|
|
917
|
+
Run it as ONE blocking foreground command — never backgrounded, never polled —
|
|
918
|
+
with a command deadline of 1800 seconds:
|
|
919
|
+
|
|
920
|
+
{ cd "${snap.path}" && ${testCmd}; } > "${logPath}" 2>&1; echo "TEST_RUN_EXIT=$?"
|
|
921
|
+
|
|
922
|
+
Run it exactly once: not again when it fails, and not again when it hits the
|
|
923
|
+
deadline. A failing run is a result every specialist reads, and a second
|
|
924
|
+
launch is the load this step exists to remove.
|
|
925
|
+
|
|
926
|
+
Do not substitute a command of your own. In a repo that has a shared test stack,
|
|
927
|
+
a bare runner picks up a default config whose setup can tear a sibling's
|
|
928
|
+
container down mid-run; a guessed glob is worse in every repo, because one
|
|
929
|
+
matching nothing still exits 0 reporting 'tests 0' — a green that ran nothing.
|
|
930
|
+
Run it from the snapshot's root, as the block does: a run from a subdirectory
|
|
931
|
+
reports a count well below what the whole tree reports, which means it ran a
|
|
932
|
+
PARTIAL copy, and nothing downstream can catch that one, because only this run
|
|
933
|
+
knows what the full tree reports.
|
|
934
|
+
|
|
935
|
+
Then read the summary the runner printed at the end of the log —
|
|
936
|
+
\`tail -n 40 "${logPath}"\`, ignoring any color escapes — and report
|
|
937
|
+
\`exitCode\` = the TEST_RUN_EXIT value, and \`tests\`, \`pass\`, \`fail\`,
|
|
938
|
+
\`cancelled\`, \`skipped\` = the counts that summary states, each copied as
|
|
939
|
+
printed; omit any the runner does not print. Copy the counts, never judge them:
|
|
940
|
+
'tests 0' is a FAILED run, not a pass, and 0 passes with no failures is
|
|
941
|
+
everything skipped — report both exactly as printed, and the caller reports
|
|
942
|
+
every dimension unrun for them.
|
|
943
|
+
|
|
944
|
+
If the command hit its deadline, crashed before printing a summary, or printed
|
|
945
|
+
no counts at all, omit every count and say what happened in \`error\`. Never
|
|
946
|
+
write 0 for a count the log does not state: an absent count is how the caller
|
|
947
|
+
learns the run produced none.`,
|
|
948
|
+
{ label: "test-run", phase: "Test run", agentType: TEST_RUN_AGENT_TYPE, schema: TEST_RUN_SCHEMA },
|
|
949
|
+
).catch((e) => ({ error: `the test-run dispatch threw: ${e?.message ?? e}` }));
|
|
950
|
+
const sharedRun = { ...(ran || { error: "the test-run agent returned nothing" }), command: testCmd, logPath };
|
|
951
|
+
log(`shared test run: ${countsOf(sharedRun) || "no counts"} — exit ${sharedRun.exitCode ?? "(absent)"} — log ${logPath}`);
|
|
952
|
+
// Failing tests are the review's to report, not each dimension's: the first
|
|
953
|
+
// selected dimension is told to file them, and every other one not to.
|
|
954
|
+
const failureOwner = dimensions[0].key;
|
|
955
|
+
|
|
753
956
|
const dimensionsUnrun = [];
|
|
754
957
|
// #1433. Per-dimension record of the specialist's own CWD-AUDIT line (see
|
|
755
958
|
// `cwdAuditFrom` above) — `{dimension, state, line}`, `state` one of
|
|
@@ -769,13 +972,13 @@ a false repoVerified.`,
|
|
|
769
972
|
// tree it must not write to, `pwd` fixes which directory that is, and the
|
|
770
973
|
// `CWD-AUDIT:` line is what makes a clean run say so — an audit reported
|
|
771
974
|
// only when dirty is indistinguishable from one never run, the same reading
|
|
772
|
-
// `unrunReason` applies to a
|
|
975
|
+
// `unrunReason` applies to a shared test run that reports nothing.
|
|
773
976
|
phase("Review");
|
|
774
977
|
const reviewed = await pipeline(
|
|
775
978
|
dimensions,
|
|
776
979
|
// #1802: a crashed specialist (null or thrown) is re-dispatched once
|
|
777
980
|
// before it counts as unrun. A specialist that RETURNED is never re-run
|
|
778
|
-
// here
|
|
981
|
+
// here — the shared run, not the specialist, is what `unrunEntries` judges.
|
|
779
982
|
(d) =>
|
|
780
983
|
retryCrashed(
|
|
781
984
|
() =>
|
|
@@ -799,26 +1002,10 @@ in this prompt as absolute paths.
|
|
|
799
1002
|
|
|
800
1003
|
${readRules(usableDiff(snap), stats, snap)}
|
|
801
1004
|
|
|
802
|
-
|
|
803
|
-
${testCmd}
|
|
804
|
-
Do not substitute a command of your own. In a repo that has a shared test stack,
|
|
805
|
-
a bare runner picks up a default config whose setup can tear a sibling's
|
|
806
|
-
container down mid-run; a guessed glob is worse in every repo, because one
|
|
807
|
-
matching nothing still exits 0 reporting 'tests 0' — a green that ran nothing.
|
|
808
|
-
Whatever you run, report it in \`test_run\` — the command verbatim and the counts
|
|
809
|
-
you saw — even when it failed or produced nothing.
|
|
810
|
-
'tests 0' is a FAILED run, not a pass: \`tests: 0\` is how this dimension gets
|
|
811
|
-
reported unrun, and an empty findings list cannot say it for you. So is a run
|
|
812
|
-
that collected tests and did none of the work — 0 passes with no failures is
|
|
813
|
-
everything skipped. And a count well below what the whole tree reports means you
|
|
814
|
-
ran a PARTIAL copy: nothing downstream can catch that one for you, because only
|
|
815
|
-
your own run knows what the full tree reports. Run from the snapshot's root, and
|
|
816
|
-
report any of these as unrun.
|
|
1005
|
+
${sharedRunNote(sharedRun, failureOwner, d.key)}
|
|
817
1006
|
|
|
818
1007
|
${environmentNote(snap)}
|
|
819
1008
|
|
|
820
|
-
A failure you cannot separate from the environment is neither a finding nor a
|
|
821
|
-
reason to file nothing — say which it is, beside the counts, in \`test_run\`.
|
|
822
1009
|
Scratch files go in ${snap.runRoot}/${d.key}/ and nowhere else. Chain the
|
|
823
1010
|
directory change into the command, \`cd "$D" && git …\`, never
|
|
824
1011
|
\`cd "$D"; git …\`, so a failed \`cd\` cannot leave a \`git\` command running in the
|
|
@@ -851,7 +1038,6 @@ to name.`,
|
|
|
851
1038
|
),
|
|
852
1039
|
|
|
853
1040
|
(review, d) => {
|
|
854
|
-
dimensionsUnrun.push(...unrunEntries(review, d.key));
|
|
855
1041
|
cwdAudit.push({ dimension: d.key, ...cwdAuditFrom(review && review.scope_searched) });
|
|
856
1042
|
phase("Verify");
|
|
857
1043
|
return parallel(
|
|
@@ -964,13 +1150,20 @@ how three reviews from one cell left four files modified in that checkout
|
|
|
964
1150
|
},
|
|
965
1151
|
);
|
|
966
1152
|
|
|
967
|
-
dimensionsUnrun.push(...unrunCrashed(reviewed, dimensions));
|
|
968
|
-
|
|
969
1153
|
const all = reviewed.flat().filter(Boolean);
|
|
970
1154
|
const survived = all.filter((f) => f.verdict === "survived");
|
|
971
1155
|
const refuted = all.filter((f) => f.verdict === "refuted");
|
|
972
1156
|
const unverified = all.filter((f) => f.verdict === "unverified");
|
|
973
1157
|
|
|
1158
|
+
// #2315. One verdict on the shared run, for every dimension whose chain did
|
|
1159
|
+
// not die — a crashed one is named below with its own reason instead, so no
|
|
1160
|
+
// key is listed twice. Read only once every dimension has been verified: the
|
|
1161
|
+
// failing-tests check reads the findings that reached the payload, across
|
|
1162
|
+
// ALL dimensions, refuted ones excluded.
|
|
1163
|
+
const live = dimensions.filter((_, i) => reviewed[i]).map((d) => d.key);
|
|
1164
|
+
dimensionsUnrun.push(...unrunEntries(sharedRun, all, live));
|
|
1165
|
+
dimensionsUnrun.push(...unrunCrashed(reviewed, dimensions));
|
|
1166
|
+
|
|
974
1167
|
const { crashed, resume } = resumeFor(unverified);
|
|
975
1168
|
|
|
976
1169
|
log(
|
|
@@ -988,13 +1181,14 @@ how three reviews from one cell left four files modified in that checkout
|
|
|
988
1181
|
// separately so the caller never mistakes "not checked" for "survived".
|
|
989
1182
|
// `dimensionsRun` names what was DISPATCHED after the size trim: a trimmed
|
|
990
1183
|
// fan-out must say so, never read as full coverage. It is not a coverage claim
|
|
991
|
-
// on its own and never was — a specialist can be dispatched and die, or
|
|
992
|
-
//
|
|
993
|
-
//
|
|
994
|
-
//
|
|
995
|
-
//
|
|
996
|
-
//
|
|
997
|
-
//
|
|
1184
|
+
// on its own and never was — a specialist can be dispatched and die, or the
|
|
1185
|
+
// review's one shared test run can execute nothing (#2315) — so
|
|
1186
|
+
// `dimensionsUnrun` names which of those keys did not cover their ground,
|
|
1187
|
+
// and why. A key in the first and NOT in the second ran a suite — not that
|
|
1188
|
+
// it is covered (#535). `unrunReason` reads the shared run's counts and
|
|
1189
|
+
// quotes its command into its message; it never checks that the test-run
|
|
1190
|
+
// agent ran the command it was handed rather than a narrower one, so a
|
|
1191
|
+
// substituted runner is not classified unrun.
|
|
998
1192
|
//
|
|
999
1193
|
// The two are siblings rather than one filtered list because they answer
|
|
1000
1194
|
// different questions. Subtracting the unrun ones from `dimensionsRun` would
|
package/scripts/review-eval.mjs
CHANGED
|
@@ -6,6 +6,10 @@
|
|
|
6
6
|
// const { runReviewOnOmp } = await import(path);
|
|
7
7
|
// const result = await runReviewOnOmp({ pr, branch, worktree, testCmd, scratch });
|
|
8
8
|
//
|
|
9
|
+
// `scratch` is the scratch ROOT, never the review side's `<root>/pr<N>`:
|
|
10
|
+
// runReview creates `pr<N>/` under it itself, and refuses a scratch that
|
|
11
|
+
// already ends in one (#2323).
|
|
12
|
+
//
|
|
9
13
|
// Since #1802 the cell that does this is the `review-pr-<pr#>` member's own —
|
|
10
14
|
// agents/fleet-review-runner.agent.md, off the controller's turn — and it calls
|
|
11
15
|
// `runReviewToFile` (bottom of this file), which wraps `runReviewOnOmp` with the
|
|
@@ -28,7 +32,7 @@
|
|
|
28
32
|
// strings are.
|
|
29
33
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
30
34
|
import { isAbsolute, join } from "node:path";
|
|
31
|
-
import { digestOf, runReview, runnerPrRefusal } from "./review-core.mjs";
|
|
35
|
+
import { digestOf, runReview, runnerPrRefusal, runnerScratchRefusal } from "./review-core.mjs";
|
|
32
36
|
|
|
33
37
|
// eval's `agent()` returns a HANDLE, not data (#1296 Q2): `agent(prompt,
|
|
34
38
|
// opts)` resolves near-instantly to an `AgentHandle` with `.wait()`, and only
|
|
@@ -146,9 +150,10 @@ export async function runReviewOnOmp(args) {
|
|
|
146
150
|
// reviewer's name (`review-pr-<pr>-b`), and writes no file, so nothing reads a
|
|
147
151
|
// half-review as a review. A dispatch mistake — a pr that is not a PR number, a
|
|
148
152
|
// scratch that is not absolute (eval's cwd is the MAIN CHECKOUT, so a relative
|
|
149
|
-
// one would put the file there)
|
|
150
|
-
//
|
|
151
|
-
// the test injects; nothing else
|
|
153
|
+
// one would put the file there), a scratch that already ends in `pr<N>` (#2323)
|
|
154
|
+
// — throws before any run: it is not a review failure, and retrying or falling
|
|
155
|
+
// back would only repeat it. `run` is the seam the test injects; nothing else
|
|
156
|
+
// passes it.
|
|
152
157
|
export async function runReviewToFile(args, run = runReviewOnOmp) {
|
|
153
158
|
const pr = args?.pr;
|
|
154
159
|
const scratch = args?.scratch;
|
|
@@ -157,6 +162,8 @@ export async function runReviewToFile(args, run = runReviewOnOmp) {
|
|
|
157
162
|
if (typeof scratch !== "string" || !isAbsolute(scratch)) {
|
|
158
163
|
throw new Error(`review-runner: args.scratch must be an absolute path, got ${JSON.stringify(scratch)}`);
|
|
159
164
|
}
|
|
165
|
+
const scratchRefusal = runnerScratchRefusal(scratch);
|
|
166
|
+
if (scratchRefusal) throw new Error(`review-runner: ${scratchRefusal}`);
|
|
160
167
|
const errors = [];
|
|
161
168
|
for (let attempt = 1; attempt <= 2; attempt++) {
|
|
162
169
|
let result;
|
|
@@ -39,6 +39,10 @@ export const SNAP = {
|
|
|
39
39
|
testCmd: "node --test",
|
|
40
40
|
};
|
|
41
41
|
export const RAN = { command: "node --test", tests: 5, pass: 5, fail: 0 };
|
|
42
|
+
// #2315. The shared test run's agent report — the `test-run` dispatch every
|
|
43
|
+
// review makes once, before any specialist. Clean, so a script that is about
|
|
44
|
+
// something else need not answer it; one that is about it names its own.
|
|
45
|
+
export const SHARED = { exitCode: 0, tests: 5, pass: 5, fail: 0 };
|
|
42
46
|
export const review = (findings, testRun = RAN) => ({
|
|
43
47
|
dimension: "correctness",
|
|
44
48
|
scope_searched: "CWD-AUDIT: clean /repo",
|
|
@@ -52,15 +56,20 @@ export const vote = (refuted) => ({ refuted, reason: "measured. CWD-AUDIT: clean
|
|
|
52
56
|
// per call in dispatch order (the last entry repeats), and counts the calls.
|
|
53
57
|
// An `Error` entry is thrown rather than returned. Counting happens
|
|
54
58
|
// synchronously on the call, so the two refuters of one pair are calls 1 and
|
|
55
|
-
// 2 of their label, and a re-dispatched pair is calls 3 and 4.
|
|
59
|
+
// 2 of their label, and a re-dispatched pair is calls 3 and 4. A script with
|
|
60
|
+
// no `test-run` entry gets `[SHARED]` for it; every other unscripted label
|
|
61
|
+
// throws. `prompts` records each dispatch's prompt under its label.
|
|
56
62
|
export function scriptedHost(script) {
|
|
57
63
|
const calls = {};
|
|
64
|
+
const prompts = {};
|
|
58
65
|
return {
|
|
59
66
|
calls,
|
|
67
|
+
prompts,
|
|
60
68
|
host: {
|
|
61
|
-
agent: async (
|
|
69
|
+
agent: async (prompt, opts) => {
|
|
62
70
|
const n = (calls[opts.label] = (calls[opts.label] ?? 0) + 1);
|
|
63
|
-
|
|
71
|
+
(prompts[opts.label] ??= []).push(prompt);
|
|
72
|
+
const seq = script[opts.label] ?? (opts.label === "test-run" ? [SHARED] : undefined);
|
|
64
73
|
if (!seq) throw new Error(`scriptedHost: unexpected dispatch ${opts.label}`);
|
|
65
74
|
const answer = seq[Math.min(n, seq.length) - 1];
|
|
66
75
|
if (answer instanceof Error) throw answer;
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
import { test } from "node:test";
|
|
22
22
|
import assert from "node:assert/strict";
|
|
23
23
|
import { runReview, DIGEST_KEYS, digestOf } from "./review-core.mjs";
|
|
24
|
-
import { pipeline, parallel, ARGS, SNAP, review, finding, vote, scriptedHost } from "./review-host-fixture.mjs";
|
|
24
|
+
import { pipeline, parallel, ARGS, SNAP, SHARED, review, finding, vote, scriptedHost } from "./review-host-fixture.mjs";
|
|
25
25
|
|
|
26
26
|
const COPIES = [["review-core.mjs", (host, args) => runReview({ ...host, pipeline, parallel }, args)]];
|
|
27
27
|
|
|
@@ -50,17 +50,40 @@ for (const [name, run] of COPIES) {
|
|
|
50
50
|
assert.match(result.dimensionsUnrun[0].reason, /returned nothing/);
|
|
51
51
|
});
|
|
52
52
|
|
|
53
|
-
|
|
53
|
+
// #2315: the suite is the review's one shared run now, and it is never
|
|
54
|
+
// re-dispatched either — a crashed test-run agent may already have launched
|
|
55
|
+
// the command, so a retry would be a second full run.
|
|
56
|
+
test(`${name}: a specialist that RETURNED over a shared run that ran nothing is not a crash — never re-dispatched`, async () => {
|
|
54
57
|
const { host, calls } = scriptedHost({
|
|
55
58
|
snapshot: [SNAP],
|
|
56
|
-
"
|
|
59
|
+
"test-run": [{ exitCode: 0, tests: 0, pass: 0, fail: 0 }],
|
|
60
|
+
"review:correctness": [review([])],
|
|
57
61
|
});
|
|
58
62
|
const result = await run(host, ARGS);
|
|
59
63
|
assert.equal(calls["review:correctness"], 1, "a returned-but-unrun review was re-dispatched as if it had crashed");
|
|
64
|
+
assert.equal(calls["test-run"], 1);
|
|
60
65
|
assert.equal(result.dimensionsUnrun.length, 1);
|
|
61
66
|
assert.match(result.dimensionsUnrun[0].reason, /0 tests/);
|
|
62
67
|
});
|
|
63
68
|
|
|
69
|
+
test(`${name}: a crashed shared test run is dispatched once, never retried, and every dimension is unrun`, async () => {
|
|
70
|
+
for (const crash of [null, new Error("spend limit")]) {
|
|
71
|
+
const { host, calls } = scriptedHost({
|
|
72
|
+
snapshot: [SNAP],
|
|
73
|
+
"test-run": [crash, SHARED],
|
|
74
|
+
"review:correctness": [review([])],
|
|
75
|
+
"review:tests": [review([])],
|
|
76
|
+
});
|
|
77
|
+
const result = await run(host, { ...ARGS, dimensions: ["correctness", "tests"] });
|
|
78
|
+
assert.equal(calls["test-run"], 1, "the shared test run was re-dispatched — a second full-suite launch");
|
|
79
|
+
assert.deepEqual(
|
|
80
|
+
result.dimensionsUnrun.map((u) => u.dimension),
|
|
81
|
+
["correctness", "tests"],
|
|
82
|
+
);
|
|
83
|
+
for (const u of result.dimensionsUnrun) assert.match(u.reason, /no counts|returned nothing/);
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
|
|
64
87
|
test(`${name}: a refuter pair whose every vote died is re-dispatched once, as a pair`, async () => {
|
|
65
88
|
const { host, calls } = scriptedHost({
|
|
66
89
|
snapshot: [SNAP],
|
|
@@ -106,6 +129,7 @@ for (const [name, run] of COPIES) {
|
|
|
106
129
|
const host = {
|
|
107
130
|
agent: async (prompt, opts) => {
|
|
108
131
|
if (opts.label === "snapshot") return structuredClone(SNAP);
|
|
132
|
+
if (opts.label === "test-run") return structuredClone(SHARED);
|
|
109
133
|
if (opts.label === "review:correctness") return structuredClone(review([crashing, surviving]));
|
|
110
134
|
if (opts.label === "verify:correctness") {
|
|
111
135
|
if (prompt.includes(crashing.claim)) {
|
|
@@ -20,7 +20,7 @@ import assert from "node:assert/strict";
|
|
|
20
20
|
import { readFileSync, existsSync } from "node:fs";
|
|
21
21
|
import { join } from "node:path";
|
|
22
22
|
import { parseFrontmatter } from "./frontmatter-check.mjs";
|
|
23
|
-
import { SNAPSHOT_AGENT_TYPE, SPAWNED_AGENT_TYPES, VERIFIER_AGENT_TYPE } from "./review-core.mjs";
|
|
23
|
+
import { SNAPSHOT_AGENT_TYPE, SPAWNED_AGENT_TYPES, TEST_RUN_AGENT_TYPE, VERIFIER_AGENT_TYPE } from "./review-core.mjs";
|
|
24
24
|
|
|
25
25
|
const REPO = join(import.meta.dirname, "..");
|
|
26
26
|
const RUNNER_PATH = join(REPO, "agents", "fleet-review-runner.agent.md");
|
|
@@ -76,9 +76,9 @@ test("every agent type fleet-review-runner is allowed to spawn has a definition
|
|
|
76
76
|
}
|
|
77
77
|
});
|
|
78
78
|
|
|
79
|
-
test("review-core.mjs's snapshot/verifier dispatch sites reference the exported constants, not a hardcoded duplicate", () => {
|
|
79
|
+
test("review-core.mjs's snapshot/test-run/verifier dispatch sites reference the exported constants, not a hardcoded duplicate", () => {
|
|
80
80
|
// #2102 review: SPAWNED_AGENT_TYPES and the runner's spawns pin above both
|
|
81
|
-
// derive from SNAPSHOT_AGENT_TYPE/VERIFIER_AGENT_TYPE, never from the
|
|
81
|
+
// derive from SNAPSHOT_AGENT_TYPE/TEST_RUN_AGENT_TYPE/VERIFIER_AGENT_TYPE, never from the
|
|
82
82
|
// dispatch call sites themselves — so a call site quietly reverted to a
|
|
83
83
|
// hardcoded string literal (instead of the constant) would leave both
|
|
84
84
|
// pins green while review-core.mjs's real dispatch silently drifts from
|
|
@@ -88,6 +88,8 @@ test("review-core.mjs's snapshot/verifier dispatch sites reference the exported
|
|
|
88
88
|
const source = readFileSync(CORE_PATH, "utf8");
|
|
89
89
|
assert.equal(SNAPSHOT_AGENT_TYPE, "fleet-review-snapshot");
|
|
90
90
|
assert.match(source, /agentType:\s*SNAPSHOT_AGENT_TYPE\b/);
|
|
91
|
+
assert.equal(TEST_RUN_AGENT_TYPE, "fleet-review-test-run");
|
|
92
|
+
assert.match(source, /agentType:\s*TEST_RUN_AGENT_TYPE\b/);
|
|
91
93
|
assert.equal(VERIFIER_AGENT_TYPE, "fleet-review-verifier");
|
|
92
94
|
assert.match(source, /agentType:\s*VERIFIER_AGENT_TYPE\b/);
|
|
93
95
|
});
|