@feigi/fleet-ctl 3.32.1 → 3.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/fleet-review-runner.agent.md +1 -1
- package/agents/fleet-review-tests.agent.md +36 -4
- package/agents/fleet-review-verifier.agent.md +43 -7
- package/commands/review-and-fix.md +8 -7
- package/package.json +1 -1
- package/scripts/compute-board.mjs +4 -1
- package/scripts/dispositions-check.mjs +102 -48
- package/scripts/fleet-tick.mjs +41 -31
- package/scripts/ledger.mjs +6 -6
- package/scripts/review-core.mjs +84 -21
- package/scripts/review-eval.mjs +59 -11
- package/skills/run-team/SKILL.md +174 -55
package/scripts/fleet-tick.mjs
CHANGED
|
@@ -371,9 +371,15 @@ export class LedgerError extends Error {}
|
|
|
371
371
|
|
|
372
372
|
// `review=wf:<runId>` | `review=member:review-pr-<n>` |
|
|
373
373
|
// `review=fallback:review-pr-<n>[-b]`, settled dead as `…=failed`; the result
|
|
374
|
-
// is `reviewed=<head>:<survived>/<refuted>/<unverified
|
|
374
|
+
// is `reviewed=<head>:<survived>/<refuted>/<unverified>:<run>`, `<run>` the
|
|
375
|
+
// name of the review's own run root under `<scratch>/pr<n>/` (`run-` and the
|
|
376
|
+
// eight characters `mktemp` chose), where its result file `review.json` sits.
|
|
375
377
|
const REVIEW = /^review=(?:wf|member|fallback):[^=\s]+(=failed)?$/;
|
|
376
|
-
export const REVIEWED = /^reviewed=([0-9a-
|
|
378
|
+
export const REVIEWED = /^reviewed=([0-9a-fA-F]{7,40}):(\d+)\/(\d+)\/(\d+):(run-[A-Za-z0-9]{8})$/;
|
|
379
|
+
// The `reviewed=` shape before it named a run: it names no result file, so
|
|
380
|
+
// nothing can tell which review a dispositions verdict answered. Only a ledger
|
|
381
|
+
// written before the plugin update and read after it can hold one.
|
|
382
|
+
const REVIEWED_RUNLESS = /^reviewed=[0-9a-fA-F]{7,40}:\d+\/\d+\/\d+$/;
|
|
377
383
|
const HELD = /^held-behind[:-]#?(\d+)$/;
|
|
378
384
|
// The durable record that a PR conflicts — written onto the held PR's own
|
|
379
385
|
// row by the merge bot, when its local-rebase fallback hits a conflict it would
|
|
@@ -386,43 +392,42 @@ const CONFLICT_HOLD = /^conflict-hold[:-]#?(\d+)$/;
|
|
|
386
392
|
// for a settled member whose transcript was never written.
|
|
387
393
|
const TIER_VERDICT = /^tier-(ok|mismatch|unverifiable)=([^:\s]+):\S+$/;
|
|
388
394
|
// dispositions-check.mjs's verdict on a review fix-applier's disposition
|
|
389
|
-
// record: `dispositions-ok=fix-pr-<M>[-x]:<
|
|
395
|
+
// record: `dispositions-ok=fix-pr-<M>[-x]:<run>`,
|
|
390
396
|
// `dispositions-mismatch=…`, `dispositions-escalate=…` (a critical or
|
|
391
397
|
// important deferral, or a second mismatch on one review, that a human rules
|
|
392
398
|
// on) or `dispositions-unchecked=…` (no rule broke, but where a deferral was
|
|
393
|
-
// filed could not be read), `<
|
|
394
|
-
// answers. `ledger.mjs dispatch` refuses a
|
|
399
|
+
// filed could not be read), `<run>` the run of the review the record
|
|
400
|
+
// answers, as its `reviewed=` names it. `ledger.mjs dispatch` refuses a
|
|
401
|
+
// finisher on anything but an ok. A
|
|
395
402
|
// mismatch returns the PR to fixDue for one retry; an escalate is a hold only
|
|
396
403
|
// a human answers, so the tick never re-offers the PR for it; an unchecked is
|
|
397
404
|
// answered by running the check again, so the tick never offers a
|
|
398
405
|
// fix-applier for it either. A token outside this shape is no verdict at all,
|
|
399
406
|
// which the gate reads as unchecked: fail closed.
|
|
400
407
|
export function dispositionsToken(tok) {
|
|
401
|
-
const m = /^dispositions-(ok|mismatch|escalate|unchecked)=([^:\s]+):([
|
|
408
|
+
const m = /^dispositions-(ok|mismatch|escalate|unchecked)=([^:\s]+):(run-[A-Za-z0-9]{8})$/i.exec(tok);
|
|
402
409
|
if (!m) return null;
|
|
403
410
|
const member = parseMember(m[2]);
|
|
404
411
|
if (member === null || member.family !== "fix-pr") return null;
|
|
405
|
-
return { verdict: m[1].toLowerCase(), member,
|
|
412
|
+
return { verdict: m[1].toLowerCase(), member, run: m[3] };
|
|
406
413
|
}
|
|
407
414
|
|
|
408
|
-
// Two spellings of one commit: either head a prefix of the other, the way a
|
|
409
|
-
// `reviewed=` head (7-40 hex) is matched against gh's full headRefOid.
|
|
410
|
-
export const sameHead = (a, b) => a.startsWith(b) || b.startsWith(a);
|
|
411
|
-
|
|
412
415
|
// Which verdict outranks which when one fix-applier carries more than one for
|
|
413
|
-
// a
|
|
416
|
+
// a review: the stricter reading wins.
|
|
414
417
|
const VERDICT_RANK = { ok: 0, unchecked: 1, mismatch: 2, escalate: 3 };
|
|
415
418
|
|
|
416
|
-
// A PR's current dispositions verdict against its latest review
|
|
417
|
-
// the tokens answering that
|
|
419
|
+
// A PR's current dispositions verdict against its latest review run: among
|
|
420
|
+
// the tokens answering that run, the one from the fix-applier with the
|
|
418
421
|
// highest retry suffix ("" < "b" < "c" …) — never row-text position, which a
|
|
419
422
|
// `row` rewrite can reorder. One fix-applier carrying two verdicts for one
|
|
420
|
-
//
|
|
423
|
+
// review reads as the stricter one. null when no token answers the run — so a
|
|
424
|
+
// second review at the same head is answered by none of the first one's
|
|
425
|
+
// verdicts.
|
|
421
426
|
/** @returns {{verdict: "ok"|"unchecked"|"mismatch"|"escalate", member: string}|null} */
|
|
422
|
-
export function currentDispositions(tokens,
|
|
427
|
+
export function currentDispositions(tokens, run) {
|
|
423
428
|
let best = null;
|
|
424
429
|
for (const t of tokens) {
|
|
425
|
-
if (
|
|
430
|
+
if (t.run !== run) continue;
|
|
426
431
|
const retry = t.member.retry ?? "";
|
|
427
432
|
const bestRetry = best?.member.retry ?? "";
|
|
428
433
|
if (best === null || retry > bestRetry
|
|
@@ -604,20 +609,21 @@ export function deriveRun({ rows, dispatched, drain }, prs, closed = new Set(),
|
|
|
604
609
|
// `fixMembers`: every fix-applier on this PR, wherever its token sits —
|
|
605
610
|
// one still unsettled anywhere holds the PR off fixDue, since `dispatch`
|
|
606
611
|
// refuses a second live one on the PR.
|
|
607
|
-
// `reviewedHead`: the latest `reviewed=<head>`.
|
|
612
|
+
// `reviewedHead`/`reviewedRun`: the latest `reviewed=<head>:…:<run>`.
|
|
613
|
+
// `pastPinHalt`: the latest
|
|
608
614
|
// finisher since that review settled `halted:past-pin` — any
|
|
609
615
|
// later finisher attempt, live or settled, replaces it, and a later
|
|
610
616
|
// returned review answers it. `unverified`: the latest `reviewed=`'s
|
|
611
617
|
// unverified count. `dispositions`: every dispositions verdict a fix-pr
|
|
612
|
-
// member of this PR carries, whatever
|
|
613
|
-
// by
|
|
618
|
+
// member of this PR carries, whatever review it answers — the gate picks
|
|
619
|
+
// by run and retry suffix, never by where a token sits.
|
|
614
620
|
// `workers`: every fix-applier and finisher bound to this PR, and every
|
|
615
621
|
// implementer on a row resolving to it — one still unsettled holds the
|
|
616
622
|
// CONFLICT line off, since it may be pushing to the branch right now.
|
|
617
623
|
const st = (pr !== null && byPr.get(pr)) || {
|
|
618
624
|
inFlight: false, reviewedAny: false, survived: 0, unverified: 0, reviewFixed: false,
|
|
619
625
|
fixMembers: new Set(), fixLanded: new Set(), conflictLanded: new Set(), held: [],
|
|
620
|
-
conflictOpen: false, reviewedHead: null, pastPinHalt: false, dispositions: [], workers: new Set(),
|
|
626
|
+
conflictOpen: false, reviewedHead: null, reviewedRun: null, pastPinHalt: false, dispositions: [], workers: new Set(),
|
|
621
627
|
};
|
|
622
628
|
// The finisher-pr token currently deciding `pastPinHalt`, picked by
|
|
623
629
|
// retry suffix ("" < "b" < "c" …) the same way compute-board.mjs's
|
|
@@ -689,11 +695,15 @@ export function deriveRun({ rows, dispatched, drain }, prs, closed = new Set(),
|
|
|
689
695
|
if (!m[1]) st.reviewedAny = true;
|
|
690
696
|
} else if (tok.startsWith("reviewed=")) {
|
|
691
697
|
const m = REVIEWED.exec(tok);
|
|
692
|
-
if (!m
|
|
698
|
+
if (!m && REVIEWED_RUNLESS.test(tok)) {
|
|
699
|
+
throw new LedgerError(`${where}: '${tok}' names no review run — reviewed= is now <head>:<survived>/<refuted>/<unverified>:<run>, `
|
|
700
|
+
+ "<run> the run-XXXXXXXX directory holding that review's review.json; rewrite the token with `ledger.mjs row`");
|
|
701
|
+
}
|
|
702
|
+
if (!m) throw new LedgerError(`${where}: '${tok}' is not reviewed=<head>:<survived>/<refuted>/<unverified>:<run>`);
|
|
693
703
|
latestFinisher = null;
|
|
694
704
|
Object.assign(st, {
|
|
695
705
|
inFlight: false, reviewedAny: true, survived: Number(m[2]), unverified: Number(m[4]), reviewFixed: false,
|
|
696
|
-
reviewedHead: m[1].toLowerCase(), pastPinHalt: false,
|
|
706
|
+
reviewedHead: m[1].toLowerCase(), reviewedRun: m[5], pastPinHalt: false,
|
|
697
707
|
});
|
|
698
708
|
} else if (labelOffMember(tok) === null) {
|
|
699
709
|
throw new LedgerError(`${where}: '${tok}' is not label-off=<finisher-pr member> — fix the row with \`ledger.mjs row\``);
|
|
@@ -788,8 +798,7 @@ export function deriveRun({ rows, dispatched, drain }, prs, closed = new Set(),
|
|
|
788
798
|
// nothing, and so did a conflict fix-applier, which is never checked. An
|
|
789
799
|
// escalate is a different verdict and is never due.
|
|
790
800
|
const mismatchDue = (st) => {
|
|
791
|
-
|
|
792
|
-
const cur = currentDispositions(st.dispositions, st.reviewedHead);
|
|
801
|
+
const cur = currentDispositions(st.dispositions, st.reviewedRun);
|
|
793
802
|
if (cur === null || cur.verdict !== "mismatch") return false;
|
|
794
803
|
const retry = parseMember(cur.member).retry ?? "";
|
|
795
804
|
return ![...st.fixLanded].some((n) => !st.conflictLanded.has(n) && (parseMember(n).retry ?? "") > retry);
|
|
@@ -863,16 +872,17 @@ export function deriveRun({ rows, dispatched, drain }, prs, closed = new Set(),
|
|
|
863
872
|
conflicts: conflicting.filter((p) => !treadmill(p)).map((p) => p.number).sort(asc),
|
|
864
873
|
conflictEscalate: conflicting.filter(treadmill).map((p) => p.number).sort(asc),
|
|
865
874
|
// Every PR, open or not, with a returned review: its latest `reviewed=`
|
|
866
|
-
// head and counts, the dispositions verdict currently answering that
|
|
867
|
-
//
|
|
868
|
-
// re-review
|
|
875
|
+
// head, run and counts, the dispositions verdict currently answering that
|
|
876
|
+
// run, and every fix-applier on the PR still unsettled (`fixLive`) — a
|
|
877
|
+
// re-review would otherwise read the earlier
|
|
869
878
|
// fix-applier's verdict while the one answering it is still working.
|
|
870
879
|
// `ledger.mjs dispatch` gates a finisher off this, read from the same
|
|
871
|
-
// per-PR fold as everything above
|
|
880
|
+
// per-PR fold as everything above, and dispositions-check.mjs reads the
|
|
881
|
+
// review file it judges against off `run`. Nothing in this tick acts on it.
|
|
872
882
|
reviewed: [...byPr.entries()].filter(([, st]) => st.reviewedHead !== null)
|
|
873
883
|
.map(([n, st]) => ({
|
|
874
|
-
pr: n, head: st.reviewedHead, survived: st.survived, unverified: st.unverified,
|
|
875
|
-
dispositions: currentDispositions(st.dispositions, st.
|
|
884
|
+
pr: n, head: st.reviewedHead, run: st.reviewedRun, survived: st.survived, unverified: st.unverified,
|
|
885
|
+
dispositions: currentDispositions(st.dispositions, st.reviewedRun),
|
|
876
886
|
fixLive: [...st.fixMembers].filter((name) => members.get(name).outcome === null).sort(),
|
|
877
887
|
}))
|
|
878
888
|
.sort((a, b) => a.pr - b.pr),
|
package/scripts/ledger.mjs
CHANGED
|
@@ -1641,30 +1641,30 @@ function refuseMalformedDispatched() {
|
|
|
1641
1641
|
function dispositionsRefusal(pr, review) {
|
|
1642
1642
|
if (review === undefined) return null;
|
|
1643
1643
|
if (review.fixLive.length > 0) {
|
|
1644
|
-
return `${review.fixLive.join(", ")} still live on PR #${pr} — a fix-applier answering review ${review.
|
|
1644
|
+
return `${review.fixLive.join(", ")} still live on PR #${pr} — a fix-applier answering review ${review.run} has not settled, `
|
|
1645
1645
|
+ "so no dispositions verdict answers its work yet; settle it, run dispositions-check.mjs for it, then dispatch the finisher again";
|
|
1646
1646
|
}
|
|
1647
1647
|
if (review.survived === 0 && review.unverified === 0) return null;
|
|
1648
1648
|
const v = review.dispositions;
|
|
1649
1649
|
if (v === null) {
|
|
1650
1650
|
return `dispositions unchecked — no dispositions-ok=/dispositions-mismatch=/dispositions-escalate=/dispositions-unchecked= token answers PR #${pr}'s latest review, `
|
|
1651
|
-
+
|
|
1651
|
+
+ `${review.run} at ${review.head}; run dispositions-check.mjs --member fix-pr-${pr}[-<x>] --scratch <scratch> for the fix-applier `
|
|
1652
1652
|
+ "that answered that review, then dispatch the finisher again";
|
|
1653
1653
|
}
|
|
1654
1654
|
if (v.verdict === "unchecked") {
|
|
1655
|
-
return `dispositions unchecked — ${v.member}'s check of review ${review.
|
|
1655
|
+
return `dispositions unchecked — ${v.member}'s check of review ${review.run} could not read where a deferral was filed (gh unreachable, `
|
|
1656
1656
|
+ "or an issue unreadable — an issue that does not exist is unreadable too, and no re-run answers that one: the check's stderr names "
|
|
1657
1657
|
+ `it, and the record's issue number needs correcting); run dispositions-check.mjs --member ${v.member} --scratch <scratch> again once gh answers, `
|
|
1658
1658
|
+ "then dispatch the finisher again";
|
|
1659
1659
|
}
|
|
1660
1660
|
if (v.verdict === "mismatch") {
|
|
1661
|
-
return `dispositions mismatch — ${v.member}'s disposition record fails the check against review ${review.
|
|
1661
|
+
return `dispositions mismatch — ${v.member}'s disposition record fails the check against review ${review.run}; `
|
|
1662
1662
|
+ `dispositions-check.mjs --member ${v.member} --scratch <scratch> names each violating entry by bucket, index and rule`;
|
|
1663
1663
|
}
|
|
1664
1664
|
if (v.verdict === "escalate") {
|
|
1665
1665
|
return `dispositions escalate — ${v.member} deferred a finding above suggestion severity because its remedy lies outside the PR's diff, `
|
|
1666
1666
|
+ "or its record drew a second mismatch on this review; either needs a human's ruling: "
|
|
1667
|
-
+ `review ${review.
|
|
1667
|
+
+ `review ${review.run} gets no finisher and no retry; dispositions-check.mjs --member ${v.member} `
|
|
1668
1668
|
+ "--scratch <scratch> names each escalated finding or violating entry by bucket and index, for the PR comment that hands it to a human";
|
|
1669
1669
|
}
|
|
1670
1670
|
return null;
|
|
@@ -1775,7 +1775,7 @@ function runDispatch() {
|
|
|
1775
1775
|
// known to conflict cannot merge, and a finisher would race the fix-applier
|
|
1776
1776
|
// rebasing it — and once its current dispositions verdict is ok — both read
|
|
1777
1777
|
// off the tick's own per-PR fold, the verdict at the same latest `reviewed=`
|
|
1778
|
-
//
|
|
1778
|
+
// run the tick reads — and no fix-applier on the PR is still live. A PR
|
|
1779
1779
|
// whose latest review counts no survived and no unverified finding had
|
|
1780
1780
|
// nothing for a fix-applier to rule on, and one never reviewed has no
|
|
1781
1781
|
// review file: neither is gated on a verdict. Refused here, before anything
|
package/scripts/review-core.mjs
CHANGED
|
@@ -79,6 +79,11 @@ export const VERDICT_SCHEMA = {
|
|
|
79
79
|
refuted: { type: "boolean" },
|
|
80
80
|
reason: { type: "string" },
|
|
81
81
|
counter_evidence: { type: "string", description: "Command + output, if any." },
|
|
82
|
+
inconclusive: {
|
|
83
|
+
type: "boolean",
|
|
84
|
+
description:
|
|
85
|
+
"True only when your own check could produce neither a valid red nor a valid green: you abstain, and `reason` names the runs. Left out of the tally, never counted as a crash.",
|
|
86
|
+
},
|
|
82
87
|
},
|
|
83
88
|
};
|
|
84
89
|
|
|
@@ -133,6 +138,24 @@ export const TEST_RUN_SCHEMA = {
|
|
|
133
138
|
},
|
|
134
139
|
};
|
|
135
140
|
|
|
141
|
+
// --- Test verdict rules -------------------------------------------------
|
|
142
|
+
// The one statement of what a red and a kill are. Every site that scores a
|
|
143
|
+
// mutant killed, or files a failing test, carries these paragraphs word for
|
|
144
|
+
// word — the prompts below interpolate them, and the Markdown sites (the
|
|
145
|
+
// `tests` and verifier agent definitions, run-team's fix-applier and finisher
|
|
146
|
+
// briefs) quote them — so a reword lands everywhere or reds a pin. An exit
|
|
147
|
+
// status is never a kill on its own: a runner can exit 1 with nothing failing,
|
|
148
|
+
// and a load-sensitive test can fail once in a full suite and pass on rerun.
|
|
149
|
+
export const VALID_RED_RULE =
|
|
150
|
+
"**Valid red:** a run that completed — the command returned on its own, with no signal and no timeout, deadline or kill having fired — in which the test the claim names is reported failing. An exit ≠ 0 with no failing test is not red, and a different test failing is not red for this claim. A red in the full suite only triggers a narrowed run; it is never evidence on its own.";
|
|
151
|
+
export const NARROWED_RUN_RULE =
|
|
152
|
+
"**Narrowed run:** the Test entrypoint with arguments appended, `<testCmd> <args>`. That appended arguments reach the Test entrypoint is all that is promised about them, so a narrowed run counts only if it reports `tests` > 0 and its output names the claimed test. If no narrowing can be proven, run the full `<testCmd>` and read the named test's own result from its output.";
|
|
153
|
+
export const KILL_RULE =
|
|
154
|
+
"**Kill:** the mutant produced a valid red in a narrowed run, and the same narrowed command, in the same tree with the mutant reverted, ran a valid green with the named test passing — completed, exit 0, `tests` > 0, `cancelled` 0. A completed exit ≠ 0 with the named test not failing is not a kill, and no kill stands without that green run on the unmutated tree. The result is inconclusive, never a kill, when the red does not reproduce in the narrowed run, when the unmutated tree is red too, or when no usable run exists.";
|
|
155
|
+
export const EVIDENCE_LINE_RULE =
|
|
156
|
+
"**Evidence line:** every kill and every valid red you claim carries one line in the claim's own free-text field: `red: <cmd> → exit N, fail K incl <test>; baseline: <cmd> → exit 0, <test> pass`. A failing-test finding filed off the review's shared run carries the `red:` half alone.";
|
|
157
|
+
const KILL_RULES = [VALID_RED_RULE, NARROWED_RUN_RULE, KILL_RULE, EVIDENCE_LINE_RULE].join("\n\n");
|
|
158
|
+
|
|
136
159
|
// --- Dimension catalog --------------------------------------------------
|
|
137
160
|
// No dimension carries a `model`/`effort` field —
|
|
138
161
|
// dispatch tier lives ONLY in each fleet-owned agent definition's own
|
|
@@ -153,8 +176,11 @@ export const DEFAULT_DIMENSIONS = [
|
|
|
153
176
|
{
|
|
154
177
|
key: "tests",
|
|
155
178
|
agentType: "fleet-review-tests",
|
|
156
|
-
prompt:
|
|
157
|
-
|
|
179
|
+
prompt: `whether each test DISCRIMINATES: apply the mutation it should catch and confirm a kill, revert, then apply one it should NOT catch and confirm the named test stays green. Vary the syntactic form — a guard catching \`// whole-line\` may let \`code; // trailing\` through. \`<testCmd>\` is the \`command\` your Tests paragraph names, run only in your own copy of the snapshot. A kill is scored by these rules and nothing else:
|
|
180
|
+
|
|
181
|
+
${KILL_RULES}
|
|
182
|
+
|
|
183
|
+
An inconclusive result is no evidence either way: file no finding above \`suggestion\` on it, and name the gap — both runs — in \`scope_searched\`.`,
|
|
158
184
|
},
|
|
159
185
|
{
|
|
160
186
|
key: "comments",
|
|
@@ -525,7 +551,8 @@ export function sharedRunNote(run, owner, key) {
|
|
|
525
551
|
`Tests: this review ran its test command ONCE, from the snapshot's root, before
|
|
526
552
|
dispatching you. Do NOT run that full command yourself — every dimension reads
|
|
527
553
|
this one run. Targeted probes and mutations in your own copy of the snapshot are
|
|
528
|
-
still yours to run
|
|
554
|
+
still yours to run, and so is the full command in your own copy of the snapshot,
|
|
555
|
+
against your own mutant or to reproduce a failure, when no narrowed run can be proven.
|
|
529
556
|
command: ${run.command}
|
|
530
557
|
counts: ${countsOf(run) || "none — the run produced no counts"}
|
|
531
558
|
exit: ${run.exitCode ?? "(not reported)"}
|
|
@@ -541,10 +568,20 @@ unrun for it — do not run the full command yourself to replace it.`);
|
|
|
541
568
|
key === owner
|
|
542
569
|
? `It has ${failingDesc(run)}, and filing them is YOUR job in this review:
|
|
543
570
|
file at least one finding about them, naming each failing test from the log. A
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
571
|
+
failing test is filed off this run only on a valid red: rerun it first in a
|
|
572
|
+
narrowed run in your own copy of the snapshot, where \`<testCmd>\` is the
|
|
573
|
+
command above, and when that run does not show it failing, file no finding
|
|
574
|
+
above \`suggestion\` on it and name both runs. A failure you cannot separate
|
|
575
|
+
from the environment is still filed — say so in the finding, so it is
|
|
576
|
+
reproduced in the worktree before anyone acts on it. If none is filed, or this
|
|
577
|
+
review's refuters reject every finding filed, every dimension of this review
|
|
578
|
+
is reported unrun. A red is read by these rules and nothing else:
|
|
579
|
+
|
|
580
|
+
${VALID_RED_RULE}
|
|
581
|
+
|
|
582
|
+
${NARROWED_RUN_RULE}
|
|
583
|
+
|
|
584
|
+
${EVIDENCE_LINE_RULE}`
|
|
548
585
|
: `It has ${failingDesc(run)}, and the ${owner} dimension files the finding
|
|
549
586
|
about them. Do not file a duplicate — cite a failure as evidence only where it
|
|
550
587
|
bears on your own lens.`,
|
|
@@ -570,13 +607,19 @@ export function cwdAuditFrom(text) {
|
|
|
570
607
|
return m ? { state: m[1], line: m[0].trim() } : { state: "missing", line: null };
|
|
571
608
|
}
|
|
572
609
|
|
|
610
|
+
// An abstaining vote (`inconclusive: true`) is left out of the tally exactly
|
|
611
|
+
// as a crashed (`null`) one is, so a run that could not decide never breaks a
|
|
612
|
+
// tie toward `refuted`. It stays in `votes`, since its `reason` names the runs
|
|
613
|
+
// the fix-applier cites, and `refutersInconclusive` counts it — the one field
|
|
614
|
+
// that tells an all-abstained finding from an all-crashed one.
|
|
573
615
|
export function verdictFor(dispatched, votes) {
|
|
574
616
|
const live = votes.filter(Boolean);
|
|
575
|
-
const
|
|
617
|
+
const decided = live.filter((v) => v.inconclusive !== true);
|
|
618
|
+
const refuted = decided.filter((v) => v.refuted).length;
|
|
576
619
|
let verdict;
|
|
577
|
-
if (
|
|
578
|
-
else verdict = refuted * 2 >=
|
|
579
|
-
return { verdict, votes: live, refutersDispatched: dispatched };
|
|
620
|
+
if (decided.length === 0) verdict = "unverified";
|
|
621
|
+
else verdict = refuted * 2 >= decided.length ? "refuted" : "survived";
|
|
622
|
+
return { verdict, votes: live, refutersDispatched: dispatched, refutersInconclusive: live.length - decided.length };
|
|
580
623
|
}
|
|
581
624
|
|
|
582
625
|
// One in-run re-dispatch for a crashed dispatch, before the result is
|
|
@@ -597,12 +640,14 @@ export function retryCrashed(dispatch, crashed) {
|
|
|
597
640
|
// omp has no cached-replay mechanism: a fresh
|
|
598
641
|
// review re-dispatches every agent() live rather than only the crashed
|
|
599
642
|
// legs, and the one re-dispatch this run gets was already spent in-run
|
|
600
|
-
// — a crashed finding is reported and deferred, never resumed.
|
|
643
|
+
// — a crashed finding is reported and deferred, never resumed. Crashed
|
|
644
|
+
// means a `null` refuter: an `unverified` finding whose dispatched refuters
|
|
645
|
+
// all abstained is no crash, and stays out of this bucket.
|
|
601
646
|
export function resumeFor(unverified) {
|
|
602
|
-
const crashed = unverified.filter((f) => f.refutersDispatched > 0);
|
|
647
|
+
const crashed = unverified.filter((f) => f.refutersDispatched > (f.refutersInconclusive ?? 0));
|
|
603
648
|
if (!crashed.length) return { crashed, resume: null };
|
|
604
649
|
const claim =
|
|
605
|
-
"Findings in `unverified` with `refutersDispatched` above
|
|
650
|
+
"Findings in `unverified` with `refutersDispatched` above `refutersInconclusive` lost a refuter to a crash — a pair whose every vote died was re-dispatched once in-run and died again — and no refuter that ran decided them. ";
|
|
606
651
|
const verb =
|
|
607
652
|
"Defer them as crashed — reported, not acted on: omp's eval has no cached-replay mechanism, so a fresh review re-dispatches every agent() live rather than only the crashed legs, and the in-run retry was this review's one re-dispatch. Re-run nothing for them.";
|
|
608
653
|
return { crashed, resume: claim + verb };
|
|
@@ -1051,15 +1096,17 @@ to name.`,
|
|
|
1051
1096
|
(review && review.findings ? review.findings : []).map((f, fi) => () => {
|
|
1052
1097
|
const n = verifiersForRun(f.severity);
|
|
1053
1098
|
if (n === 0) return Promise.resolve({ ...f, dimension: d.key, ...verdictFor(0, []) });
|
|
1054
|
-
// A pair
|
|
1055
|
-
// re-dispatched once, as a pair, before
|
|
1056
|
-
//
|
|
1099
|
+
// A pair in which no refuter decided and at least one died (or whose
|
|
1100
|
+
// dispatch threw) is re-dispatched once, as a pair, before
|
|
1101
|
+
// `verdictFor` reads it — one deciding vote means the pair did not
|
|
1102
|
+
// crash and is ruled on that vote, and a pair that ALL abstained ran
|
|
1103
|
+
// and could not decide, so a second dispatch would only repeat it.
|
|
1057
1104
|
return retryCrashed(
|
|
1058
1105
|
() =>
|
|
1059
1106
|
parallel(
|
|
1060
1107
|
Array.from({ length: n }, (_, i) => () =>
|
|
1061
1108
|
agent(
|
|
1062
|
-
`Try to REFUTE this finding from PR #${pr}. Default to refuted=true if uncertain.
|
|
1109
|
+
`Try to REFUTE this finding from PR #${pr}. Default to refuted=true if uncertain in your reasoning — a run that could not decide is not that uncertainty, and the abstain rule below covers it.
|
|
1063
1110
|
|
|
1064
1111
|
claim: ${f.claim}
|
|
1065
1112
|
where: ${f.file || "?"}:${f.line || "?"}
|
|
@@ -1082,6 +1129,17 @@ search scope AND what your pattern would have missed. A grep over one ref
|
|
|
1082
1129
|
does not support a claim about history; a pattern built from the token a diff
|
|
1083
1130
|
removed does not support a claim that the category is empty.
|
|
1084
1131
|
|
|
1132
|
+
When your check scores a mutant killed or a test red, \`<testCmd>\` is
|
|
1133
|
+
\`${testCmd}\`, run only in your own copy of the snapshot, and these rules
|
|
1134
|
+
decide it and nothing else:
|
|
1135
|
+
|
|
1136
|
+
${KILL_RULES}
|
|
1137
|
+
|
|
1138
|
+
Abstain when your own check could produce neither a valid red nor a valid
|
|
1139
|
+
green: set \`inconclusive\` to true and name the runs in \`reason\`, each with
|
|
1140
|
+
its evidence line. \`refuted\` is still required, and the tally ignores it on
|
|
1141
|
+
an abstaining vote — an undecided run never defaults to refuted=true.
|
|
1142
|
+
|
|
1085
1143
|
Chain the directory change into the command, \`cd "$D" && git …\`, never
|
|
1086
1144
|
\`cd "$D"; git …\`, so a failed \`cd\` cannot leave a \`git\` command running in the
|
|
1087
1145
|
checkout — and bracket a fixture's own git with \`git rev-parse --show-toplevel\`:
|
|
@@ -1133,7 +1191,11 @@ how three reviews from one cell left four files modified in that checkout.`,
|
|
|
1133
1191
|
),
|
|
1134
1192
|
),
|
|
1135
1193
|
),
|
|
1136
|
-
|
|
1194
|
+
// Retry unless a refuter decided or every one abstained.
|
|
1195
|
+
(votes) =>
|
|
1196
|
+
!votes ||
|
|
1197
|
+
!votes.length ||
|
|
1198
|
+
(!votes.some((v) => v && v.inconclusive !== true) && !votes.every((v) => v && v.inconclusive === true)),
|
|
1137
1199
|
).then(
|
|
1138
1200
|
(votes) => ({ ...f, dimension: d.key, ...verdictFor(n, votes) }),
|
|
1139
1201
|
// A rejection here means retryCrashed's OWN final attempt
|
|
@@ -1183,8 +1245,9 @@ how three reviews from one cell left four files modified in that checkout.`,
|
|
|
1183
1245
|
// Refuted findings are RETURNED, not dropped. A refutation is itself a claim,
|
|
1184
1246
|
// and the controller has reversed a refutation on new evidence before.
|
|
1185
1247
|
// `unverified` are findings the adversarial pass did not settle — a suggestion
|
|
1186
|
-
// that skipped it by policy, or one whose refuters
|
|
1187
|
-
// separately so the caller never mistakes "not
|
|
1248
|
+
// that skipped it by policy, or one whose refuters crashed or abstained and
|
|
1249
|
+
// decided nothing — surfaced separately so the caller never mistakes "not
|
|
1250
|
+
// checked" for "survived".
|
|
1188
1251
|
// `dimensionsRun` names what was DISPATCHED after the size trim: a trimmed
|
|
1189
1252
|
// fan-out must say so, never read as full coverage. It is not a coverage claim
|
|
1190
1253
|
// on its own and never was — a specialist can be dispatched and die, or the
|
package/scripts/review-eval.mjs
CHANGED
|
@@ -30,8 +30,8 @@
|
|
|
30
30
|
// runReview's own snapshot/verifier dispatch, unmodified — omp's `agent()`
|
|
31
31
|
// resolves a bare frontmatter `name:` exactly, which is already what those
|
|
32
32
|
// strings are.
|
|
33
|
-
import { mkdir, writeFile } from "node:fs/promises";
|
|
34
|
-
import { isAbsolute, join } from "node:path";
|
|
33
|
+
import { mkdir, unlink, writeFile } from "node:fs/promises";
|
|
34
|
+
import { basename, dirname, isAbsolute, join, normalize } from "node:path";
|
|
35
35
|
import { digestOf, runReview, runnerPrRefusal, runnerScratchRefusal } from "./review-core.mjs";
|
|
36
36
|
|
|
37
37
|
// eval's `agent()` returns a HANDLE, not data: `agent(prompt,
|
|
@@ -139,10 +139,17 @@ export async function runReviewOnOmp(args) {
|
|
|
139
139
|
|
|
140
140
|
// The whole of the omp review runner's job, so the agent's own cell is three
|
|
141
141
|
// lines and this contract can be run by a test: the full result
|
|
142
|
-
// object goes to `<
|
|
142
|
+
// object goes to `<run root>/review.json` — the one artefact handed to
|
|
143
143
|
// the fix-applier — and only the digest comes back, because the digest is
|
|
144
144
|
// all the controller reads and a 26–61 KB result is what the file exists to
|
|
145
|
-
// keep out of its context.
|
|
145
|
+
// keep out of its context. The run root is the review's own
|
|
146
|
+
// `<scratch>/pr<pr>/run-XXXXXXXX`, the directory review-core.mjs's snapshot
|
|
147
|
+
// step made with `mktemp` and the snapshot it returns sits in, so two reviews
|
|
148
|
+
// of one PR — at one head or two — never write one file, and the ledger token
|
|
149
|
+
// names the run whose file it is. No file is ever replaced. The file leaves
|
|
150
|
+
// with its run root: the snapshot step of a later review of the same PR
|
|
151
|
+
// removes that PR's run roots not touched for seven days, so a token older
|
|
152
|
+
// than that names a file that is gone.
|
|
146
153
|
//
|
|
147
154
|
// Failure is a throw or an empty return. The holder of the review call retries
|
|
148
155
|
// once; a second failure returns `failed` with both errors and the fallback
|
|
@@ -151,8 +158,9 @@ export async function runReviewOnOmp(args) {
|
|
|
151
158
|
// scratch that is not absolute (eval's cwd is the MAIN CHECKOUT, so a relative
|
|
152
159
|
// one would put the file there), a scratch that already ends in `pr<N>`
|
|
153
160
|
// — throws before any run: it is not a review failure, and retrying or falling
|
|
154
|
-
// back would only repeat it.
|
|
155
|
-
//
|
|
161
|
+
// back would only repeat it. So does a write fault other than a file already
|
|
162
|
+
// at the path: the review ran, the environment could not keep it. `run` is the
|
|
163
|
+
// seam the test injects; nothing else passes it.
|
|
156
164
|
export async function runReviewToFile(args, run = runReviewOnOmp) {
|
|
157
165
|
const pr = args?.pr;
|
|
158
166
|
const scratch = args?.scratch;
|
|
@@ -177,7 +185,7 @@ export async function runReviewToFile(args, run = runReviewOnOmp) {
|
|
|
177
185
|
// the file on disk — so a `run` that resolves to a truthy, object-shaped
|
|
178
186
|
// result missing `.counts` would otherwise throw on the destructure below
|
|
179
187
|
// OUTSIDE this function's own retry try/catch (which wraps only the
|
|
180
|
-
// `run(args)` call), leaving a half-written
|
|
188
|
+
// `run(args)` call), leaving a half-written review file on disk
|
|
181
189
|
// that a fix-applier reading it would mistake for a completed review that
|
|
182
190
|
// never actually finished. Treated the same as an empty return: retried,
|
|
183
191
|
// then reported `failed`, and no file is written for it.
|
|
@@ -185,15 +193,44 @@ export async function runReviewToFile(args, run = runReviewOnOmp) {
|
|
|
185
193
|
errors.push(`attempt ${attempt}: empty return (${String(result)})`);
|
|
186
194
|
continue;
|
|
187
195
|
}
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
196
|
+
// The run root is read off the result, never rebuilt: only the snapshot
|
|
197
|
+
// step's shell knew the `mktemp` name. A result whose snapshot is not in
|
|
198
|
+
// a run root of this PR under this scratch names no run the ledger token
|
|
199
|
+
// could point a reader back to, so it is a failed attempt like the one
|
|
200
|
+
// above.
|
|
201
|
+
const runRoot = runRootOf(result.snapshot, scratch, pr);
|
|
202
|
+
if (runRoot === null) {
|
|
203
|
+
errors.push(`attempt ${attempt}: the result's snapshot ${JSON.stringify(result.snapshot)} is not in a run root `
|
|
204
|
+
+ `${join(scratch, `pr${pr}`, "run-XXXXXXXX")} — no run the review file could be kept in`);
|
|
205
|
+
continue;
|
|
206
|
+
}
|
|
207
|
+
const path = join(runRoot, "review.json");
|
|
208
|
+
try {
|
|
209
|
+
await mkdir(runRoot, { recursive: true });
|
|
210
|
+
// `wx`: an existing file at this path is another review's result, and
|
|
211
|
+
// stays.
|
|
212
|
+
await writeFile(path, `${JSON.stringify(result, null, 2)}\n`, { flag: "wx" });
|
|
213
|
+
} catch (e) {
|
|
214
|
+
// Only that collision is a failed attempt: a second review run lands in
|
|
215
|
+
// a root of its own. Any other fault — permissions, a full disk — is
|
|
216
|
+
// the environment's, so it throws rather than re-run a full review that
|
|
217
|
+
// would hit it again and report it as a review failure. A write that
|
|
218
|
+
// fails partway has left its truncated bytes at the path: removed
|
|
219
|
+
// before the throw, never on a collision, where the file is another
|
|
220
|
+
// review's.
|
|
221
|
+
if (e?.code !== "EEXIST") {
|
|
222
|
+
await unlink(path).catch(() => {});
|
|
223
|
+
throw e;
|
|
224
|
+
}
|
|
225
|
+
errors.push(`attempt ${attempt}: could not write ${path}: ${e?.message ?? String(e)}`);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
191
228
|
const { survived, refuted, unverified } = result.counts;
|
|
192
229
|
return {
|
|
193
230
|
status: "completed",
|
|
194
231
|
path,
|
|
195
232
|
// The review's result token, ready for `ledger.mjs row`.
|
|
196
|
-
ledger: `reviewed=${result.head}:${survived}/${refuted}/${unverified}`,
|
|
233
|
+
ledger: `reviewed=${result.head}:${survived}/${refuted}/${unverified}:${basename(runRoot)}`,
|
|
197
234
|
attempts: attempt,
|
|
198
235
|
errors,
|
|
199
236
|
digest: digestOf(result),
|
|
@@ -201,3 +238,14 @@ export async function runReviewToFile(args, run = runReviewOnOmp) {
|
|
|
201
238
|
}
|
|
202
239
|
return { status: "failed", errors, fallback: `review-pr-${pr}-b` };
|
|
203
240
|
}
|
|
241
|
+
|
|
242
|
+
// The run root a result's `snapshot` sits in — `<scratch>/pr<pr>/run-` and
|
|
243
|
+
// the eight characters `mktemp` put in place of `XXXXXXXX` — or null when it
|
|
244
|
+
// sits anywhere else. A `..` segment is refused before normalising, which
|
|
245
|
+
// would otherwise resolve it into a root the path never named.
|
|
246
|
+
function runRootOf(snapshot, scratch, pr) {
|
|
247
|
+
if (typeof snapshot !== "string" || !isAbsolute(snapshot) || snapshot.split("/").includes("..")) return null;
|
|
248
|
+
const runRoot = dirname(normalize(snapshot));
|
|
249
|
+
if (dirname(runRoot) !== join(scratch, `pr${pr}`) || !/^run-[A-Za-z0-9]{8}$/.test(basename(runRoot))) return null;
|
|
250
|
+
return runRoot;
|
|
251
|
+
}
|