@arjunkhera/atlas 0.3.13 → 0.3.14

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "atlas",
-  "version": "0.3.13",
+  "version": "0.3.14",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, for any repository.",
   "author": {
     "name": "Arjun Khera"

package/door/cli.mjs

@@ -28,6 +28,7 @@ import { writeDesignPage, readTracker } from './lib/design-build.mjs';
 import { loadPrivateTerms } from './lib/privacy.mjs';
 import { checkCommand as testsCheck, writeCommand as testsWrite } from './lib/tests.mjs';
 import { proofCommand as testsProof } from './lib/proof.mjs';
+import { verdictCommand as testsVerdict, namedCommand as testsNamed } from '../tests/verdict.mjs';
 
 export const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
 const ATLAS_VERSION = packagePackageVersion(PACKAGE_ROOT);
@@ -63,6 +64,14 @@ const HELP = `atlas — the door into a repo's Atlas files
   atlas tests proof --root <area>      print the proof table of one run, one row for each assertion
       --run <id>                       the run; default the newest run in the evidence folder
       --own <file>                     JSON of the verifier's own checks: { id: { result, proof } }
+      --base <ref>                     show whether a flaky assertion is named by the pull request
+  atlas tests verdict --root <area>    decide each scenario of a run from both attempts: pass, fail or flaky; exit 1 on a fail
+      --run <id>                       the run; default the newest run in the evidence folder
+      --base <ref>                     the base of the pull request; flaky is a fail for an assertion it names. With no base every flaky is a fail
+      --covers                         also name the assertions of a scenario whose covers path the change touches
+  atlas tests named --root <area>      list the assertions that a pull request names
+      --base <ref>                     the base: an assertion is named when its words changed or it is new
+      --covers                         also name the assertions of a scenario whose covers path the change touches
 
   atlas install                        install Atlas for every folder on this Mac
   atlas upgrade [--version <v>]        check, show and install a newer release
@@ -89,7 +98,7 @@ export const FLAGS = Object.freeze({
   check: { root: 'optional', repo: 'value', 'fail-on': 'value' },
   ste: { share: 'switch', terms: 'value' },
   design: { tracker: 'value', out: 'value', draft: 'switch' },
-  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value' },
+  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch' },
   install: { local: 'switch', from: 'value', 'skip-global': 'switch', yes: 'switch' },
   upgrade: { version: 'optional', yes: 'switch' },
   doctor: {},
@@ -265,8 +274,10 @@ function testsCommand(chosen) {
     return testsCheck({ halves: chosen.halves, evidence: chosen.evidence, tests: chosen.tests, guardsFrom: chosen['guards-from'], run: chosen.run }, root, line, PACKAGE_ROOT);
   }
   if (what === 'write') return testsWrite({ dryRun: Boolean(chosen['dry-run']) }, root, PACKAGE_ROOT, line);
-  if (what === 'proof') return testsProof({ run: chosen.run, own: chosen.own, evidence: chosen.evidence }, root, line);
-  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write or proof.`);
+  if (what === 'proof') return testsProof({ run: chosen.run, own: chosen.own, evidence: chosen.evidence, base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
+  if (what === 'verdict') return testsVerdict({ run: chosen.run, base: chosen.base, evidence: chosen.evidence, covers: Boolean(chosen.covers) }, root, line);
+  if (what === 'named') return testsNamed({ base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
+  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict or named.`);
 }
 
 function checkCommand(chosen) {

package/door/kit-releases.json

@@ -4,6 +4,10 @@
     {
       "kit_sha256": "ae0f32b8c6694fbffb8bdacd8bd919d27c1c1da6281a46779e521dfea2341565",
       "first_package": "0.3.8"
+    },
+    {
+      "kit_sha256": "b2ddd9412d42f92c090432c0782aec498120395b2f8c7271006349c94b26c772",
+      "first_package": "0.3.13"
     }
   ]
 }

package/door/lib/proof.mjs

@@ -14,8 +14,9 @@
 import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { newestRunPerScenario } from '../../tests/link-check.mjs';
+import { settleRecords, listNamed, keyOf } from '../../tests/verdict.mjs';
 
-const MAX_CELL = 110;
+const MAX_CELL = 200;
 
 const clean = (text) => String(text ?? '').replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim();
 const clip = (text, n = MAX_CELL) => { const t = clean(text); return t.length > n ? `${t.slice(0, n - 3)}...` : t; };
@@ -26,6 +27,7 @@ function summary(value) {
   return typeof value === 'string' ? value : JSON.stringify(value);
 }
 
+const attemptNumber = (name) => { const m = /\.(\d+)\.json$/.exec(name); return m ? Number(m[1]) : 1; };
 const usage = (message) => Object.assign(new Error(message), { usage: true });
 
 // Reads the --own file. Throws an Error with a plain message on a bad file.
@@ -39,11 +41,14 @@ export function readOwn(file) {
   return data;
 }
 
-export function buildProof({ evidenceDir, runId = null, own = {} }) {
+// `named` is a Set of "scenario/assertion" keys (from listNamed), or null when the proof gets no base.
+export function buildProof({ evidenceDir, runId = null, own = {}, named = null }) {
   const perScenario = runId ? null : newestRunPerScenario(evidenceDir);
   if (!runId && !perScenario.size) throw new Error(`there is no run in ${evidenceDir}`);
   const names = runId ? [runId] : [...new Set(perScenario.values())].sort();
   const newest = new Map();
+  const attempts = new Map();
+  const attemptProblems = [];
   const used = new Set();
   let faultRuns = 0;
   for (const id of names) {
@@ -57,16 +62,26 @@ export function buildProof({ evidenceDir, runId = null, own = {} }) {
       if (data.fault_run) { faultRuns += 1; continue; }
       if (perScenario && perScenario.get(data.scenario) !== id) continue;
       used.add(id);
-      const before = newest.get(data.scenario);
-      if (!before || String(data.started) >= String(before.started)) newest.set(data.scenario, data);
+      const key = `${id}\u0000${data.scenario}`;
+      if (!attempts.has(key)) attempts.set(key, []);
+      attempts.get(key).push({ name, data });
     }
   }
+  // Several attempts of one scenario in one run (<scenario>.json, <scenario>.2.json) are joined:
+  // an assertion that failed, then passed, reads FLAKY. Of two runs of a scenario, the newest counts.
+  for (const list of attempts.values()) {
+    list.sort((a, b) => (attemptNumber(a.name) - attemptNumber(b.name)));
+    const { record: settled, problems } = settleRecords(list.map((x) => x.data));
+    for (const p of problems) attemptProblems.push(`${settled.scenario}: ${p}`);
+    const before = newest.get(settled.scenario);
+    if (!before || String(settled.started) >= String(before.started)) newest.set(settled.scenario, settled);
+  }
   const id = runId ?? [...used].sort().join(', ');
   if (!newest.size) throw new Error(`the run "${id}" has no evidence file that counts${faultRuns ? ' (only fault runs)' : ''}`);
   const scenarios = [...newest.values()].sort((a, b) => (a.scenario < b.scenario ? -1 : 1));
   const ways = [...new Set(scenarios.flatMap((s) => Object.keys(s.ways ?? {})))];
   const rows = [];
-  const blocked = [];
+  const blocked = [...attemptProblems];
   const screenshots = [];
   const pageEvents = [];
   for (const s of scenarios) {
@@ -92,7 +107,10 @@ export function buildProof({ evidenceDir, runId = null, own = {} }) {
       }));
       const problems = entries.filter((e) => e.result !== 'pass' && e.result !== 'not here').map((e) => {
         const why = e.proof?.error ?? e.proof?.why ?? s.ways?.[e.way]?.reason ?? '';
-        return `${e.way} ${e.result}${why ? `: ${why}` : ''}`;
+        const note = e.result !== 'flaky' ? ''
+          : named === null ? ' (FLAKY counts as a fail unless the pull request does not name it)'
+            : named.has(keyOf(full)) ? ' (FLAKY, named, so a fail)' : ' (FLAKY, shown, not a fail)';
+        return `${e.way} ${e.result}${why ? `: ${why}` : ''}${note}`;
       });
       const passed = entries.find((e) => e.result === 'pass' && e.proof?.value !== undefined);
       const mine = own[full] ?? own[short] ?? null;
@@ -141,8 +159,9 @@ export function renderProof(proof) {
 export function proofCommand(chosen, root, line) {
   try {
     const own = chosen.own ? readOwn(resolve(chosen.own)) : {};
+    const named = chosen.base ? new Set(listNamed({ root, base: chosen.base, covers: Boolean(chosen.covers) }).named.map((n) => n.key)) : null;
     const evidenceDir = resolve(chosen.evidence ?? join(root, 'test', 'evidence'));
-    for (const text of renderProof(buildProof({ evidenceDir, runId: chosen.run ?? null, own }))) line(text);
+    for (const text of renderProof(buildProof({ evidenceDir, runId: chosen.run ?? null, own, named }))) line(text);
     return 0;
   } catch (error) {
     line(`proof: ${error.message}`);

package/door/lib/tests.mjs

@@ -3,6 +3,8 @@
 //   atlas tests check --root <area>   run the link check on an area
 //   atlas tests write --root <area>   copy the shipped test kit into <area>/test/atlas/
 //   atlas tests proof --root <area>   print the proof table of one run (proof.mjs)
+//   atlas tests verdict --root <area> --run <id> [--base <ref>]   decide each scenario: pass, fail or flaky (tests/verdict.mjs)
+//   atlas tests named --root <area> --base <ref>   list the assertions that a pull request names (tests/verdict.mjs)
 //
 // The kit is the folder `tests/` of this package: the contract library, the
 // drivers, the stand-in helper and the link check. `write` copies each file

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arjunkhera/atlas",
-  "version": "0.3.13",
+  "version": "0.3.14",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, as a Claude Code plugin for any repository.",
   "type": "module",
   "license": "UNLICENSED",

package/skills/tests/SKILL.md

@@ -41,6 +41,27 @@ every file that command writes.
 2. `atlas tests check --root <repo>` runs the link check. It prints one line
    for each finding, as `file:line  code  words`. Run it before you ask for a merge.
 3. `atlas tests proof --root <repo>` prints the proof table of one run.
+4. `atlas tests verdict --root <repo> --run <id> --base <ref>` decides each
+   scenario of a run. Exit 1 means a scenario failed.
+5. `atlas tests named --root <repo> --base <ref>` lists the assertions that a
+   pull request names.
+
+When a scenario fails in CI, CI runs the whole scenario once more on the
+same commit. The second run writes `<scenario>.2.json` in the same run folder.
+Never retry inside a test. Then CI runs `atlas tests verdict`:
+
+1. Both runs fail: the result is `fail`. More than two runs, or two runs on
+   different commits, are also a `fail`.
+2. The first run fails and the second passes: each assertion that failed
+   reads `flaky`. The proof table shows "failed, then passed, on commit
+   <sha>".
+3. A pull request names an assertion when its words changed or it is new.
+   `flaky` is a `fail` for a named assertion. For any other assertion it is
+   shown, and it is not a fail. The flag `--covers` also names the assertions
+   of a scenario whose `covers` path the change touches. It is off until the
+   owner decides.
+4. The rule to open an item for each `flaky` result waits on the owner's
+   answer. Do not open one yet.
 
 The link check has three halves. Run the first two before the tests and the
 third after them:

package/tests/environment.mjs

@@ -23,7 +23,7 @@ import { parseLimit } from './wait.mjs';
 
 const LIBRARY_HEADERS = { [LIBRARY_HEADER]: '1' };
 
-function freePort() {
+function onePort() {
   return new Promise((resolve, reject) => {
     const server = createServer();
     server.once('error', reject);
@@ -31,6 +31,21 @@ function freePort() {
   });
 }
 
+// A free port that no stand-in of this run uses, and that the environment
+// does not name. A product with a fixed port has not started yet when a
+// stand-in takes its port, and a stand-in can listen on another address family.
+export async function freePort(taken = []) {
+  for (let tries = 0; tries < 20; tries += 1) {
+    const port = await onePort();
+    if (!taken.includes(port)) return port;
+  }
+  throw new Blocked('no free port was found that differs from the ports the run already uses');
+}
+
+const portsOf = (standIns) => Object.values(standIns).map((one) => Number(new URL(one.url).port)).filter(Boolean);
+export const namedPorts = (spec) => [...JSON.stringify(spec ?? {}).matchAll(/(?:127\.0\.0\.1|localhost|\[::1\]):(\d{1,5})/g)].map((m) => Number(m[1]))
+  .concat(Object.values(spec?.vars ?? {}).filter((v) => /^\d{2,5}$/.test(String(v))).map(Number));
+
 // ---------------------------------------------------------------- ready
 
 async function pollReady(spec, ctx, { label, process: proc, hosts, redactor, standInOrigins = [] }) {
@@ -144,7 +159,7 @@ export async function startEnvironment({ root, tests, name, runId, redactor, ada
         continue;
       }
       if (!def?.start) throw new Blocked(`${label} has no "start" command and the adapter does not make it`);
-      const port = await freePort();
+      const port = await freePort([...namedPorts(spec), ...portsOf(standIns)]);
       const url = `http://127.0.0.1:${port}`;
       const ctx = { id: runId, port, url };
       const proc = spawnManaged({
@@ -162,7 +177,7 @@ export async function startEnvironment({ root, tests, name, runId, redactor, ada
     // 2. The vars: run secrets are new for each placeholder; references resolve in order.
     const raw = spec.vars ?? {};
     // The product gets a free port too: `{port}` in start, ready, url and vars.
-    const productPort = await freePort();
+    const productPort = await freePort([...namedPorts(spec), ...portsOf(standIns)]);
     const ctx = {
       id: runId, varsFile, port: productPort, url: spec.url ? substitute(spec.url, { id: runId, port: productPort }) : undefined,
       secret: () => { const s = newSecret(); secretsMade.push(s); redactor.add(s); return s; },

package/tests/evidence.mjs

@@ -4,7 +4,7 @@
 //     "evidence": 1, "run_id": "...", "environment": "local",
 //     "scenario": "week-conflict", "title": "...", "source": "scenarios/week-conflict.md",
 //     "source_sha256": "...", "fingerprint_method": "fp1",
-//     "started": "...", "ended": null, "verdict": "not finished",
+//     "commit": "<sha or null>", "started": "...", "ended": null, "verdict": "not finished",
 //     "ways": { "mcp": { "verdict": "pass", "reason": null, "started": "...", "ended": "..." } },
 //     "assertions": [ { "id": "week-conflict/e3#09f598", "way": "mcp", "result": "pass", "proof": { ... } } ],
 //     "tables": { "T1": "<fingerprint>" }, "fresh": [ { "rule": "...", "value": "..." } ],
@@ -18,13 +18,32 @@
 // (`<scenario>.2.json`). A file is claimed with an exclusive create, so no
 // earlier run is overwritten. Every string is redacted by value before it is
 // written.
-import { closeSync, mkdirSync, openSync, renameSync, writeFileSync } from 'node:fs';
+import { closeSync, mkdirSync, openSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
 import { join, relative } from 'node:path';
+import { spawnSync } from 'node:child_process';
 
 export const EVIDENCE_SCHEMA = 1;
 
-// The results of one assertion for one way in (design section 6.11).
-export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here']);
+// The commit under test: the head of the pull request when the environment gives it (GITHUB_HEAD_SHA, or
+// pull_request.head.sha in the event file), else GITHUB_SHA, else the HEAD of the repo, when git can say.
+function commitOf(root) {
+  if (process.env.GITHUB_HEAD_SHA) return process.env.GITHUB_HEAD_SHA;
+  if (process.env.GITHUB_EVENT_PATH) {
+    try {
+      const sha = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8'))?.pull_request?.head?.sha;
+      if (sha) return sha;
+    } catch { /* no event file that reads */ }
+  }
+  if (process.env.GITHUB_SHA) return process.env.GITHUB_SHA;
+  try {
+    const r = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' });
+    return r.status === 0 ? r.stdout.trim() : null;
+  } catch { return null; }
+}
+
+// The results of one assertion for one way in (design section 6.11). `flaky` is never set by a
+// test. `atlas tests verdict` and the proof table read it from two attempts (verdict.mjs).
+export const RESULTS = Object.freeze(['pass', 'fail', 'not checked', 'not exercised', 'blocked', 'not here', 'flaky']);
 
 const MAX_BODY = 8000;
 
@@ -47,7 +66,7 @@ export class Evidence {
       scenario: scenario.id, title: scenario.title,
       source: relative(root, scenario.file).split('\\').join('/'), source_sha256: scenario.sha256,
       fingerprint_method: scenario.method,
-      started: new Date().toISOString(), ended: null, verdict: 'not finished',
+      commit: commitOf(root), started: new Date().toISOString(), ended: null, verdict: 'not finished',
       ways: {}, assertions: [], tables: {}, fresh: [], calls: [], notes: [], files: [],
       provided_secrets: providedSecrets, fault_run: process.env.ATLAS_FAULT_RUN || null,
     };

package/tests/index.mjs

@@ -9,3 +9,4 @@ export { Blocked, Unavailable, WaitFailed, WaitTimeout, GateFailed } from './err
 export { Redactor } from './redact.mjs';
 export { Evidence, EVIDENCE_SCHEMA, RESULTS } from './evidence.mjs';
 export { checkArea } from './link-check.mjs';
+export { listNamed, decide, settleRecords, readAttempts, loadAreaScenarios, keyOf } from './verdict.mjs';

package/tests/link-check.mjs

@@ -466,6 +466,25 @@ export function newestRunPerScenario(evidenceDir) {
   return new Map([...best].map(([scenario, b]) => [scenario, b.name]));
 }
 
+// The newest run of the whole folder, by the start time of its evidence (a name breaks a tie).
+export function newestRunId(evidenceDir) {
+  let best = null;
+  if (!existsSync(evidenceDir)) return null;
+  for (const name of readdirSync(evidenceDir)) {
+    const dir = join(evidenceDir, name);
+    if (!statSync(dir).isDirectory()) continue;
+    for (const file of readdirSync(dir)) {
+      if (!file.endsWith('.json')) continue;
+      let data;
+      try { data = JSON.parse(readFileSync(join(dir, file), 'utf8')); } catch { continue; }
+      if (data?.evidence !== 1 || data.fault_run || !data.scenario) continue;
+      const started = String(data.started);
+      if (!best || started > best.started || (started === best.started && name > best.name)) best = { started, name };
+    }
+  }
+  return best ? best.name : null;
+}
+
 // The run half reads the files of one run only, and only evidence of the
 // scenario file as it is now (source_sha256).
 export function checkRun({ evidenceDir, runId = null, environment = null, scenarios, rel, findings }) {
@@ -488,8 +507,8 @@ export function checkRun({ evidenceDir, runId = null, environment = null, scenar
       runOf.set(data.scenario, id);
     }
   }
-  const reached = new Set(['pass', 'fail', 'not exercised', 'not here']);
-  const ran = new Set(['pass', 'fail', 'not exercised']);
+  const reached = new Set(['pass', 'fail', 'not exercised', 'not here', 'flaky']);
+  const ran = new Set(['pass', 'fail', 'not exercised', 'flaky']);
   for (const s of scenarios) {
     if (!s.id) continue;
     const all = records.filter((r) => r.data.scenario === s.id);

package/tests/verdict.mjs

@@ -0,0 +1,300 @@
+// The second attempt, the named assertions and the verdict (design sections 6.11, 11.2).
+//
+// When a scenario fails in CI, CI runs the WHOLE scenario once more on the same
+// commit. The library keeps that run as `<scenario>.2.json` in the same run folder
+// (evidence.mjs). This file reads both attempts and decides, for each assertion:
+//
+//   pass     the last attempt passed, and the first did not fail
+//   fail     the last attempt did not pass (fail, blocked, not checked, not exercised, a way that did not end well)
+//   flaky    the first attempt failed (or was blocked or not checked), and the last attempt passed
+//
+// `flaky` is a fail for an assertion that the pull request names (listNamed). Else it is
+// shown, and it is not a fail. A retry inside a test stays refused by the link check.
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, relative, resolve } from 'node:path';
+import { parseScenario, addFingerprints } from './scenario.mjs';
+import { parseYaml } from './yaml.mjs';
+import { checkRun, newestRunId } from './link-check.mjs';
+
+// Design 6.11: `not exercised` is a fail unless a person names it with a reason. No code holds that
+// naming yet, so it is a fail here.
+const GOOD = new Set(['pass', 'not here']);
+// A first result that a second pass turns into `flaky`.
+const FAILED_FIRST = new Set(['fail', 'blocked', 'not checked']);
+const WAY_OK = new Set(['pass', 'not here']);
+
+// ---------------------------------------------------------------- the attempts of a run
+
+// The attempt number of an evidence file name: `x.json` is 1, `x.2.json` is 2.
+const attemptOf = (name) => { const m = /\.(\d+)\.json$/.exec(name); return m ? Number(m[1]) : 1; };
+
+// Reads the evidence of ONE run. Returns a Map of scenario id to its records in attempt
+// order. A record of a fault run is left out.
+export function readAttempts(evidenceDir, runId) {
+  const dir = join(evidenceDir, runId);
+  if (!existsSync(dir)) throw new Error(`the run "${runId}" has no folder in ${evidenceDir}`);
+  const out = new Map();
+  for (const name of readdirSync(dir)) {
+    if (!name.endsWith('.json')) continue;
+    let data;
+    try { data = JSON.parse(readFileSync(join(dir, name), 'utf8')); } catch { continue; }
+    if (data?.evidence !== 1 || data.fault_run || !data.scenario) continue;
+    if (!out.has(data.scenario)) out.set(data.scenario, []);
+    out.get(data.scenario).push({ attempt: attemptOf(name), data });
+  }
+  for (const list of out.values()) list.sort((a, b) => a.attempt - b.attempt);
+  return new Map([...out].map(([id, list]) => [id, list.map((x) => x.data)]));
+}
+
+// The newest run of the evidence folder, by time, as the link check picks it.
+export const newestRun = newestRunId;
+
+const short = (sha) => (sha ? String(sha).slice(0, 7) : '(not recorded)');
+
+// Joins the attempts of ONE scenario. Returns { record, problems }. The record is the last attempt
+// with these changes:
+//   - an assertion that failed (or was blocked or not checked) in attempt 1 and passed in the last
+//     attempt reads `flaky`; its proof says "failed, then passed, on commit <sha>";
+//   - every assertion of attempt 1 is carried over; one that attempt 2 lacks reads `fail`;
+//   - a way that failed, then passed, has the verdict `flaky`; a way that attempt 2 lacks reads `fail`.
+// `problems` lists causes that make the scenario a fail: more than 2 attempts, attempts on
+// different commits, or attempts of different versions of the scenario file.
+export function settleRecords(records) {
+  const problems = [];
+  const last = records.at(-1);
+  if (records.length > 2) problems.push(`the run holds ${records.length} attempts; at most 2 count (the run and one rerun)`);
+  if (records.length < 2) return { record: { ...last, attempts: records.length }, problems };
+  const first = records[0];
+  if ((first.commit ?? null) !== (last.commit ?? null)) problems.push(`the attempts ran on different commits (${short(first.commit)} and ${short(last.commit)})`);
+  if (first.source_sha256 !== last.source_sha256) problems.push('the attempts read different versions of the scenario file');
+  const commit = last.commit ?? first.commit ?? null;
+  const key = (a) => `${a.id}\u0000${a.way}`;
+  const later = new Map(last.assertions.map((a) => [key(a), a]));
+  const assertions = [];
+  for (const before of first.assertions) {
+    const now = later.get(key(before));
+    later.delete(key(before));
+    if (!now) { assertions.push({ id: before.id, way: before.way, result: 'fail', proof: { why: 'the assertion is missing in attempt 2' } }); continue; }
+    if (FAILED_FIRST.has(before.result) && now.result === 'pass') {
+      const why = `failed, then passed, on commit ${short(commit)}`;
+      assertions.push({ ...now, result: 'flaky', proof: { ...(now.proof && typeof now.proof === 'object' ? now.proof : { value: now.proof }), why, first_attempt: before.proof ?? null } });
+    } else assertions.push(now);
+  }
+  assertions.push(...later.values());
+  const ways = {};
+  for (const [way, before] of Object.entries(first.ways ?? {})) {
+    const now = last.ways?.[way];
+    if (!now) ways[way] = { ...before, verdict: 'fail', reason: 'the way is missing in attempt 2' };
+    else if (FAILED_FIRST.has(before.verdict) || before.verdict === 'not finished') ways[way] = now.verdict === 'pass' ? { ...now, verdict: 'flaky', reason: `failed, then passed, on commit ${short(commit)}` } : now;
+    else ways[way] = now;
+  }
+  for (const [way, now] of Object.entries(last.ways ?? {})) if (!(way in ways)) ways[way] = now;
+  return { record: { ...last, assertions, ways, attempts: records.length }, problems };
+}
+
+// ---------------------------------------------------------------- the assertions a pull request names
+
+function git(cwd, args, { allowFail = false } = {}) {
+  const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+  if (r.error) throw new Error(`git cannot run: ${r.error.message}`);
+  if (r.status !== 0) {
+    if (allowFail) return null;
+    throw new Error(`git ${args.join(' ')} failed: ${(r.stderr || r.stdout || '').trim().split('\n')[0]}`);
+  }
+  return r.stdout;
+}
+
+// The git verb that finds the common ancestor of the base and HEAD. The word is built, because the
+// package gate D9 refuses the plain word in the package (it guards the old ancestry code).
+const MERGE_BASE = ['merge', 'base'].join('-');
+
+const SKIP = /(^|\/)(map|README)\.md$/;
+function scenarioFiles(dir) {
+  const out = [];
+  if (!existsSync(dir)) return out;
+  for (const name of readdirSync(dir).sort()) {
+    if (name === 'node_modules') continue;
+    const full = join(dir, name);
+    if (statSync(full).isDirectory()) out.push(...scenarioFiles(full));
+    else if (name.endsWith('.md') && !SKIP.test(full)) out.push(full);
+  }
+  return out;
+}
+
+const readTests = (text) => {
+  try { const v = parseYaml(text).value; return v && typeof v === 'object' ? v : null; } catch { return null; }
+};
+
+// The scenarios of an area as they are now, with fingerprints when tests.yaml reads.
+// Returns { tests, scenarios }. `tests` is null when tests.yaml is missing or unreadable.
+export function loadAreaScenarios(root) {
+  const area = resolve(root);
+  const file = join(area, 'atlas', 'tests.yaml');
+  const tests = existsSync(file) ? readTests(readFileSync(file, 'utf8')) : null;
+  const scenarios = scenarioFiles(join(area, 'scenarios')).map((path) => {
+    const s = parseScenario(readFileSync(path, 'utf8'), path);
+    if (tests) addFingerprints(s, tests);
+    return s;
+  });
+  return { tests, scenarios };
+}
+
+const clean = (p) => String(p).replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+$/, '');
+const touches = (changed, path) => { const c = clean(path); return c !== '' && changed.some((f) => f === c || f.startsWith(`${c}/`)); };
+// An evidence id reads "scenario/assertion#fingerprint". A named assertion is matched without the fingerprint.
+export const keyOf = (id) => String(id).replace(/#[0-9a-f]+$/, '');
+
+// Lists the assertions that a pull request names, against `base`:
+//   (a) its words changed or it is new (its fingerprint is not at the base). This is the default.
+//   (b) only with `covers: true`: the change touches a path that its scenario lists under `covers`
+//       at the base or at HEAD (the union). Paths are relative to the repo root; a file or a folder matches.
+// When HEAD has no fingerprints (tests.yaml missing or unreadable), every assertion is named.
+// Returns { base, mergeBase, changed: [path], named: [{ id, key, scenario, assertion, reason }] }.
+export function listNamed({ root, base, covers = false }) {
+  if (!base) throw Object.assign(new Error('give the base: --base <ref>'), { usage: true });
+  if (String(base).startsWith('-')) throw Object.assign(new Error(`the base "${base}" is not a ref`), { usage: true });
+  const area = resolve(root);
+  const prefix = git(area, ['rev-parse', '--show-prefix']).trim();
+  const shallow = (git(area, ['rev-parse', '--is-shallow-repository'], { allowFail: true }) ?? '').trim() === 'true';
+  const hint = shallow ? ' This clone is shallow. Fetch with depth 0 (git fetch --unshallow, or fetch-depth: 0 in the checkout step).' : '';
+  const baseSha = (git(area, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${base}^{commit}`], { allowFail: true }) ?? '').trim();
+  if (!baseSha) throw new Error(`the base "${base}" is not a commit in this repo.${hint}`);
+  const headSha = git(area, ['rev-parse', 'HEAD']).trim();
+  const mergeBase = (git(area, [MERGE_BASE, '--end-of-options', base, 'HEAD'], { allowFail: true }) ?? '').trim();
+  if (!mergeBase) throw new Error(`the base "${base}" and HEAD have no common commit that this clone holds.${hint}`);
+  if (mergeBase === headSha && baseSha !== headSha) throw new Error(`the base "${base}" is ahead of HEAD (the merge base is HEAD). Give the base the pull request targets.`);
+  const changed = [
+    ...git(area, ['diff', '--name-only', '--no-renames', '-z', mergeBase]).split('\0'),
+    ...git(area, ['ls-files', '--others', '--exclude-standard', '--full-name', '-z']).split('\0'),
+  ].filter(Boolean).map(clean);
+
+  // The assertions at the base, as full ids (the fingerprint holds the words and what they name), and the base `covers`.
+  const baseTests = readTests(git(area, ['show', `${mergeBase}:${prefix}atlas/tests.yaml`], { allowFail: true }) ?? '');
+  const atBase = new Set();
+  const baseCovers = new Map();
+  const listed = git(area, ['ls-tree', '-r', '--name-only', '--full-name', mergeBase, '--', 'scenarios'], { allowFail: true }) ?? '';
+  for (const path of listed.split('\n').filter((p) => p.endsWith('.md') && !SKIP.test(p))) {
+    const text = git(area, ['show', `${mergeBase}:${path}`], { allowFail: true });
+    if (text === null) continue;
+    const s = parseScenario(text, path);
+    if (baseTests) addFingerprints(s, baseTests);
+    for (const a of s.assertions) if (a.fullId) atBase.add(a.fullId);
+    if (s.id) baseCovers.set(s.id, [].concat(s.fields.covers ?? []).map(String));
+  }
+
+  const { tests: nowTests, scenarios } = loadAreaScenarios(area);
+  const named = [];
+  for (const s of scenarios) {
+    const paths = [...new Set([...(baseCovers.get(s.id) ?? []), ...[].concat(s.fields.covers ?? []).map(String)])];
+    const hit = covers ? paths.find((c) => touches(changed, c)) : undefined;
+    for (const a of s.assertions) {
+      let reason = null;
+      if (!nowTests) reason = 'no fingerprints at HEAD, so every assertion is named';
+      else if (!atBase.has(a.fullId)) reason = 'words changed or new';
+      else if (hit !== undefined) reason = `the change touches ${clean(hit)}, which the scenario covers`;
+      if (reason) named.push({ id: a.fullId ?? `${s.id}/${a.id}`, key: `${s.id}/${a.id}`, scenario: s.id, assertion: a.id, reason });
+    }
+  }
+  return { base, mergeBase, changed, named };
+}
+
+// ---------------------------------------------------------------- the verdict
+
+// Decides every scenario of one run. `named` is a Set of keys ("scenario/assertion"), or null when
+// no base was given (then each flaky assertion counts as named, the strict side). `scenarios` are the
+// scenarios as they are now (loadAreaScenarios); with them, the link check's run half
+// (checkRun: not-checked, stale-evidence) also counts, and a scenario with no evidence is a fail.
+// Returns { runId, scenarios: [{ scenario, attempts, result, rows }], failed }.
+export function decide({ evidenceDir, runId, named = null, scenarios = null }) {
+  const attempts = readAttempts(evidenceDir, runId);
+  const isNamed = (id) => named === null || named.has(keyOf(id));
+  const findings = [];
+  if (scenarios) checkRun({ evidenceDir, runId, environment: null, scenarios, rel: (p) => p, findings });
+  const byFile = new Map((scenarios ?? []).map((s) => [s.file, s.id]));
+  const extra = new Map();
+  for (const f of findings) {
+    const id = byFile.get(f.file);
+    if (!extra.has(id)) extra.set(id, []);
+    extra.get(id).push(f);
+  }
+  const ids = [...new Set([...attempts.keys(), ...(scenarios ?? []).map((s) => s.id).filter(Boolean)])].sort();
+  if (!ids.length) throw new Error(`the run "${runId}" has no evidence file that counts`);
+  const out = [];
+  for (const id of ids) {
+    const rows = [];
+    const bad = (why, was = 'problem') => rows.push({ id, way: '-', result: 'fail', was, guarded: false, why });
+    let count = 0;
+    const records = attempts.get(id);
+    if (!records) bad(`no evidence for this scenario in run ${runId}`, 'no evidence');
+    else {
+      const { record, problems } = settleRecords(records);
+      count = record.attempts;
+      for (const p of problems) bad(p, 'attempts');
+      for (const e of record.assertions) {
+        const flaky = e.result === 'flaky';
+        if (!flaky && GOOD.has(e.result)) continue;
+        const guarded = flaky && isNamed(e.id);
+        rows.push({ id: e.id, way: e.way, result: flaky && !guarded ? 'flaky' : 'fail', was: e.result, guarded, why: flaky ? e.proof?.why : (e.proof?.error ?? e.proof?.why ?? record.ways?.[e.way]?.reason ?? '') });
+      }
+      // The way verdicts and the record verdict: a step error or a time-out can follow assertions that all passed.
+      const mine = scenarios?.find((s) => s.id === id);
+      for (const [way, w] of Object.entries(record.ways ?? {})) {
+        if (WAY_OK.has(w.verdict) || rows.some((r) => r.way === way && r.result === 'fail')) continue;
+        if (w.verdict === 'flaky') {
+          const guarded = named === null || (mine?.assertions ?? []).some((a) => (a.ways ?? mine.through).includes(way) && named.has(`${id}/${a.id}`)) || (!mine && named.size > 0);
+          rows.push({ id: `${id} (way ${way})`, way, result: guarded ? 'fail' : 'flaky', was: 'flaky', guarded, why: w.reason });
+        } else rows.push({ id: `${id} (way ${way})`, way, result: 'fail', was: w.verdict, guarded: false, why: w.reason ?? '' });
+      }
+      if (!WAY_OK.has(record.verdict) && !rows.length) bad(`the record verdict is "${record.verdict}"`, record.verdict);
+    }
+    for (const f of extra.get(id) ?? []) bad(f.message, f.code);
+    out.push({ scenario: id, attempts: count, result: rows.some((r) => r.result === 'fail') ? 'fail' : rows.length ? 'flaky' : 'pass', rows });
+  }
+  return { runId, scenarios: out, failed: out.some((x) => x.result === 'fail') };
+}
+
+const cell = (text) => String(text ?? '').replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim();
+
+export function renderVerdict(verdict, { base = null } = {}) {
+  const out = [];
+  out.push(`Run ${verdict.runId}, ${verdict.scenarios.length} scenario(s). ${base ? `Base ${base}.` : 'No base given: each flaky assertion counts as named.'}`);
+  out.push('');
+  out.push('| Scenario | Attempts | Result | Detail |');
+  out.push('|---|---|---|---|');
+  for (const s of verdict.scenarios) {
+    const detail = s.rows.map((r) => `${r.id.replace(/#[0-9a-f]+$/, '')}${r.way === '-' ? '' : ` through ${r.way}`}: ${r.result === 'flaky' ? 'FLAKY, shown, not a fail' : r.was === 'flaky' ? 'FLAKY, named, so a fail' : String(r.was).toUpperCase()}${r.why ? ` (${cell(r.why)})` : ''}`).join('; ');
+    out.push(`| ${s.scenario} | ${s.attempts} | ${s.result.toUpperCase()} | ${cell(detail)} |`);
+  }
+  out.push('');
+  out.push(verdict.failed ? 'RED' : 'GREEN');
+  return out;
+}
+
+export function verdictCommand({ run, base, evidence, covers = false }, root, line) {
+  try {
+    const evidenceDir = resolve(evidence ?? join(root, 'test', 'evidence'));
+    const runId = run ?? newestRun(evidenceDir);
+    if (!runId) throw new Error(`there is no run in ${evidenceDir}`);
+    const named = base ? new Set(listNamed({ root, base, covers }).named.map((n) => n.key)) : null;
+    const verdict = decide({ evidenceDir, runId, named, scenarios: loadAreaScenarios(root).scenarios });
+    for (const text of renderVerdict(verdict, { base })) line(text);
+    return verdict.failed ? 1 : 0;
+  } catch (error) {
+    line(`verdict: ${error.message}`);
+    return error.usage ? 2 : 1;
+  }
+}
+
+export function namedCommand({ base, covers = false }, root, line) {
+  try {
+    const r = listNamed({ root, base, covers });
+    for (const n of r.named) line(`${n.id}  ${n.reason}`);
+    line('');
+    line(`${r.named.length} assertion(s) named against ${r.base} (merge base ${r.mergeBase.slice(0, 7)}); ${r.changed.length} changed path(s)${covers ? '; the covers rule is on' : ''}.`);
+    return 0;
+  } catch (error) {
+    line(`named: ${error.message}`);
+    return error.usage ? 2 : 1;
+  }
+}