@arjunkhera/atlas 0.3.12 → 0.3.14

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "atlas",
-  "version": "0.3.12",
+  "version": "0.3.14",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, for any repository.",
   "author": {
     "name": "Arjun Khera"

package/door/cli.mjs

@@ -28,6 +28,7 @@ import { writeDesignPage, readTracker } from './lib/design-build.mjs';
 import { loadPrivateTerms } from './lib/privacy.mjs';
 import { checkCommand as testsCheck, writeCommand as testsWrite } from './lib/tests.mjs';
 import { proofCommand as testsProof } from './lib/proof.mjs';
+import { verdictCommand as testsVerdict, namedCommand as testsNamed } from '../tests/verdict.mjs';
 
 export const PACKAGE_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
 const ATLAS_VERSION = packagePackageVersion(PACKAGE_ROOT);
@@ -63,6 +64,14 @@ const HELP = `atlas — the door into a repo's Atlas files
   atlas tests proof --root <area>      print the proof table of one run, one row for each assertion
       --run <id>                       the run; default the newest run in the evidence folder
       --own <file>                     JSON of the verifier's own checks: { id: { result, proof } }
+      --base <ref>                     show whether a flaky assertion is named by the pull request
+  atlas tests verdict --root <area>    decide each scenario of a run from both attempts: pass, fail or flaky; exit 1 on a fail
+      --run <id>                       the run; default the newest run in the evidence folder
+      --base <ref>                     the base of the pull request; flaky is a fail for an assertion it names. With no base every flaky is a fail
+      --covers                         also name the assertions of a scenario whose covers path the change touches
+  atlas tests named --root <area>      list the assertions that a pull request names
+      --base <ref>                     the base: an assertion is named when its words changed or it is new
+      --covers                         also name the assertions of a scenario whose covers path the change touches
 
   atlas install                        install Atlas for every folder on this Mac
   atlas upgrade [--version <v>]        check, show and install a newer release
@@ -89,7 +98,7 @@ export const FLAGS = Object.freeze({
   check: { root: 'optional', repo: 'value', 'fail-on': 'value' },
   ste: { share: 'switch', terms: 'value' },
   design: { tracker: 'value', out: 'value', draft: 'switch' },
-  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value' },
+  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch' },
   install: { local: 'switch', from: 'value', 'skip-global': 'switch', yes: 'switch' },
   upgrade: { version: 'optional', yes: 'switch' },
   doctor: {},
@@ -265,8 +274,10 @@ function testsCommand(chosen) {
     return testsCheck({ halves: chosen.halves, evidence: chosen.evidence, tests: chosen.tests, guardsFrom: chosen['guards-from'], run: chosen.run }, root, line, PACKAGE_ROOT);
   }
   if (what === 'write') return testsWrite({ dryRun: Boolean(chosen['dry-run']) }, root, PACKAGE_ROOT, line);
-  if (what === 'proof') return testsProof({ run: chosen.run, own: chosen.own, evidence: chosen.evidence }, root, line);
-  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write or proof.`);
+  if (what === 'proof') return testsProof({ run: chosen.run, own: chosen.own, evidence: chosen.evidence, base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
+  if (what === 'verdict') return testsVerdict({ run: chosen.run, base: chosen.base, evidence: chosen.evidence, covers: Boolean(chosen.covers) }, root, line);
+  if (what === 'named') return testsNamed({ base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
+  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict or named.`);
 }
 
 function checkCommand(chosen) {

package/door/kit-releases.json

@@ -1,4 +1,13 @@
 {
   "note": "Every kit of the folder tests/ that a release of Atlas shipped, as the kit hash of its manifest (the sha256 of the sorted lines of path and file hash). The kit of this package is always accepted and need not be listed. When a release changes anything in tests/, add the kit hash of the release before it here. atlas tests check reads this list; git history is never read.",
-  "releases": []
+  "releases": [
+    {
+      "kit_sha256": "ae0f32b8c6694fbffb8bdacd8bd919d27c1c1da6281a46779e521dfea2341565",
+      "first_package": "0.3.8"
+    },
+    {
+      "kit_sha256": "b2ddd9412d42f92c090432c0782aec498120395b2f8c7271006349c94b26c772",
+      "first_package": "0.3.13"
+    }
+  ]
 }

package/door/lib/proof.mjs

@@ -8,11 +8,15 @@
 // { "<scenario>/e1": { "result": "pass", "proof": "revision 1" } }; a full id
 // (with its #fingerprint) works as a key too. A way that is blocked or failed
 // shows its reason in the proof column. Evidence of a fault run is left out.
+// A scenario that kept files (screenshots of the browser driver) gets a list under the table:
+// each file with its run folder, and the count of console errors and failed requests of its page.
+// The pictures stay in the CI run (design section 8.3); the table holds their names only.
 import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { newestRunPerScenario } from '../../tests/link-check.mjs';
+import { settleRecords, listNamed, keyOf } from '../../tests/verdict.mjs';
 
-const MAX_CELL = 110;
+const MAX_CELL = 200;
 
 const clean = (text) => String(text ?? '').replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim();
 const clip = (text, n = MAX_CELL) => { const t = clean(text); return t.length > n ? `${t.slice(0, n - 3)}...` : t; };
@@ -23,6 +27,7 @@ function summary(value) {
   return typeof value === 'string' ? value : JSON.stringify(value);
 }
 
+const attemptNumber = (name) => { const m = /\.(\d+)\.json$/.exec(name); return m ? Number(m[1]) : 1; };
 const usage = (message) => Object.assign(new Error(message), { usage: true });
 
 // Reads the --own file. Throws an Error with a plain message on a bad file.
@@ -36,11 +41,14 @@ export function readOwn(file) {
   return data;
 }
 
-export function buildProof({ evidenceDir, runId = null, own = {} }) {
+// `named` is a Set of "scenario/assertion" keys (from listNamed), or null when the proof gets no base.
+export function buildProof({ evidenceDir, runId = null, own = {}, named = null }) {
   const perScenario = runId ? null : newestRunPerScenario(evidenceDir);
   if (!runId && !perScenario.size) throw new Error(`there is no run in ${evidenceDir}`);
   const names = runId ? [runId] : [...new Set(perScenario.values())].sort();
   const newest = new Map();
+  const attempts = new Map();
+  const attemptProblems = [];
   const used = new Set();
   let faultRuns = 0;
   for (const id of names) {
@@ -54,20 +62,41 @@ export function buildProof({ evidenceDir, runId = null, own = {} }) {
       if (data.fault_run) { faultRuns += 1; continue; }
       if (perScenario && perScenario.get(data.scenario) !== id) continue;
       used.add(id);
-      const before = newest.get(data.scenario);
-      if (!before || String(data.started) >= String(before.started)) newest.set(data.scenario, data);
+      const key = `${id}\u0000${data.scenario}`;
+      if (!attempts.has(key)) attempts.set(key, []);
+      attempts.get(key).push({ name, data });
     }
   }
+  // Several attempts of one scenario in one run (<scenario>.json, <scenario>.2.json) are joined:
+  // an assertion that failed, then passed, reads FLAKY. Of two runs of a scenario, the newest counts.
+  for (const list of attempts.values()) {
+    list.sort((a, b) => (attemptNumber(a.name) - attemptNumber(b.name)));
+    const { record: settled, problems } = settleRecords(list.map((x) => x.data));
+    for (const p of problems) attemptProblems.push(`${settled.scenario}: ${p}`);
+    const before = newest.get(settled.scenario);
+    if (!before || String(settled.started) >= String(before.started)) newest.set(settled.scenario, settled);
+  }
   const id = runId ?? [...used].sort().join(', ');
   if (!newest.size) throw new Error(`the run "${id}" has no evidence file that counts${faultRuns ? ' (only fault runs)' : ''}`);
   const scenarios = [...newest.values()].sort((a, b) => (a.scenario < b.scenario ? -1 : 1));
   const ways = [...new Set(scenarios.flatMap((s) => Object.keys(s.ways ?? {})))];
   const rows = [];
-  const blocked = [];
+  const blocked = [...attemptProblems];
+  const screenshots = [];
+  const pageEvents = [];
   for (const s of scenarios) {
     for (const [way, w] of Object.entries(s.ways ?? {})) {
       if (w.verdict === 'blocked' || w.verdict === 'fail') blocked.push(`${s.scenario} through ${way} is ${w.verdict}${w.reason ? `: ${clip(w.reason, 200)}` : ''}`);
     }
+    for (const file of s.files ?? []) {
+      if (/\.(png|jpe?g|webp)$/i.test(file)) screenshots.push({ scenario: s.scenario, file, run: s.run_id });
+    }
+    const events = (s.calls ?? []).map((c) => c.request?.event).filter(Boolean);
+    // A scenario that kept a picture used a page, so a count of 0 is news too.
+    if (events.length || (s.files ?? []).some((f) => /\.(png|jpe?g|webp)$/i.test(f))) {
+      const count = (kind) => events.filter((e) => e === kind).length;
+      pageEvents.push({ scenario: s.scenario, consoleErrors: count('console-error') + count('page-error'), failedRequests: count('request-failed'), httpErrors: count('http-error'), stopped: count('request-stopped') });
+    }
     const ids = [...new Set((s.assertions ?? []).map((a) => a.id))];
     for (const full of ids) {
       const short = full.replace(/#[0-9a-f]+$/, '');
@@ -78,7 +107,10 @@ export function buildProof({ evidenceDir, runId = null, own = {} }) {
       }));
       const problems = entries.filter((e) => e.result !== 'pass' && e.result !== 'not here').map((e) => {
         const why = e.proof?.error ?? e.proof?.why ?? s.ways?.[e.way]?.reason ?? '';
-        return `${e.way} ${e.result}${why ? `: ${why}` : ''}`;
+        const note = e.result !== 'flaky' ? ''
+          : named === null ? ' (FLAKY counts as a fail unless the pull request does not name it)'
+            : named.has(keyOf(full)) ? ' (FLAKY, named, so a fail)' : ' (FLAKY, shown, not a fail)';
+        return `${e.way} ${e.result}${why ? `: ${why}` : ''}${note}`;
       });
       const passed = entries.find((e) => e.result === 'pass' && e.proof?.value !== undefined);
       const mine = own[full] ?? own[short] ?? null;
@@ -92,7 +124,7 @@ export function buildProof({ evidenceDir, runId = null, own = {} }) {
       });
     }
   }
-  return { runId: id, environment: scenarios[0].environment, ways, rows, blocked, faultRuns, scenarios: scenarios.map((s) => s.scenario) };
+  return { runId: id, environment: scenarios[0].environment, ways, rows, blocked, faultRuns, screenshots, pageEvents, scenarios: scenarios.map((s) => s.scenario) };
 }
 
 export function renderProof(proof) {
@@ -107,6 +139,16 @@ export function renderProof(proof) {
     out.push('Blocked or failed ways in:');
     for (const b of proof.blocked) out.push(`- ${clean(b)}`);
   }
+  if (proof.screenshots.length) {
+    out.push('');
+    out.push('Screenshots (kept in test/evidence of the run; the CI run holds them for 14 days):');
+    for (const shot of proof.screenshots) out.push(`- ${clean(shot.scenario)}: test/evidence/${clean(shot.run)}/${clean(shot.file)}`);
+  }
+  if (proof.pageEvents.length) {
+    out.push('');
+    out.push('Page events:');
+    for (const e of proof.pageEvents) out.push(`- ${clean(e.scenario)}: ${e.consoleErrors} console error(s), ${e.failedRequests} failed request(s), ${e.httpErrors} HTTP error answer(s), ${e.stopped} request(s) stopped by the guards`);
+  }
   if (proof.faultRuns) {
     out.push('');
     out.push(`${proof.faultRuns} evidence file(s) of fault runs were left out.`);
@@ -117,8 +159,9 @@ export function renderProof(proof) {
 export function proofCommand(chosen, root, line) {
   try {
     const own = chosen.own ? readOwn(resolve(chosen.own)) : {};
+    const named = chosen.base ? new Set(listNamed({ root, base: chosen.base, covers: Boolean(chosen.covers) }).named.map((n) => n.key)) : null;
     const evidenceDir = resolve(chosen.evidence ?? join(root, 'test', 'evidence'));
-    for (const text of renderProof(buildProof({ evidenceDir, runId: chosen.run ?? null, own }))) line(text);
+    for (const text of renderProof(buildProof({ evidenceDir, runId: chosen.run ?? null, own, named }))) line(text);
     return 0;
   } catch (error) {
     line(`proof: ${error.message}`);

package/door/lib/tests.mjs

@@ -3,6 +3,8 @@
 //   atlas tests check --root <area>   run the link check on an area
 //   atlas tests write --root <area>   copy the shipped test kit into <area>/test/atlas/
 //   atlas tests proof --root <area>   print the proof table of one run (proof.mjs)
+//   atlas tests verdict --root <area> --run <id> [--base <ref>]   decide each scenario: pass, fail or flaky (tests/verdict.mjs)
+//   atlas tests named --root <area> --base <ref>   list the assertions that a pull request names (tests/verdict.mjs)
 //
 // The kit is the folder `tests/` of this package: the contract library, the
 // drivers, the stand-in helper and the link check. `write` copies each file

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arjunkhera/atlas",
-  "version": "0.3.12",
+  "version": "0.3.14",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, as a Claude Code plugin for any repository.",
   "type": "module",
   "license": "UNLICENSED",
@@ -25,5 +25,13 @@
     "agents/",
     ".claude-plugin/",
     ".mcp.json"
-  ]
+  ],
+  "peerDependencies": {
+    "playwright": ">=1.40.0"
+  },
+  "peerDependenciesMeta": {
+    "playwright": {
+      "optional": true
+    }
+  }
 }

package/skills/tests/SKILL.md

@@ -41,6 +41,27 @@ every file that command writes.
 2. `atlas tests check --root <repo>` runs the link check. It prints one line
    for each finding, as `file:line  code  words`. Run it before you ask for a merge.
 3. `atlas tests proof --root <repo>` prints the proof table of one run.
+4. `atlas tests verdict --root <repo> --run <id> --base <ref>` decides each
+   scenario of a run. Exit 1 means a scenario failed.
+5. `atlas tests named --root <repo> --base <ref>` lists the assertions that a
+   pull request names.
+
+When a scenario fails in CI, CI runs the whole scenario once more on the
+same commit. The second run writes `<scenario>.2.json` in the same run folder.
+Never retry inside a test. Then CI runs `atlas tests verdict`:
+
+1. Both runs fail: the result is `fail`. More than two runs, or two runs on
+   different commits, are also a `fail`.
+2. The first run fails and the second passes: each assertion that failed
+   reads `flaky`. The proof table shows "failed, then passed, on commit
+   <sha>".
+3. A pull request names an assertion when its words changed or it is new.
+   `flaky` is a `fail` for a named assertion. For any other assertion it is
+   shown, and it is not a fail. The flag `--covers` also names the assertions
+   of a scenario whose `covers` path the change touches. It is off until the
+   owner decides.
+4. The rule to open an item for each `flaky` result waits on the owner's
+   answer. Do not open one yet.
 
 The link check has three halves. Run the first two before the tests and the
 third after them:

package/tests/contract.mjs

@@ -34,6 +34,7 @@
 //                     a stand-in made in this process, instead of a start command
 //   adapter.warmUp    async (env) => {}   make the product call its stand-ins once
 //   adapter.drivers   { wayOrDriver: async (ctx) => driver }  a driver the kit does not ship
+//   adapter.playwright the Playwright module, for a test of the browser driver that has none installed
 //   adapter.functions the module (or { way: module }) for the function driver
 //   adapter.loadModule async (file) => module   loads the `module` of a function way in; the
 //                     kit gives the path under the product root, so a fault run loads the copy
@@ -145,10 +146,10 @@ export function scenario(path, body, options = {}) {
     if (!runsHere) {
       it(`${sc.id} [not here: runs-in excludes ${environment}]`, () => {
         const ev = evidence();
-        for (const way of sc.through) {
-          for (const a of sc.assertions) ev.setResult(a.fullId, way, 'not here', { why: `runs-in does not list ${environment}` });
-          ev.setWay(way, { verdict: 'not here', started: new Date().toISOString(), ended: new Date().toISOString() });
+        for (const a of sc.assertions) {
+          for (const way of a.ways) ev.setResult(a.fullId, way, 'not here', { why: `runs-in does not list ${environment}` });
         }
+        for (const way of sc.ways) ev.setWay(way, { verdict: 'not here', started: new Date().toISOString(), ended: new Date().toISOString() });
       });
       return;
     }
@@ -167,6 +168,8 @@ export function scenario(path, body, options = {}) {
           error = thrown;
         }
         const outcome = settle({ sc, way, ev, error });
+        // A failed run keeps a picture of each open page (a driver that has one), before the run closes.
+        if (run && outcome.verdict === 'fail') await keepOnFail(state, sc.id);
         // The run is closed: a late call (after a time limit, say) changes nothing.
         run?.close();
         await cleanUp(state, run, ev);
@@ -182,6 +185,12 @@ export function scenario(path, body, options = {}) {
   });
 }
 
+async function keepOnFail(state, label) {
+  for (const pending of state.drivers.values()) {
+    try { await (await pending).keepOnFail?.(`fail-${label}`); } catch { /* a missing picture is not the news */ }
+  }
+}
+
 function withLimit(promise, ms, text) {
   let timer;
   const limit = new Promise((_, reject) => { timer = setTimeout(() => reject(new Error(`the scenario ran out of time: its limit is ${text}`)), ms); timer.unref?.(); });
@@ -195,7 +204,9 @@ async function cleanUp(state, run, ev) {
 
 // Turns the end of a body into results, a verdict and a list of problems.
 function settle({ sc, way, ev, error }) {
-  const mine = ev.record.assertions.filter((a) => a.way === way);
+  // The entries of this way in, and the entries of a way in that only a step names (the page).
+  // A run through `mcp` checks the page assertions too, so it settles them.
+  const mine = ev.record.assertions.filter((a) => a.way === way || !sc.through.includes(a.way));
   const problems = [];
   let verdict = 'pass';
   let reason = null;
@@ -220,6 +231,21 @@ function settle({ sc, way, ev, error }) {
   if (verdict === 'blocked') problems.unshift(`BLOCKED: ${reason}`);
   else if (reason && !(error instanceof GateFailed)) problems.push(`the run stopped: ${reason}`);
   ev.setWay(way, { verdict, reason, ended: new Date().toISOString() });
+  // A way in that only a step names has no run of its own. Its verdict is the verdict of its assertions.
+  for (const extra of sc.ways.filter((w) => !sc.through.includes(w))) {
+    const own = ev.record.assertions.filter((a) => a.way === extra).map((a) => a.result);
+    // A gate that failed, or a block, leaves the later assertions `not checked`. That is no `fail` of the page.
+    const stopped = own.every((r) => r === 'pass' || r === 'not here' || r === 'not checked' || r === 'blocked') && own.some((r) => r === 'not checked' || r === 'blocked');
+    let extraVerdict;
+    let extraReason = null;
+    if (own.includes('fail')) { extraVerdict = 'fail'; extraReason = `an assertion through ${extra} failed`; }
+    else if (stopped) {
+      extraVerdict = 'blocked';
+      extraReason = error instanceof GateFailed ? 'the gate stopped the run' : verdict === 'blocked' ? reason : 'the run stopped before every assertion through this way in was checked';
+    } else if (own.every((r) => r === 'pass' || r === 'not here')) extraVerdict = own.every((r) => r === 'not here') ? 'not here' : 'pass';
+    else { extraVerdict = 'fail'; extraReason = `an assertion through ${extra} is not a pass`; }
+    ev.setWay(extra, { verdict: extraVerdict, reason: extraReason, started: ev.record.ways[way].started, ended: new Date().toISOString() });
+  }
   return { verdict, problems };
 }
 
@@ -240,7 +266,12 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
     if (!state.drivers.has(wayName)) {
       const spec = tests['ways-in'][wayName];
       if (!spec) throw new Error(`there is no way in "${wayName}" in tests.yaml`);
-      state.drivers.set(wayName, makeDriver({ way: wayName, spec, env, root, hosts: env.hosts, redactor: state.redactor, record, adapter, runId: RUN_ID }));
+      // A driver is shared by the runs of the scenario (the page way in has no run of its own), so it reaches
+      // the record and the attach of the run that is live now, and not those of the run that made it.
+      // A driver is shared by the runs of a scenario (the page way in has no run of its own), so it reaches the
+      // record and the attach of the run that is live now (state.live), not those of the run that made it.
+      // attach: where a driver keeps a file (a screenshot) as evidence, in the folder of this scenario.
+      state.drivers.set(wayName, makeDriver({ way: wayName, spec, env, root, hosts: env.hosts, redactor: state.redactor, record: (call) => state.live?.record(call), adapter, runId: RUN_ID, attach: (name) => state.live.attach(name) }));
     }
     return state.drivers.get(wayName);
   };
@@ -257,8 +288,9 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
   const settleOne = (full, fn, opts = {}, mode) => {
     open(`run.${mode}`);
     const { a, fp } = find(full);
-    const target = opts.way ?? way;
-    if (!sc.through.includes(target)) throw new Error(`"${target}" is not a way in of this scenario`);
+    // An assertion that reads a result of a step with its own way in (the page) belongs to that way.
+    const target = opts.way ?? (a.ways.includes(way) ? way : a.ways[0]);
+    if (!a.ways.includes(target)) throw new Error(`"${target}" is not a way in of ${a.id}: its ways are ${a.ways.join(', ')}`);
     const entry = ev.entry(a.fullId, target);
     const calls = callsSince();
     let failure = null;
@@ -274,8 +306,14 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
         if (isThenable(value)) { failure = 'a check must not be async: wait first, then check'; value = undefined; }
       } catch (e) { failure = e?.message ?? String(e); }
     }
-    if (failure) ev.setResult(a.fullId, target, 'fail', { error: failure, calls });
-    else if (entry.result !== 'fail') ev.setResult(a.fullId, target, 'pass', { value: value === undefined ? undefined : clip(value, state.redactor), calls });
+    // `settled_by` names the run (the way in of `through:`) that settled the assertion. A page assertion of a
+    // scenario with two `through` ways is settled twice. A fail stands over a later pass. The result of the other
+    // run is listed under `also`, so no result is overwritten without a trace.
+    const earlier = entry.proof?.settled_by && entry.proof.settled_by !== way ? entry : null;
+    const also = earlier ? [...(earlier.proof.also ?? []), { run: earlier.proof.settled_by, result: earlier.result }] : (entry.proof?.also ?? undefined);
+    if (failure) ev.setResult(a.fullId, target, 'fail', { error: failure, calls, settled_by: way, also });
+    else if (entry.result !== 'fail') ev.setResult(a.fullId, target, 'pass', { value: value === undefined ? undefined : clip(value, state.redactor), calls, settled_by: way, also });
+    else if (earlier || entry.proof) { entry.proof = { ...entry.proof, also: [...(entry.proof.also ?? []), { run: way, result: 'pass' }] }; ev.write(); }
     if (failure && mode === 'gate') throw new GateFailed(`[${a.id}] ${failure}`);
     return a;
   };
@@ -393,6 +431,7 @@ function makeRun({ ctx, state, way, env, ev, adapter }) {
     // Called by the library after the body and the results are settled.
     close: () => { closed = true; },
   };
+  state.live = { record, attach: (name) => run.attach(name) };
   run.actions = typeof adapter.actions === 'function' ? adapter.actions(run) : (adapter.actions ?? {});
   return run;
 }