@arjunkhera/atlas 0.3.43 → 0.3.44

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "atlas",
-  "version": "0.3.43",
+  "version": "0.3.44",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, for any repository.",
   "author": {
     "name": "Arjun Khera"

package/door/cli.mjs

@@ -34,6 +34,7 @@ import { runKept } from '../tests/kept.mjs';
 import { sweepCommand as testsSweep, coversCommand as testsCovers } from '../tests/sweep.mjs';
 import { issuesCommand as testsIssues } from './lib/issues.mjs';
 import { changedCommand as testsChanged } from './lib/changed-tests.mjs';
+import { beforeCommand as testsBefore } from './lib/fails-before.mjs';
 import { upkeepCommand } from './lib/upkeep.mjs';
 import { judgeCommand } from './lib/judge.mjs';
 
@@ -86,6 +87,13 @@ const HELP = `atlas — the door into a repo's Atlas files
       --head <ref>                     the head of the pull request; default HEAD. Only committed files count
       --out <file>                     write the JSON here; default the terminal
       --text                           print a short table instead of the JSON
+  atlas tests before --root <repo> --base <ref>   each new or changed test case must fail on the old code and pass on the new code; exit 1 when one does not
+      --head <ref>                     the head of the pull request; default HEAD
+      --list <file>                    a changed-test list from atlas tests changed; default the command makes it
+      --side <old|new>                 run one side only, and write its run file (CI runs each side in its own job)
+      --old <file> --new <file>        read the two run files and give the answer; runs nothing
+      --out <file>                     write the JSON here; default the terminal
+      --text                           print a short table instead of the JSON
   atlas tests named --root <area>      list the assertions that a pull request names
       --base <ref>                     the base: an assertion is named when its words changed or it is new
       --covers                         also name the assertions of a scenario whose covers path the change touches
@@ -151,7 +159,7 @@ export const FLAGS = Object.freeze({
   ste: { share: 'switch', terms: 'value' },
   design: { tracker: 'value', out: 'value', draft: 'switch', root: 'optional' },
   view: { product: 'value', name: 'value', out: 'value' },
-  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', head: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch', scenario: 'value', language: 'value' },
+  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', head: 'value', list: 'value', side: 'value', old: 'value', new: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch', scenario: 'value', language: 'value' },
   upkeep: { root: 'optional', event: 'value', json: 'switch' },
   judge: { tasks: 'value', verdict: 'value', seal: 'value', runs: 'value', out: 'value', repo: 'value', budget: 'value', model: 'value' },
   install: { local: 'switch', from: 'value', 'skip-global': 'switch', yes: 'switch' },
@@ -349,13 +357,14 @@ async function testsCommand(chosen) {
   if (what === 'proof') return testsProof({ run: chosen.run, own: chosen.own, evidence: chosen.evidence, base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
   if (what === 'verdict') return testsVerdict({ run: chosen.run, base: chosen.base, evidence: chosen.evidence, covers: Boolean(chosen.covers) }, root, line);
   if (what === 'changed') return testsChanged({ base: chosen.base, head: chosen.head, out: chosen.out, text: Boolean(chosen.text) }, root, line);
+  if (what === 'before') return testsBefore({ base: chosen.base, head: chosen.head, list: chosen.list, out: chosen.out, side: chosen.side, old: chosen.old, new: chosen.new, text: Boolean(chosen.text) }, root, line);
   if (what === 'named') return testsNamed({ base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
   if (what === 'sweep') return testsSweep({ evidence: chosen.evidence, since: chosen.since, base: chosen.base, out: chosen.out, text: Boolean(chosen.text) }, root, line);
   if (what === 'covers') return testsCovers({ itemId: chosen._[2], evidence: chosen.evidence, json: Boolean(chosen.json) }, root, line);
   if (what === 'kept') return (await runKept({ root, evidenceDir: chosen.evidence, only: chosen.scenario ?? null, write: (text) => process.stdout.write(text) })).exit;
   if (what === 'rerun') return testsRerun({ run: chosen.run, evidence: chosen.evidence, tests: chosen.tests, language: chosen.language ?? 'javascript' }, root, line);
   if (what === 'issues') return testsIssues({ findings: chosen.findings, repo: chosen.repo, max: chosen.max, bot: chosen.bot, dryRun: Boolean(chosen['dry-run']), args: [...chosen._, ...Object.values(chosen).filter((v) => typeof v === 'string')] }, line);
-  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict, named, changed, sweep, covers, kept, issues or rerun.`);
+  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict, named, changed, before, sweep, covers, kept, issues or rerun.`);
 }
 
 function upkeepVerb(chosen) {

package/door/lib/changed-tests.mjs

@@ -46,10 +46,12 @@ const keyOf = (id) => String(id).replace(/#[0-9a-f]+$/, '');
 const RUNNER_CONFIG = /^(vitest\.config\.[^/]+|vitest\.[^/]+\.config\.[^/]+|jest\.config\.[^/]+|playwright\.config\.[^/]+|\.mocharc(\.[^/]+)?|pytest\.ini|tox\.ini)$/;
 const UNIT_TEST = /^([^/]+\.(test|spec)\.[^/]+|test_[^/]+\.py|[^/]+_test\.py|[^/]+_test\.go)$/;
 // A helper folder at any depth. These names mean tests in every tool that uses them.
-const HELPER_ANY = new Set(['__tests__', '__snapshots__', 'testdata']);
+// `test` and `fixtures` are among them (shape/test/, pkg/fixtures/): fails-before needs a deep helper
+// or a deep data file to be the same in both runs (design docs/design-docs/fails-before/).
+const HELPER_ANY = new Set(['test', '__tests__', '__snapshots__', 'testdata', 'fixtures']);
 // A helper folder only at the top of the repo or of an area. Deeper, such a name can be product code
 // (the Atlas test kit ships in packages/atlas/tests/).
-const HELPER_TOP = new Set(['test', 'tests', 'spec', 'fixtures', 'golden']);
+const HELPER_TOP = new Set(['tests', 'spec', 'golden']);
 // Test settings that live inside a package.json, beside the test scripts.
 const PACKAGE_TEST_KEYS = Object.freeze(['jest', 'vitest', 'mocha', 'ava', 'c8', 'nyc']);
 const SCENARIO_TEST = /(^|\/)tests?\/scenarios\/[^/]+(\.test\.mjs|_test\.py)$/;

package/door/lib/fails-before.mjs

@@ -0,0 +1,382 @@
+// `atlas tests before`: each new or changed test case of a pull request must fail on the old code
+// and pass on the new code (roadmap R4, tracker item 1bbafe3d; design docs/design-docs/fails-before/).
+//
+// It reads the changed-test list (changed-tests.mjs) and takes each unit test file that the pull
+// request adds, changes or renames. Run 1 is on the old code: a tree of the merge base, with the
+// head's test files and test helpers copied in. Run 2 is a tree of the head. Both runs use one
+// path and one environment, so only the code under test differs. Each report is read with the JUnit
+// reader of the kit.
+//
+//   before      after   result
+//   assertion   pass    proven
+//   pass        pass    passes before   (red: the test cannot catch this change)
+//   any         other   fails after     (red)
+//   missing     pass    load error      (not proven)
+//   error       pass    error before    (not proven)
+//   skipped on either side  skipped     (red)
+//   two cases with one name duplicate   (red)
+//
+// Green does not mean proven: the gate counts only `proven` cases (proven_count).
+// The command runs head code. It holds no secret: each run gets a fixed environment and nothing else.
+// In CI each side runs in its own job on a fresh runner (--side old, --side new), and a third job
+// reads both run files (--old, --new) with the base copy of Atlas. Each tree comes from git archive.
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, posix, resolve } from 'node:path';
+import { readJunit } from '../../tests/junit.mjs';
+import { listChanged } from './changed-tests.mjs';
+
+export const VERSION = 1;
+
+function git(cwd, args, { allowFail = false, buffer = false } = {}) {
+  const r = spawnSync('git', args, { cwd, encoding: buffer ? 'buffer' : 'utf8', maxBuffer: 256 * 1024 * 1024 });
+  if (r.error) throw new Error(`git cannot run: ${r.error.message}`);
+  if (r.status !== 0) {
+    if (allowFail) return null;
+    throw new Error(`git ${args.join(' ')} failed: ${String(r.stderr || r.stdout || '').trim().split('\n')[0]}`);
+  }
+  return r.stdout;
+}
+const usage = (message) => Object.assign(new Error(message), { usage: true });
+
+// ---------------------------------------------------------------- the cases of one test file
+
+// The runner of a test file, by its name. null: no runner yet.
+export function runnerOf(path) {
+  const name = posix.basename(path);
+  if (/\.test\.(mjs|js|cjs)$/.test(name)) return 'node';
+  if (/^test_[^/]+\.py$|_test\.py$/.test(name)) return 'pytest';
+  return null;
+}
+
+// Skips a string, a template string or a comment at `i`. Returns the index after it, or `i`.
+function skip(src, i) {
+  const c = src[i];
+  if (c === '/' && src[i + 1] === '/') { const end = src.indexOf('\n', i); return end < 0 ? src.length : end; }
+  if (c === '/' && src[i + 1] === '*') { const end = src.indexOf('*/', i + 2); if (end < 0) throw new Error('open comment'); return end + 2; }
+  if (c === '"' || c === "'" || c === '`') {
+    for (let j = i + 1; j < src.length; j += 1) {
+      if (src[j] === '\\') { j += 1; continue; }
+      if (src[j] === c) return j + 1;
+      if (c === '`' && src[j] === '$' && src[j + 1] === '{') j = close(src, j + 1) - 1;
+      else if (c !== '`' && src[j] === '\n') break;
+    }
+    throw new Error('open string');
+  }
+  return i;
+}
+
+// The index after the bracket that closes the one at `open`. A regular expression is read as code,
+// so a bracket inside one can end a case early; then the text still differs where the case differs.
+function close(src, open) {
+  const pair = { '(': ')', '{': '}', '[': ']' };
+  const stack = [];
+  for (let i = open; i < src.length;) {
+    const after = skip(src, i);
+    if (after !== i) { i = after; continue; }
+    const c = src[i];
+    if (pair[c]) stack.push(pair[c]);
+    else if (c === ')' || c === '}' || c === ']') {
+      if (stack.pop() !== c) throw new Error('bracket mismatch');
+      if (!stack.length) return i + 1;
+    }
+    i += 1;
+  }
+  throw new Error('open bracket');
+}
+
+// The test cases of one file: { cases: Map of name to its text, rest, duplicates }. `rest` is the
+// text outside every case, so an edit there (a beforeEach, a shared value) can mark every case.
+// null when the file cannot be read to the end. JavaScript: each test(…) or it(…) call whose first
+// argument is a fixed string. Python: each def test_… function, at any indent, to the end of its block.
+export function casesOf(text, path) {
+  const out = { cases: new Map(), rest: '', duplicates: new Set() };
+  if (text === null) return out;
+  const src = String(text).replace(/\r\n?/g, '\n');
+  const add = (name, body) => {
+    if (out.cases.has(name)) out.duplicates.add(name);
+    out.cases.set(name, out.cases.has(name) ? `${out.cases.get(name)}\n${body}` : body);
+  };
+  const rest = [];
+  try {
+    if (path.endsWith('.py')) {
+      const lines = src.split('\n');
+      for (let i = 0; i < lines.length; i += 1) {
+        const m = /^(\s*)(?:async\s+)?def\s+(test\w*)\s*\(/.exec(lines[i]);
+        if (!m) { rest.push(lines[i]); continue; }
+        let j = i + 1;
+        while (j < lines.length && (lines[j].trim() === '' || lines[j].search(/\S/) > m[1].length)) j += 1;
+        add(m[2], lines.slice(i, j).join('\n').trimEnd());
+        i = j - 1;
+      }
+    } else {
+      const call = /(?<![\w.$])(test|it)(\.(?:only|skip|todo))?\(\s*(["'`])([^"'`\n]*)\3/g;
+      let m;
+      let from = 0;
+      while ((m = call.exec(src))) {
+        if (m.index < from) continue;
+        const end = close(src, src.indexOf('(', m.index));
+        add(m[4], src.slice(m.index, end));
+        rest.push(src.slice(from, m.index));
+        from = end;
+        call.lastIndex = end;
+      }
+      rest.push(src.slice(from));
+    }
+  } catch {
+    return null;
+  }
+  // A piece between two cases that holds only white space and separators is no code.
+  out.rest = rest.map((piece) => piece.replace(/^[\s;,]+|[\s;,]+$/g, '')).filter(Boolean).join('\n').replace(/\s+/g, ' ').trim();
+  return out;
+}
+
+// ---------------------------------------------------------------- one report
+
+// The text of each <testcase> element, in order, so a failure can be sorted.
+function caseTexts(xml) {
+  return [...String(xml).matchAll(/<testcase\b[^>]*?(?:\/>|>([\s\S]*?)<\/testcase>)/g)].map((m) => m[1] ?? '');
+}
+
+// The results of one report: Map of case name to pass, assertion, error or skipped.
+// For pytest, a name with parameters ("test_x[1]") counts under its function name.
+export function readReport(xml) {
+  const cases = readJunit(xml);
+  const texts = caseTexts(xml);
+  const rank = { pass: 0, skipped: 1, error: 2, assertion: 3 };
+  const out = new Map();
+  cases.forEach((c, i) => {
+    let result = c.result;
+    if (result === 'fail') result = /AssertionError|ERR_ASSERTION/.test(`${c.message ?? ''}\n${texts[i] ?? ''}`) ? 'assertion' : 'error';
+    const name = c.name.replace(/\[.*\]$/, '');
+    // When one name holds several results, the worst one counts: a fail beats a pass.
+    if (!out.has(name) || rank[result] > rank[out.get(name)]) out.set(name, result);
+  });
+  return out;
+}
+
+export function resultOf(before, after) {
+  if (before === 'skipped' || after === 'skipped') return 'skipped';
+  if (after !== 'pass') return 'fails after';
+  if (before === 'assertion') return 'proven';
+  if (before === 'pass') return 'passes before';
+  if (before === 'missing') return 'load error';
+  return 'error before';
+}
+
+export const RED_RESULTS = Object.freeze(['passes before', 'fails after', 'skipped', 'duplicate']);
+
+// ---------------------------------------------------------------- the plan
+
+// A JavaScript case that only moved its line breaks or spaces is not changed. In Python the indent
+// is code, so its text counts as it is.
+const squash = (text, path) => (path.endsWith('.py') ? text : text.replace(/\s+/g, ' ').trim());
+const isProduct = (path) => !/^(docs|archive)\//.test(path) && !/^[^/]+\.md$/i.test(path);
+const OVERLAY = new Set(['unit-test', 'test-helper', 'runner-config', 'kit-manifest', 'check-block']);
+
+// What to run: the changed-test list, the files and, for each file that runs, its picked cases.
+export function plan({ root, base, head = 'HEAD', list = null }) {
+  if (!base && !list) throw usage('give the base: --base <ref>');
+  const top = git(resolve(root), ['rev-parse', '--show-toplevel']).trim();
+  const changed = list ? JSON.parse(readFileSync(resolve(list), 'utf8')) : listChanged({ root: top, base, head });
+  if (changed.changed_tests !== 1) throw new Error(`the changed-test list has version ${changed.changed_tests}; this command reads version 1`);
+  const show = (sha, path) => git(top, ['show', `${sha}:${path}`], { allowFail: true });
+  const files = [];
+  const plans = [];
+  for (const r of changed.rows.filter((x) => x.group === 'check-block' && x.status !== 'removed')) {
+    if (!files.some((f) => f.path === r.path)) files.push({ path: r.path, runner: null, result: 'not run', why: 'a scenario test needs its environment; the planted-fault procedure covers it' });
+  }
+  for (const r of changed.rows.filter((x) => x.group === 'unit-test' && x.status !== 'removed')) {
+    const runner = runnerOf(r.path);
+    if (!runner) { files.push({ path: r.path, runner: null, result: 'not run', why: 'no runner for this file yet' }); continue; }
+    const before = r.status === 'added' ? casesOf(null, r.path) : casesOf(show(changed.merge_base, r.from ?? r.path), r.from ?? r.path);
+    const after = casesOf(show(changed.head, r.path), r.path);
+    // all: every case of the file counts as changed, also those that the reader did not find.
+    let all = false;
+    const picked = [];
+    if (after === null || before === null) all = true;
+    else {
+      if (r.status !== 'added' && before.rest !== after.rest) all = true;
+      for (const [name, text] of after.cases) {
+        if (all || !before.cases.has(name) || squash(before.cases.get(name), r.path) !== squash(text, r.path)) picked.push({ name, change: before.cases.has(name) ? 'changed' : 'added', duplicate: after.duplicates.has(name) });
+      }
+    }
+    if (!picked.length && !all && r.status !== 'added') continue;
+    files.push({ path: r.path, runner, result: 'ran', why: null });
+    plans.push({ path: r.path, runner, picked, all: all || r.status === 'added', known: after ? [...after.cases.keys()] : [] });
+  }
+  files.sort((x, y) => (x.path < y.path ? -1 : 1));
+  return { top, changed, files, plans, productChanged: changed.other_paths.some(isProduct) };
+}
+
+// ---------------------------------------------------------------- one run
+
+// The environment of every run. Nothing of the parent passes through but PATH.
+export function runEnv(work) {
+  return { PATH: process.env.PATH ?? '/usr/bin:/bin', HOME: join(work, 'home'), TMPDIR: join(work, 'tmp'), TZ: 'UTC', LANG: 'C.UTF-8', CI: 'true' };
+}
+
+const commandOf = (runner, file, report) => (runner === 'node'
+  ? ['node', '--test', '--test-reporter=junit', `--test-reporter-destination=${report}`, file]
+  : ['python3', '-B', '-m', 'pytest', '-p', 'no:cacheprovider', '-q', `--junitxml=${report}`, file]);
+
+// A fresh tree of `sha` at `tree`, from git archive: no .git, so a test cannot read which commit it is.
+function freshTree(top, tree, sha) {
+  rmSync(tree, { recursive: true, force: true });
+  mkdirSync(tree, { recursive: true });
+  const archive = spawnSync('git', ['archive', '--format=tar', sha], { cwd: top, maxBuffer: 1024 * 1024 * 1024 });
+  if (archive.status !== 0) throw new Error(`git archive failed: ${String(archive.stderr).trim()}`);
+  const untar = spawnSync('tar', ['-x', '-C', tree], { input: archive.stdout });
+  if (untar.status !== 0) throw new Error(`tar failed: ${String(untar.stderr).trim()}`);
+  if (existsSync(join(top, 'node_modules')) && !existsSync(join(tree, 'node_modules'))) symlinkSync(join(top, 'node_modules'), join(tree, 'node_modules'), 'dir');
+}
+
+// Runs one side: `old` (the merge base with the head's test files) or `new` (the head).
+// Returns the run file: { side, merge_base, head, tree, report, env, files: { path: { command, cases, why } } }.
+export function runSide({ top, changed, plans, side, work = join(tmpdir(), 'atlas-before') }) {
+  if (side !== 'old' && side !== 'new') throw usage('--side is old or new');
+  const tree = join(work, 'repo');
+  const report = join(work, 'report', 'report.xml');
+  const env = runEnv(work);
+  const out = { fails_before_run: VERSION, side, merge_base: changed.merge_base, head: changed.head, tree, report, env, files: {} };
+  const python = plans.some((p) => p.runner === 'pytest');
+  try {
+    for (const p of plans) {
+      // A fresh tree, home and report folder for each file, so no file of one run reaches the next.
+      freshTree(top, tree, side === 'old' ? changed.merge_base : changed.head);
+      if (side === 'old') {
+        for (const r of changed.rows.filter((x) => OVERLAY.has(x.group))) {
+          if (r.from) rmSync(join(tree, r.from), { force: true });
+          if (r.status === 'removed') { rmSync(join(tree, r.path), { force: true }); continue; }
+          const bytes = git(top, ['show', `${changed.head}:${r.path}`], { allowFail: true, buffer: true });
+          if (bytes === null) continue;
+          mkdirSync(dirname(join(tree, r.path)), { recursive: true });
+          writeFileSync(join(tree, r.path), bytes);
+        }
+      }
+      for (const dir of ['home', 'tmp', 'report']) { rmSync(join(work, dir), { recursive: true, force: true }); mkdirSync(join(work, dir), { recursive: true }); }
+      const command = commandOf(p.runner, p.path, report);
+      if (python && p.runner === 'pytest' && spawnSync('python3', ['-B', '-c', 'import pytest'], { cwd: tree, env }).status !== 0) {
+        out.files[p.path] = { command, cases: null, why: 'pytest is not installed', none: true };
+        continue;
+      }
+      const r = spawnSync(command[0] === 'node' ? process.execPath : command[0], command.slice(1), { cwd: tree, env, encoding: 'utf8', timeout: 10 * 60 * 1000, killSignal: 'SIGKILL', maxBuffer: 64 * 1024 * 1024 });
+      let cases = null;
+      let why = null;
+      if (r.error) why = `the runner did not finish: ${r.error.message}`;
+      else if (!existsSync(report)) why = `the runner wrote no report (exit ${r.status})`;
+      else {
+        try { cases = Object.fromEntries(readReport(readFileSync(report, 'utf8'))); } catch (error) { why = `the report does not read: ${error.message}`; }
+      }
+      out.files[p.path] = { command, cases, why };
+    }
+  } finally {
+    rmSync(tree, { recursive: true, force: true });
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------- the answer
+
+const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
+
+// Reads the two run files and gives the answer. It runs nothing.
+export function combine(prepared, oldRun, newRun) {
+  const { changed, files, plans, productChanged } = prepared;
+  const answer = { fails_before: VERSION, base: changed.base, merge_base: changed.merge_base, head: changed.head, product_changed: productChanged, red: false, proven_count: 0, files: files.map((f) => ({ ...f })), cases: [], env: null, runs: null };
+  if (!productChanged || !plans.length) {
+    for (const p of plans) for (const c of p.picked) answer.cases.push({ path: p.path, name: c.name, change: c.change, before: null, after: null, result: c.duplicate ? 'duplicate' : 'not proven' });
+    return finish(answer);
+  }
+  if (oldRun?.side !== 'old' || newRun?.side !== 'new') throw new Error('give one run file of the side old and one of the side new');
+  for (const run of [oldRun, newRun]) {
+    if (run.fails_before_run !== VERSION) throw new Error(`a run file has version ${run.fails_before_run}; this command reads version ${VERSION}`);
+    if (run.merge_base !== changed.merge_base || run.head !== changed.head) throw new Error(`the run file of side ${run.side} is of other commits`);
+  }
+  if (!same(oldRun.env, newRun.env) || oldRun.tree !== newRun.tree || oldRun.report !== newRun.report) throw new Error('the two runs did not use one tree path, one report path and one environment');
+  answer.env = newRun.env;
+  answer.runs = { tree: newRun.tree, report: newRun.report };
+  for (const p of plans) {
+    const b = oldRun.files[p.path];
+    const a = newRun.files[p.path];
+    const file = answer.files.find((f) => f.path === p.path);
+    if (!a || !b) throw new Error(`a run file has no result for ${p.path}`);
+    if (!same(a.command, b.command)) throw new Error(`the two runs started ${p.path} with other command lines`);
+    if (a.none || b.none) { Object.assign(file, { result: 'not run', why: a.why ?? b.why }); continue; }
+    if (a.why || b.why) file.why = [b.why && `old code: ${b.why}`, a.why && `new code: ${a.why}`].filter(Boolean).join('; ');
+    const headCases = a.cases ?? {};
+    // A case of the head report that the reader did not find counts as changed.
+    const picked = [...p.picked];
+    const unknown = Object.keys(headCases).filter((name) => !p.known.includes(name));
+    for (const name of p.all ? Object.keys(headCases) : unknown) if (!picked.some((c) => c.name === name)) picked.push({ name, change: 'changed', duplicate: false });
+    if (!picked.length) { answer.cases.push({ path: p.path, name: null, change: 'changed', before: null, after: 'missing', result: 'fails after' }); continue; }
+    for (const c of picked) {
+      const before = b.cases?.[c.name] ?? 'missing';
+      const after = headCases[c.name] ?? 'missing';
+      answer.cases.push({ path: p.path, name: c.name, change: c.change, before, after, result: c.duplicate ? 'duplicate' : resultOf(before, after) });
+    }
+  }
+  return finish(answer);
+}
+
+// Both runs, one after the other, then the answer. For a local run.
+export function failsBefore({ root, base, head = 'HEAD', list = null, work = join(tmpdir(), 'atlas-before') }) {
+  const prepared = plan({ root, base, head, list });
+  if (!prepared.productChanged || !prepared.plans.length) return combine(prepared, null, null);
+  const oldRun = runSide({ ...prepared, side: 'old', work });
+  const newRun = runSide({ ...prepared, side: 'new', work });
+  return combine(prepared, oldRun, newRun);
+}
+
+function finish(answer) {
+  answer.cases.sort((x, y) => (x.path === y.path ? String(x.name).localeCompare(String(y.name)) : x.path < y.path ? -1 : 1));
+  answer.red = answer.cases.some((c) => RED_RESULTS.includes(c.result));
+  answer.proven_count = answer.cases.filter((c) => c.result === 'proven').length;
+  return answer;
+}
+
+// ---------------------------------------------------------------- the command
+
+const oneLine = (text) => String(text ?? '').replace(/[\u0000-\u001f\u007f]/g, ' ');
+
+export function renderText(a) {
+  const out = [`fails-before against ${oneLine(a.base)} (merge base ${a.merge_base.slice(0, 7)}): ${a.proven_count} of ${a.cases.length} case(s) proven, in ${a.files.filter((f) => f.result === 'ran').length} file(s); product code changed: ${a.product_changed ? 'yes' : 'no'}`];
+  for (const c of a.cases) out.push(`  ${c.result.padEnd(14)} ${oneLine(c.path)}  ${oneLine(c.name ?? '(the whole file)')}`);
+  for (const f of a.files.filter((x) => x.result === 'not run')) out.push(`  ${'not run'.padEnd(14)} ${oneLine(f.path)}  ${oneLine(f.why)}`);
+  for (const f of a.files.filter((x) => x.result === 'ran' && x.why)) out.push(`  note           ${oneLine(f.path)}  ${oneLine(f.why)}`);
+  const red = a.cases.filter((c) => RED_RESULTS.includes(c.result));
+  if (red.length) out.push(`RED: ${red.length} case(s): ${[...new Set(red.map((c) => c.result))].join(', ')}. A case that passes on the old code cannot catch this change.`);
+  else if (!a.product_changed) out.push('GREEN: no product code changed, so the new cases are not proven.');
+  else if (a.product_changed && !a.cases.length) out.push('GREEN: product code changed, and no new or changed case runs. Nothing is proven.');
+  else out.push(`GREEN: no red case. Green does not mean proven: ${a.proven_count} proven, ${a.cases.length - a.proven_count} not proven.`);
+  return out;
+}
+
+export function beforeCommand({ base, head, list, out, side, old: oldFile, new: newFile, text = false }, root, line) {
+  try {
+    const write = (value) => {
+      const json = `${JSON.stringify(value, null, 2)}\n`;
+      if (out) writeFileSync(resolve(out), json);
+      else if (!text) process.stdout.write(json);
+    };
+    if (side) {
+      const prepared = plan({ root, base, head: head ?? 'HEAD', list });
+      const run = runSide({ ...prepared, side });
+      write(run);
+      if (text) line(`fails-before: ran the ${side} side of ${Object.keys(run.files).length} file(s).`);
+      return 0;
+    }
+    let answer;
+    if (oldFile || newFile) {
+      const read = (file) => (file ? JSON.parse(readFileSync(resolve(file), 'utf8')) : null);
+      answer = combine(plan({ root, base, head: head ?? 'HEAD', list }), read(oldFile), read(newFile));
+    } else answer = failsBefore({ root, base, head: head ?? 'HEAD', list });
+    write(answer);
+    if (text) for (const l of renderText(answer)) line(l);
+    return answer.red ? 1 : 0;
+  } catch (error) {
+    line(`before: ${error.message}`);
+    return error.usage ? 2 : 1;
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arjunkhera/atlas",
-  "version": "0.3.43",
+  "version": "0.3.44",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, as a Claude Code plugin for any repository.",
   "type": "module",
   "license": "UNLICENSED",

package/skills/lead/SKILL.md

@@ -101,7 +101,7 @@ set. A test keeps the two lists equal.
 | `atlas kind-drift` | When the repo record and `atlas.yaml` may disagree on the kind; this command retires in stage 2, the onboarding stage |
 | `atlas code-read` | To resolve the citations of a code digest |
 | `atlas ste` | Before each publish; `--share` before a page goes to anyone else |
-| `atlas tests` | `write` puts the test kit in an area. `check` runs before a merge. `kept` runs the kept tests of a path 1 area and writes their evidence. `proof`, `verdict`, `named`, `covers` and `rerun` read the results of a run. `changed` lists each test that a pull request changes, as JSON, for the gate's test checks. `sweep` and `issues` run the daily sweep |
+| `atlas tests` | `write` puts the test kit in an area. `check` runs before a merge. `kept` runs the kept tests of a path 1 area and writes their evidence. `proof`, `verdict`, `named`, `covers` and `rerun` read the results of a run. `changed` and `before` run test checks of the gate: the changed tests, and the proof that a new case fails on the old code. `sweep` and `issues` run the daily sweep |
 | `atlas design` | `folder` to learn where designs live; `check` before you show a design; `build` to make its page |
 | `atlas view` | `build --product <id>` makes the project view page of one product, when the owner asks for it or after an Atlas release changes it. Publish it with only the grant the command prints |
 | `atlas upkeep` | `check` before you ask for a merge: the docs index, dead doc links, and the tracker item a pull request names. The daily upkeep run calls it on the main branch |