@arjunkhera/atlas 0.3.6 → 0.3.8

package/tests/link-check.mjs

@@ -0,0 +1,576 @@
+#!/usr/bin/env node
+// The link check (design section 8.2). Exact code, with no model. It runs in
+// CI on every pull request, from the main branch, so a pull request cannot
+// change its own judge. It has three halves.
+//
+//   static  reads text: every scenario, atlas/tests.yaml, the tests and the
+//           support code. It fails on the nine cases below.
+//   lint    the JavaScript half. It fails on a skip, an "only", a todo, a
+//           retry, a soft check, or a test with no assert statement.
+//   run     reads the evidence files of ONE run (--run <id>). With no --run,
+//           it reads the newest run of each scenario, one run per scenario. It fails on an assertion
+//           that the run did not check, on evidence of a changed scenario file
+//           (source_sha256), on a run that used an old case table, and, in the
+//           CI environment, on a scenario that lists ci in runs-in and has no
+//           way in with pass, fail or not exercised.
+//
+// Besides the nine cases, the static half checks the kit folder
+// <area>/test/atlas/ against its manifest.json (a changed or extra file is a
+// finding), and against the kit hashes that a release of Atlas shipped.
+//
+// The nine static cases, with their codes:
+//   1. no-test              an assertion has no test that names its full id
+//   2. unknown-assertion    a test names an assertion that does not exist
+//   3. old-fingerprint      a test names an old fingerprint
+//   4. unknown-name         a name in backticks is not known
+//   5. wait-no-limit        a wait has no time limit
+//   6. when-no-step         a "when" assertion names no step
+//   7. tests-yaml-*         tests.yaml has an unknown field, or holds a secret value
+//   8. persona-shared-secret  two personas share a secret and do not say same-identity
+//   9. table-over-budget    a case table has more cases than the budget
+//
+// Usage:
+//   node link-check.mjs --root <area> [--halves static,lint,run] [--evidence <dir>]
+//                       [--tests <dir>] [--guards-from <tests.yaml of the main branch>]
+//                       [--run <run id>] [--kit-releases <kit-releases.json>]
+// ATLAS_GUARDS_FROM sets --guards-from when the flag is not given.
+// One line for each finding: <file>:<line>  <code>  <message>. Exit 0 when
+// there is none, 1 when there is one, 2 for a bad call.
+import { createHash } from 'node:crypto';
+import { existsSync, lstatSync, readdirSync, readFileSync, statSync } from 'node:fs';
+import { dirname, join, relative, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { parseYaml, YamlError } from './yaml.mjs';
+import { checkTestsYaml, compareGuardParts, resolveEnvironments } from './tests-yaml.mjs';
+import { parseScenario, addFingerprints } from './scenario.mjs';
+
+const FULL_ID = /^([a-z0-9][a-z0-9-]*)\/(e\d+)#([0-9a-f]{6})$/;
+// Folders skipped by name, at any depth. The kit folder and the evidence folder
+// are skipped by exact path only (<area>/test/atlas, <area>/test/evidence): a
+// folder that merely carries the name "atlas" or "evidence" is read like any other.
+const SKIP_DIRS = new Set(['node_modules', '.git']);
+
+// ---------------------------------------------------------------- files
+
+function walk(dir, { skip = SKIP_DIRS, skipPaths = [], accept = () => true } = {}) {
+  if (!existsSync(dir)) return [];
+  const out = [];
+  for (const name of readdirSync(dir).sort()) {
+    const full = join(dir, name);
+    const stat = statSync(full);
+    if (stat.isDirectory()) { if (!skip.has(name) && !skipPaths.includes(full)) out.push(...walk(full, { skip, skipPaths, accept })); }
+    else if (accept(full)) out.push(full);
+  }
+  return out;
+}
+
+const lineAt = (text, index) => text.slice(0, index).split('\n').length;
+
+// Replaces comments, and the insides of strings, with spaces. Line breaks and
+// positions stay, so a match maps back to its line.
+export function stripJs(code) {
+  let out = '';
+  let i = 0;
+  const blank = (s) => s.replace(/[^\n]/g, ' ');
+  while (i < code.length) {
+    const c = code[i];
+    const n = code[i + 1];
+    if (c === '/' && n === '/') { const end = code.indexOf('\n', i); const stop = end < 0 ? code.length : end; out += blank(code.slice(i, stop)); i = stop; continue; }
+    if (c === '/' && n === '*') { const end = code.indexOf('*/', i + 2); const stop = end < 0 ? code.length : end + 2; out += blank(code.slice(i, stop)); i = stop; continue; }
+    if (c === '"' || c === "'" || c === '`') {
+      let j = i + 1;
+      while (j < code.length && code[j] !== c) { if (code[j] === '\\') j += 1; j += 1; }
+      out += c + blank(code.slice(i + 1, j)) + (code[j] ?? '');
+      i = j + 1;
+      continue;
+    }
+    out += c;
+    i += 1;
+  }
+  return out;
+}
+
+// The text from an opening bracket to its match, in already stripped code.
+function balanced(code, from) {
+  const open = code[from];
+  const close = { '(': ')', '{': '}', '[': ']' }[open];
+  let depth = 0;
+  for (let i = from; i < code.length; i += 1) {
+    if (code[i] === open) depth += 1;
+    else if (code[i] === close) { depth -= 1; if (depth === 0) return code.slice(from, i + 1); }
+  }
+  return code.slice(from);
+}
+
+// ---------------------------------------------------------------- the static half
+
+function loadScenarios(root, findings, rel, tests) {
+  const dir = join(root, 'scenarios');
+  const files = walk(dir, { skip: new Set(['node_modules']), accept: (f) => f.endsWith('.md') && !/(^|\/)(map|README)\.md$/.test(f) });
+  const list = [];
+  const ids = new Map();
+  for (const file of files) {
+    const scenario = parseScenario(readFileSync(file, 'utf8'), file);
+    for (const p of scenario.problems) findings.push({ file: rel(file), line: p.line, code: 'scenario-format', message: p.message });
+    if (scenario.id) {
+      if (ids.has(scenario.id)) findings.push({ file: rel(file), line: scenario.fieldLine('id'), code: 'scenario-format', message: `the id "${scenario.id}" is also used in ${rel(ids.get(scenario.id))}` });
+      ids.set(scenario.id, file);
+    }
+    if (tests) addFingerprints(scenario, tests);
+    list.push(scenario);
+  }
+  return list;
+}
+
+function checkNames(scenario, tests, findings, rel) {
+  const file = rel(scenario.file);
+  const t = tests;
+  const base = new Set([
+    ...Object.keys(t.personas ?? {}), ...Object.keys(t['ways-in'] ?? {}), ...Object.keys(t.data ?? {}), ...Object.keys(t.setups ?? {}),
+    ...Object.keys(t.fixtures ?? {}), ...Object.keys(t.judges ?? {}), ...Object.keys(t.secrets ?? {}), ...Object.keys(t['stand-ins'] ?? {}),
+    ...scenario.tables.keys(),
+  ]);
+  const startNames = new Set(scenario.start.filter((s) => s.name).map((s) => s.name));
+  const unknown = (line, name, where) => findings.push({
+    file, line, code: 'unknown-name',
+    message: `the name \`${name}\` in ${where} is not a persona, way in, data rule, setup, fixture, judge, table, start item or kept result`,
+  });
+
+  // The front matter lists.
+  const lists = [['through', t['ways-in']], ['personas', t.personas], ['setups', t.setups], ['fixtures', t.fixtures], ['runs-in', resolveEnvironments(t.environments ?? {})]];
+  for (const [key, known] of lists) {
+    const value = scenario.fields[key];
+    if (value === undefined) continue;
+    for (const name of [].concat(value).map(String)) {
+      if (!Object.hasOwn(known ?? {}, name)) findings.push({ file, line: scenario.fieldLine(key), code: 'unknown-name', message: `"${name}" in ${key}: is not in tests.yaml` });
+    }
+  }
+
+  for (const item of scenario.start) {
+    for (const name of item.names) if (!base.has(name) && !startNames.has(name)) unknown(item.line, name, 'Start with');
+  }
+  const kept = new Set();
+  for (const step of scenario.steps) {
+    for (const k of step.keeps) kept.add(k);
+    for (const name of step.names) if (!base.has(name) && !startNames.has(name) && !kept.has(name)) unknown(step.line, name, `step ${step.n}`);
+  }
+  const allKept = new Set(scenario.steps.flatMap((s) => s.keeps));
+  for (const a of scenario.assertions) {
+    for (const name of a.names) if (!base.has(name) && !startNames.has(name) && !allKept.has(name)) unknown(a.line, name, `assertion ${a.id}`);
+  }
+}
+
+function checkWaitsAndWhens(scenario, findings, rel) {
+  const file = rel(scenario.file);
+  for (const step of scenario.steps) {
+    if (step.wait && !step.wait.limit) {
+      findings.push({ file, line: step.line, code: 'wait-no-limit', message: `step ${step.n} is a wait with no time limit; end it with "Give up after <time>"` });
+    }
+  }
+  if (!scenario.steps.length) return;
+  const keptStep = new Map();
+  for (const step of scenario.steps) for (const k of step.keeps) keptStep.set(k, step.n);
+  for (const a of scenario.assertions) {
+    if (!/^\s*when\b/i.test(a.words)) continue;
+    const refs = [...a.words.matchAll(/\bsteps?\s+(\d+)/gi)].map((m) => Number(m[1]));
+    const named = a.names.some((n) => keptStep.has(n));
+    const ok = refs.length ? refs.every((n) => scenario.steps.some((s) => s.n === n)) : named;
+    if (!ok) findings.push({ file, line: a.line, code: 'when-no-step', message: `${a.id} starts with "when" and names no step that makes it true; write "step N" or name a result that a step keeps` });
+  }
+}
+
+function checkBudget(scenario, tests, findings, rel) {
+  const limit = tests.budget?.cases;
+  if (!Number.isFinite(limit)) return;
+  for (const table of scenario.tables.values()) {
+    if (table.count > limit) findings.push({ file: rel(scenario.file), line: table.line, code: 'table-over-budget', message: `the table ${table.name} makes ${table.count} cases and the budget allows ${limit}` });
+  }
+}
+
+// The ids that tests name. Only a literal first argument can be read.
+const CALL = /\brun\.(check|gate|notExercised|waitFor)\s*\(/g;
+
+export function readRefs(file, code, rel, findings) {
+  const refs = [];
+  for (const m of code.matchAll(CALL)) {
+    const rest = code.slice(m.index + m[0].length);
+    const literal = /^\s*(['"`])([^'"`\n]*)\1/.exec(rest);
+    const line = lineAt(code, m.index);
+    if (!literal || literal[1] === '`' && literal[2].includes('${')) {
+      findings?.push({ file: rel(file), line, code: 'id-not-literal', message: `run.${m[1]} must take the full assertion id as a plain string, so the check can read it` });
+      continue;
+    }
+    const id = FULL_ID.exec(literal[2]);
+    if (!id) { findings?.push({ file: rel(file), line, code: 'bad-id', message: `"${literal[2]}" is not a full assertion id like <scenario>/e1#abc123` }); continue; }
+    refs.push({ file, line, kind: m[1], scenario: id[1], n: id[2], fp: id[3], full: literal[2] });
+  }
+  return refs;
+}
+
+function scenarioTestFiles(root, testsDir) {
+  return walk(join(root, testsDir), { skipPaths: [join(root, testsDir, 'atlas'), join(root, testsDir, 'evidence')], accept: (f) => /\.(m?js|cjs)$/.test(f) });
+}
+
+// ---------------------------------------------------------------- the kit
+
+// The hash of a kit: the sha256 of its files, as sorted "path hash" lines.
+// `files` maps each path to the sha256 of that file.
+export function kitHashOf(files) {
+  const lines = Object.keys(files).sort().map((path) => `${path} ${files[path]}\n`).join('');
+  return createHash('sha256').update(lines).digest('hex');
+}
+
+const sha256Of = (bytes) => createHash('sha256').update(bytes).digest('hex');
+
+// Reads the list of released kit hashes (packages/atlas/shape/kit-releases.json).
+export function loadKitReleases(file) {
+  const data = JSON.parse(readFileSync(file, 'utf8'));
+  return (data.releases ?? []).map((r) => String(r.kit_sha256));
+}
+
+function kitFilesOnDisk(dir, prefix = '') {
+  const out = [];
+  for (const name of readdirSync(dir).sort()) {
+    const full = join(dir, name);
+    const stat = lstatSync(full);
+    if (stat.isSymbolicLink()) out.push({ path: `${prefix}${name}`, link: true });
+    else if (stat.isDirectory()) out.push(...kitFilesOnDisk(full, `${prefix}${name}/`));
+    else out.push({ path: `${prefix}${name}`, full });
+  }
+  return out;
+}
+
+// Every file of <area>/test/atlas/ must match manifest.json, and the manifest
+// must match a kit that a release of Atlas shipped. `kitReleases` is a list of
+// kit hashes; null skips the second test.
+export function checkKit({ root, testsDir, rel, findings, kitReleases = null }) {
+  const dir = join(root, testsDir, 'atlas');
+  if (!existsSync(dir)) return;
+  const manifestPath = join(dir, 'manifest.json');
+  const say = (file, code, message) => findings.push({ file: rel(file), line: 1, code, message });
+  if (!existsSync(manifestPath)) { say(dir, 'kit-no-manifest', 'the kit folder has no manifest.json; run "atlas tests write"'); return; }
+  let manifest;
+  try { manifest = JSON.parse(readFileSync(manifestPath, 'utf8')); } catch { say(manifestPath, 'kit-no-manifest', 'manifest.json is not JSON'); return; }
+  const listed = manifest?.files && typeof manifest.files === 'object' ? manifest.files : {};
+  const onDisk = kitFilesOnDisk(dir).filter((f) => f.path !== 'manifest.json');
+  for (const one of onDisk) {
+    const full = join(dir, one.path);
+    if (one.link) { say(full, 'kit-extra', `${one.path} in the kit folder is a link; the kit holds plain files`); continue; }
+    if (!Object.hasOwn(listed, one.path)) { say(full, 'kit-extra', `${one.path} is in the kit folder and not in manifest.json; the kit folder holds only what "atlas tests write" put there`); continue; }
+    if (sha256Of(readFileSync(one.full)) !== listed[one.path]) say(full, 'kit-changed', `${one.path} differs from manifest.json; it was edited. Restore it with "atlas tests write"`);
+  }
+  const have = new Set(onDisk.map((f) => f.path));
+  for (const path of Object.keys(listed)) if (!have.has(path)) say(join(dir, path), 'kit-missing', `${path} is in manifest.json and not in the kit folder`);
+  if (kitReleases && !kitReleases.includes(kitHashOf(listed))) {
+    say(manifestPath, 'kit-unknown-release', 'the hashes of manifest.json do not match any kit that a release of Atlas shipped; it was edited or came from another source. Restore it with "atlas tests write"');
+  }
+}
+
+function checkLinks(scenarios, root, testsDir, findings, rel) {
+  const byId = new Map(scenarios.filter((s) => s.id).map((s) => [s.id, s]));
+  const refs = [];
+  for (const file of scenarioTestFiles(root, testsDir)) {
+    refs.push(...readRefs(file, readFileSync(file, 'utf8'), rel, findings));
+  }
+  const by = new Map();
+  for (const r of refs) {
+    const s = byId.get(r.scenario);
+    const a = s?.assertions.find((x) => x.id === r.n);
+    if (!s || !a) { findings.push({ file: rel(r.file), line: r.line, code: 'unknown-assertion', message: `${r.full} names ${s ? `the assertion ${r.n}, which ${r.scenario} does not have` : `the scenario ${r.scenario}, which does not exist`}` }); continue; }
+    const key = `${r.scenario}/${r.n}`;
+    if (!by.has(key)) by.set(key, []);
+    by.get(key).push({ ...r, a });
+    if (a.fp && r.fp !== a.fp) findings.push({ file: rel(r.file), line: r.line, code: 'old-fingerprint', message: `${r.full} has an old fingerprint; ${key} now prints #${a.fp}` });
+    else if (r.kind === 'check' && a.gate) findings.push({ file: rel(r.file), line: r.line, code: 'gate-mismatch', message: `${key} is marked gate in the scenario; the test must use run.gate` });
+    else if (r.kind === 'gate' && !a.gate) findings.push({ file: rel(r.file), line: r.line, code: 'gate-mismatch', message: `${key} is not marked gate in the scenario; the test must use run.check` });
+  }
+  for (const s of scenarios) {
+    for (const a of s.assertions) {
+      const named = (by.get(`${s.id}/${a.id}`) ?? []).filter((r) => r.kind !== 'waitFor');
+      if (!named.length) findings.push({ file: rel(s.file), line: a.line, code: 'no-test', message: `no test names ${s.id}/${a.id}#${a.fp ?? '??????'}` });
+    }
+  }
+}
+
+export function checkStatic({ root, testsDir, guardsFrom, rel, findings }) {
+  const yamlPath = join(root, 'atlas', 'tests.yaml');
+  let tests = null;
+  if (!existsSync(yamlPath)) {
+    findings.push({ file: rel(yamlPath), line: 1, code: 'tests-yaml-missing', message: 'there is no atlas/tests.yaml' });
+  } else {
+    try {
+      const parsed = parseYaml(readFileSync(yamlPath, 'utf8'));
+      for (const f of checkTestsYaml(parsed)) findings.push({ file: rel(yamlPath), line: f.line, code: f.code, message: f.message });
+      tests = parsed.value && typeof parsed.value === 'object' ? parsed.value : null;
+      if (guardsFrom) {
+        const baseText = readFileSync(guardsFrom, 'utf8');
+        for (const c of compareGuardParts(baseText, parsed.value)) {
+          const line = parsed.lineOf(c.part.split('.')) ?? 1;
+          findings.push({ file: rel(yamlPath), line, code: 'guard-changed', message: `the guard part "${c.part}" differs from the main branch; a change to a guard part comes to the owner` });
+        }
+      }
+    } catch (error) {
+      if (!(error instanceof YamlError)) throw error;
+      findings.push({ file: rel(yamlPath), line: error.line ?? 1, code: 'tests-yaml', message: `the file cannot be read: ${error.message.replace(/ \(line \d+\)$/, '')}` });
+    }
+  }
+  const scenarios = loadScenarios(root, findings, rel, tests);
+  if (tests) {
+    for (const s of scenarios) {
+      checkNames(s, tests, findings, rel);
+      checkBudget(s, tests, findings, rel);
+    }
+    if (tests['outside-text'] === true) {
+      for (const [name, fx] of Object.entries(tests.fixtures ?? {})) {
+        if (!fx || fx['made-up'] !== true) findings.push({ file: rel(yamlPath), line: parseYaml(readFileSync(yamlPath, 'utf8')).lineOf(['fixtures', name]) ?? 1, code: 'fixture-not-made-up', message: `outside-text is true, so the fixture "${name}" must say made-up: true` });
+      }
+    }
+  }
+  for (const s of scenarios) checkWaitsAndWhens(s, findings, rel);
+  checkLinks(scenarios, root, testsDir, findings, rel);
+  return { tests, scenarios };
+}
+
+// ---------------------------------------------------------------- the lint half
+
+const BANNED = [
+  [/\b(?:test|it|describe|suite)\s*\.\s*(?:skip|only|todo)\b/, 'a skipped, focused or todo test'],
+  [/\b(?:test|it|describe|suite)\s*\(\s*[^,()]*,\s*\{[^}]*\b(?:skip|only|todo)\s*:/, 'a skip, only or todo option on a test'],
+  [/\b(?:t|ctx|context)\s*\.\s*(?:skip|todo)\s*\(/, 'a test that skips itself'],
+  [/\b(?:xit|xtest|xdescribe|fit|fdescribe)\s*\(/, 'a skipped or focused test'],
+  [/\btest\s*\.\s*(?:fail|failing)\b/, 'a test marked as expected to fail'],
+  [/\bretries\s*:|\bretry\s*:|\.retry\s*\(|\bretryTimes\b|\bflaky\s*:\s*true/, 'a retry'],
+  [/\bexpect\s*\.\s*soft\b|\bassert\s*\.\s*soft\b|\bsoftAssert\w*\b/, 'a soft check'],
+  [/\bprocess\s*\.\s*exit\s*\(/, 'an exit from inside a test'],
+];
+const KEYWORDS = new Set(['if', 'for', 'while', 'switch', 'catch', 'function', 'return', 'await', 'async', 'typeof', 'new', 'super']);
+const ASSERTS = /\bassert\b|\bexpect\s*\(|\bshould\b/;
+
+// Names of functions whose body holds an assertion.
+function assertingNames(code) {
+  const names = new Set();
+  const patterns = [
+    /\bfunction\s*\*?\s*([A-Za-z_$][\w$]*)\s*\(/g,
+    /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?(?:function\b[^(]*)?\(/g,
+    /\b(?:const|let|var)\s+([A-Za-z_$][\w$]*)\s*=\s*(?:async\s*)?[A-Za-z_$][\w$]*\s*=>/g,
+    /^\s*(?:async\s+)?([A-Za-z_$][\w$]*)\s*\([^)]*\)\s*\{/gm,
+    /\b([A-Za-z_$][\w$]*)\s*:\s*(?:async\s*)?\([^)]*\)\s*=>/g,
+  ];
+  for (const re of patterns) {
+    for (const m of code.matchAll(re)) {
+      if (KEYWORDS.has(m[1])) continue;
+      const after = m.index + m[0].length;
+      const brace = code.indexOf('{', m.index);
+      const arrow = code.indexOf('=>', m.index);
+      let body;
+      if (m[0].endsWith('{')) body = balanced(code, after - 1);
+      else if (arrow >= 0 && arrow < after + 400) {
+        const start = arrow + 2;
+        body = /^\s*\{/.test(code.slice(start)) ? balanced(code, code.indexOf('{', start)) : code.slice(start, code.indexOf('\n', start) < 0 ? undefined : code.indexOf('\n', start));
+      } else if (brace >= 0) body = balanced(code, brace);
+      else continue;
+      if (ASSERTS.test(body)) names.add(m[1]);
+    }
+  }
+  return names;
+}
+
+function lintFiles(root, testsDir) {
+  const base = join(root, testsDir);
+  const all = scenarioTestFiles(root, testsDir);
+  const text = new Map(all.map((f) => [f, readFileSync(f, 'utf8')]));
+  const seeds = all.filter((f) => /\bscenario\s*\(/.test(text.get(f)) || /(^|\/)(support|scenarios)\//.test(f.slice(base.length)));
+  const picked = new Set(seeds);
+  const queue = [...seeds];
+  while (queue.length) {
+    const file = queue.pop();
+    for (const m of text.get(file).matchAll(/\bfrom\s+['"](\.{1,2}\/[^'"]+)['"]|\bimport\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]/g)) {
+      const target = resolve(dirname(file), m[1] ?? m[2]);
+      if (text.has(target) && !picked.has(target)) { picked.add(target); queue.push(target); }
+    }
+  }
+  return { files: [...picked].sort(), text, all };
+}
+
+export function checkLint({ root, testsDir, rel, findings }) {
+  const { files, text, all } = lintFiles(root, testsDir);
+  const asserting = new Set();
+  for (const f of all) for (const n of assertingNames(stripJs(text.get(f)))) asserting.add(n);
+  const holds = (body) => ASSERTS.test(body) || [...asserting].some((n) => new RegExp(`(?<![\\w$.])${n.replace(/\$/g, '\\$')}\\s*\\(|\\.${n.replace(/\$/g, '\\$')}\\s*\\(`).test(body));
+  for (const file of files) {
+    const raw = text.get(file);
+    const code = stripJs(raw);
+    for (const [pattern, what] of BANNED) {
+      const re = new RegExp(pattern.source, 'g');
+      for (const m of code.matchAll(re)) findings.push({ file: rel(file), line: lineAt(code, m.index), code: 'lint', message: `${what}` });
+    }
+    // A check or a gate must hold an assert statement.
+    for (const m of code.matchAll(/\brun\s*\.\s*(check|gate)\s*\(/g)) {
+      const open = m.index + m[0].length - 1;
+      const body = balanced(code, open);
+      const afterId = body.indexOf(',');
+      if (afterId < 0 || !holds(body.slice(afterId + 1))) findings.push({ file: rel(file), line: lineAt(code, m.index), code: 'lint', message: `the run.${m[1]} holds no assert statement` });
+    }
+    // An ordinary test (test or it, not a scenario) must hold one too.
+    for (const m of code.matchAll(/(?<![\w$.])(?:test|it)\s*\(/g)) {
+      const body = balanced(code, m.index + m[0].length - 1);
+      if (!holds(body)) findings.push({ file: rel(file), line: lineAt(code, m.index), code: 'lint', message: 'a test with no assert statement' });
+    }
+  }
+}
+
+// ---------------------------------------------------------------- the run half
+
+// The run ids in an evidence folder, newest first. A run is a sub-folder that
+// holds evidence files; its time is the newest `started` of its files.
+export function runIds(evidenceDir) {
+  if (!existsSync(evidenceDir)) return [];
+  const runs = [];
+  for (const name of readdirSync(evidenceDir)) {
+    const dir = join(evidenceDir, name);
+    if (!statSync(dir).isDirectory()) continue;
+    let newest = null;
+    for (const file of readdirSync(dir)) {
+      if (!file.endsWith('.json')) continue;
+      try {
+        const data = JSON.parse(readFileSync(join(dir, file), 'utf8'));
+        if (data?.evidence === 1 && (newest === null || String(data.started) > newest)) newest = String(data.started);
+      } catch { /* unreadable files are reported later */ }
+    }
+    if (newest !== null) runs.push({ name, newest });
+  }
+  return runs.sort((x, y) => (x.newest < y.newest ? 1 : x.newest > y.newest ? -1 : x.name < y.name ? 1 : -1)).map((r) => r.name);
+}
+
+export const newestRun = (evidenceDir) => runIds(evidenceDir)[0] ?? null;
+
+// With no run id, each scenario uses its own newest run: the run folder that
+// holds its newest evidence file. Evidence of a fault run does not count.
+// Returns a Map of scenario id to run id.
+export function newestRunPerScenario(evidenceDir) {
+  const best = new Map();
+  if (!existsSync(evidenceDir)) return new Map();
+  for (const name of readdirSync(evidenceDir)) {
+    const dir = join(evidenceDir, name);
+    if (!statSync(dir).isDirectory()) continue;
+    for (const file of readdirSync(dir)) {
+      if (!file.endsWith('.json')) continue;
+      let data;
+      try { data = JSON.parse(readFileSync(join(dir, file), 'utf8')); } catch { continue; }
+      if (data?.evidence !== 1 || data.fault_run || !data.scenario) continue;
+      const started = String(data.started);
+      const before = best.get(data.scenario);
+      if (!before || started > before.started || (started === before.started && name > before.name)) best.set(data.scenario, { started, name });
+    }
+  }
+  return new Map([...best].map(([scenario, b]) => [scenario, b.name]));
+}
+
+// The run half reads the files of one run only, and only evidence of the
+// scenario file as it is now (source_sha256).
+export function checkRun({ evidenceDir, runId = null, environment = null, scenarios, rel, findings }) {
+  const records = [];
+  const runOf = new Map();
+  const perScenario = runId === null ? newestRunPerScenario(evidenceDir) : null;
+  const names = runId !== null ? [runId] : [...new Set(perScenario.values())];
+  for (const id of names) {
+    const runDir = join(evidenceDir, id);
+    if (!existsSync(runDir)) continue;
+    for (const name of readdirSync(runDir).sort()) {
+      if (!name.endsWith('.json')) continue;
+      const file = join(runDir, name);
+      let data;
+      try { data = JSON.parse(readFileSync(file, 'utf8')); } catch { findings.push({ file: rel(file), line: 1, code: 'evidence-unreadable', message: 'the evidence file is not JSON' }); continue; }
+      if (data?.evidence !== 1) continue;
+      if (data.fault_run) continue;
+      if (perScenario && perScenario.get(data.scenario) !== id) continue;
+      records.push({ file, data });
+      runOf.set(data.scenario, id);
+    }
+  }
+  const reached = new Set(['pass', 'fail', 'not exercised', 'not here']);
+  const ran = new Set(['pass', 'fail', 'not exercised']);
+  for (const s of scenarios) {
+    if (!s.id) continue;
+    const all = records.filter((r) => r.data.scenario === s.id);
+    const id = runId ?? runOf.get(s.id) ?? null;
+    const mine = all.filter((r) => r.data.source_sha256 === s.sha256);
+    if (all.length && !mine.length) {
+      findings.push({ file: rel(s.file), line: 1, code: 'stale-evidence', message: `the evidence of run ${id} for ${s.id} was made from another version of this scenario file; run it again` });
+      continue;
+    }
+    for (const a of s.assertions) {
+      for (const way of s.through) {
+        const seen = mine.some((r) => r.data.assertions?.some((e) => e.id === a.fullId && e.way === way && reached.has(e.result)));
+        if (!seen) findings.push({ file: rel(s.file), line: a.line, code: 'not-checked', message: `no run checked ${s.id}/${a.id}#${a.fp} through ${way}` });
+      }
+    }
+    if (mine.length && environment === 'ci' && [].concat(s.fields['runs-in'] ?? []).map(String).includes('ci')) {
+      const did = mine.some((r) => r.data.assertions?.some((e) => ran.has(e.result)));
+      if (!did) findings.push({ file: rel(s.file), line: s.fieldLine('runs-in'), code: 'ci-not-run', message: `${s.id} lists ci in runs-in, and no way in has pass, fail or not exercised in run ${id}` });
+    }
+    const newest = [...mine].sort((x, y) => String(y.data.started).localeCompare(String(x.data.started)))[0];
+    for (const [name, fp] of Object.entries(newest?.data.tables ?? {})) {
+      const now = s.tables.get(name);
+      if (now && now.fingerprint !== fp) findings.push({ file: rel(s.file), line: now.line, code: 'stale-table', message: `the newest run of ${s.id} read the table ${name} as #${fp}; it now prints #${now.fingerprint}; run it again` });
+    }
+  }
+}
+
+// ---------------------------------------------------------------- the whole check
+
+export const HALVES = Object.freeze(['static', 'lint', 'run']);
+
+// Returns { findings: [{ file, line, code, message }], scenarios }.
+export function checkArea({ root, halves = HALVES, evidence, tests = 'test', guardsFrom = process.env.ATLAS_GUARDS_FROM || null, run = null, environment = process.env.ATLAS_ENV || (process.env.CI ? 'ci' : 'local'), kitReleases = null, cwd = process.cwd() } = {}) {
+  const abs = resolve(root ?? '.');
+  const rel = (p) => { const r = relative(cwd, p); return r.startsWith('..') ? p : (r || '.'); };
+  const findings = [];
+  const bad = halves.filter((h) => !HALVES.includes(h));
+  if (bad.length) throw new Error(`there is no half "${bad[0]}"; use ${HALVES.join(', ')}`);
+  if (!existsSync(abs)) throw new Error(`${abs} does not exist`);
+  // The scenarios are read for every half; their findings only show with `static`.
+  const sink = halves.includes('static') ? findings : [];
+  const staticResult = checkStatic({ root: abs, testsDir: tests, guardsFrom, rel, findings: sink });
+  const { scenarios } = staticResult;
+  if (halves.includes('static')) checkKit({ root: abs, testsDir: tests, rel, findings, kitReleases });
+  if (halves.includes('lint')) checkLint({ root: abs, testsDir: tests, rel, findings });
+  if (halves.includes('run')) checkRun({ evidenceDir: resolve(evidence ?? join(abs, tests, 'evidence')), runId: run, environment, scenarios, rel, findings });
+  findings.sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : a.line - b.line));
+  return { findings, scenarios };
+}
+
+export const formatFinding = (f) => `${f.file}:${f.line}  ${f.code}  ${f.message}`;
+
+// ---------------------------------------------------------------- the command line
+
+export function parseArgs(argv) {
+  const out = { halves: [...HALVES] };
+  const takes = new Set(['root', 'halves', 'evidence', 'tests', 'guards-from', 'run', 'kit-releases']);
+  for (let i = 0; i < argv.length; i += 1) {
+    const word = argv[i];
+    if (!word.startsWith('--')) throw new Error(`unexpected argument "${word}"`);
+    const key = word.slice(2);
+    if (!takes.has(key)) throw new Error(`there is no flag "--${key}"`);
+    const value = argv[i + 1];
+    if (value === undefined || value.startsWith('--')) throw new Error(`the flag "--${key}" needs a value`);
+    i += 1;
+    if (key === 'halves') out.halves = value.split(',').map((s) => s.trim()).filter(Boolean);
+    else out[key === 'guards-from' ? 'guardsFrom' : key === 'kit-releases' ? 'kitReleases' : key] = value;
+  }
+  return out;
+}
+
+export function main(argv, write = (text) => process.stdout.write(text)) {
+  let args;
+  try { args = parseArgs(argv); } catch (error) { process.stderr.write(`link-check: ${error.message}\n`); return 2; }
+  if (!args.root) { process.stderr.write('link-check: --root <area> is needed\n'); return 2; }
+  let result;
+  try { if (args.kitReleases) args.kitReleases = loadKitReleases(args.kitReleases); result = checkArea(args); } catch (error) { process.stderr.write(`link-check: ${error.message}\n`); return 2; }
+  for (const f of result.findings) write(`${formatFinding(f)}\n`);
+  write(`\n${result.scenarios.length} scenario(s) read; halves: ${args.halves.join(', ')}; ${result.findings.length} finding(s).\n`);
+  write(result.findings.length ? 'RED\n' : 'GREEN\n');
+  return result.findings.length ? 1 : 0;
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) process.exitCode = main(process.argv.slice(2));

package/tests/procs.mjs

@@ -0,0 +1,43 @@
+// Child processes that the library starts: the product, a stand-in, an MCP
+// server. Each one runs in its own process group, so a stop reaches the whole
+// tree. Whatever is still running when this process ends is killed.
+import { spawn, spawnSync } from 'node:child_process';
+import { setTimeout as sleep } from 'node:timers/promises';
+
+const live = new Set();
+let hooked = false;
+
+export function killGroup(child, signal) {
+  try { process.kill(-child.pid, signal); } catch { try { child.kill(signal); } catch { /* it is gone */ } }
+}
+
+// Stops what is still running when this process ends, even on a crash.
+function hook() {
+  if (hooked) return;
+  hooked = true;
+  process.once('exit', () => { for (const child of live) killGroup(child, 'SIGKILL'); });
+}
+
+export function spawnManaged({ command, cwd, env, redactor, stdin = false }) {
+  hook();
+  const child = spawn('sh', ['-c', command], { cwd, env, detached: true, stdio: [stdin ? 'pipe' : 'ignore', 'pipe', 'pipe'] });
+  live.add(child);
+  let tail = '';
+  const keep = (chunk) => { tail = (tail + chunk.toString('utf8')).slice(-4000); };
+  child.stdout.on('data', keep);
+  child.stderr.on('data', keep);
+  const exited = new Promise((resolve) => child.once('exit', (code, signal) => { live.delete(child); resolve({ code, signal }); }));
+  child.once('error', () => live.delete(child));
+  return {
+    child, exited,
+    tail: () => (redactor ? redactor.text(tail) : tail).trim(),
+    async stop(stopCommand) {
+      if (child.exitCode !== null || child.signalCode !== null) return;
+      if (stopCommand && stopCommand !== 'signal') spawnSync('sh', ['-c', stopCommand], { cwd, env, timeout: 10000 });
+      killGroup(child, 'SIGTERM');
+      const done = await Promise.race([exited.then(() => true), sleep(5000, undefined, { ref: false }).then(() => false)]);
+      if (!done) { killGroup(child, 'SIGKILL'); await Promise.race([exited, sleep(2000, undefined, { ref: false })]); }
+    },
+  };
+}
+