@arjunkhera/atlas 0.3.31 → 0.3.32

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "atlas",
-  "version": "0.3.31",
+  "version": "0.3.32",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, for any repository.",
   "author": {
     "name": "Arjun Khera"

package/door/cli.mjs

@@ -30,6 +30,7 @@ import { loadPrivateTerms } from './lib/privacy.mjs';
 import { checkCommand as testsCheck, writeCommand as testsWrite } from './lib/tests.mjs';
 import { proofCommand as testsProof } from './lib/proof.mjs';
 import { verdictCommand as testsVerdict, namedCommand as testsNamed, rerunCommand as testsRerun } from '../tests/verdict.mjs';
+import { runKept } from '../tests/kept.mjs';
 import { sweepCommand as testsSweep, coversCommand as testsCovers } from '../tests/sweep.mjs';
 import { issuesCommand as testsIssues } from './lib/issues.mjs';
 import { upkeepCommand } from './lib/upkeep.mjs';
@@ -97,6 +98,10 @@ const HELP = `atlas — the door into a repo's Atlas files
       --dry-run                        print what it would open; change nothing
                                        The token is only GITHUB_TOKEN. The base URL is GITHUB_API_URL (default https://api.github.com)
 
+  atlas tests kept --root <area>       path 1: run the kept tests that a scenario names in kept-tests, and write their evidence
+      --evidence <dir>                 where the evidence files go; default <area>/test/evidence
+      --scenario <id>                  run one scenario only
+
   atlas tests rerun --root <area> --run <id>   list the scenario test files whose first attempt did not end well
       --tests <dir>                    the test folder of the area; default test. The test files are in <dir>/scenarios
       --evidence <dir>                 where the evidence files are; default <area>/test/evidence
@@ -131,7 +136,7 @@ export const FLAGS = Object.freeze({
   ste: { share: 'switch', terms: 'value' },
   design: { tracker: 'value', out: 'value', draft: 'switch', root: 'optional' },
   view: { product: 'value', name: 'value', out: 'value' },
-  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch' },
+  tests: { root: 'optional', halves: 'value', evidence: 'value', 'guards-from': 'value', tests: 'value', 'dry-run': 'switch', run: 'value', own: 'value', base: 'value', covers: 'switch', findings: 'value', repo: 'value', max: 'value', bot: 'value', since: 'value', out: 'value', text: 'switch', json: 'switch', scenario: 'value' },
   upkeep: { root: 'optional', event: 'value', json: 'switch' },
   install: { local: 'switch', from: 'value', 'skip-global': 'switch', yes: 'switch' },
   upgrade: { version: 'optional', yes: 'switch' },
@@ -330,9 +335,10 @@ async function testsCommand(chosen) {
   if (what === 'named') return testsNamed({ base: chosen.base, covers: Boolean(chosen.covers) }, root, line);
   if (what === 'sweep') return testsSweep({ evidence: chosen.evidence, since: chosen.since, base: chosen.base, out: chosen.out, text: Boolean(chosen.text) }, root, line);
   if (what === 'covers') return testsCovers({ itemId: chosen._[2], evidence: chosen.evidence, json: Boolean(chosen.json) }, root, line);
+  if (what === 'kept') return (await runKept({ root, evidenceDir: chosen.evidence, only: chosen.scenario ?? null, write: (text) => process.stdout.write(text) })).exit;
   if (what === 'rerun') return testsRerun({ run: chosen.run, evidence: chosen.evidence, tests: chosen.tests }, root, line);
   if (what === 'issues') return testsIssues({ findings: chosen.findings, repo: chosen.repo, max: chosen.max, bot: chosen.bot, dryRun: Boolean(chosen['dry-run']), args: [...chosen._, ...Object.values(chosen).filter((v) => typeof v === 'string')] }, line);
-  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict, named, sweep, covers, issues or rerun.`);
+  throw new Error(`there is no tests verb "${what ?? ''}". Use check, write, proof, verdict, named, sweep, covers, kept, issues or rerun.`);
 }
 
 function upkeepVerb(chosen) {

package/door/kit-releases.json

@@ -24,6 +24,10 @@
     {
       "kit_sha256": "1e62be9a3fb632161c593d44ac7ad404ef04186fe2fbcd108645fbe1559bf860",
       "first_package": "0.3.28"
+    },
+    {
+      "kit_sha256": "08c868aa39e6c9f471da2571e9a1d5939932a59856cea9c4176e3173f93454db",
+      "first_package": "0.3.30"
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@arjunkhera/atlas",
-  "version": "0.3.31",
+  "version": "0.3.32",
   "description": "Atlas: the delivery lifecycle, its crews and the Atlas tools, as a Claude Code plugin for any repository.",
   "type": "module",
   "license": "UNLICENSED",

package/skills/lead/SKILL.md

@@ -100,7 +100,7 @@ set. A test keeps the two lists equal.
 | `atlas kind-drift` | When the repo record and `atlas.yaml` may disagree on the kind; this command retires in stage 2, the onboarding stage |
 | `atlas code-read` | To resolve the citations of a code digest |
 | `atlas ste` | Before each publish; `--share` before a page goes to anyone else |
-| `atlas tests` | `write` puts the test kit in an area. `check` runs before a merge. `proof`, `verdict`, `named`, `covers` and `rerun` read the results of a run. `sweep` and `issues` run the daily sweep |
+| `atlas tests` | `write` puts the test kit in an area. `check` runs before a merge. `kept` runs the kept tests of a path 1 area and writes their evidence. `proof`, `verdict`, `named`, `covers` and `rerun` read the results of a run. `sweep` and `issues` run the daily sweep |
 | `atlas design` | `folder` to learn where designs live; `check` before you show a design; `build` to make its page |
 | `atlas view` | `build --product <id>` makes the project view page of one product, when the owner asks for it or after an Atlas release changes it. Publish it with only the grant the command prints |
 | `atlas upkeep` | `check` before you ask for a merge: the docs index, dead doc links, and the tracker item a pull request names. The daily upkeep run calls it on the main branch |

package/skills/tests/SKILL.md

@@ -192,10 +192,12 @@ CI runs the tests from the main branch, so a pull request cannot change its own
 
 ## Onboard an area
 
-Path 1 is for an area that has integration tests which call its routes,
-and which people keep using. Old unit tests that call functions do not make path 1.
-Use path 2: the area has no integration test that calls its routes. If it has
-route tests that people keep, stop. Tell the owner that path 1 comes later.
+Pick the path at step 1 of the map.
+
+- Path 1: the area has integration tests that people keep using. They call
+  its routes, or start its tool or its server. Follow "Path 1: adopt the
+  kept tests" below. Old unit tests that only call functions do not make path 1.
+- Path 2: the area has no such tests. Follow the steps here.
 
 1. Send the `atlas:code-explorer` crew to map the area. Ask for routes,
    tools, pages, storage, calls to outside services and existing tests.
@@ -230,6 +232,56 @@ a note service:
    and two scenarios.
 3. Output: one pull request. The owner reads two scenarios and three lines of guards.
 
+## Path 1: adopt the kept tests
+
+The kept tests stay as they are. Each kept test file gets a scenario: the
+scenario holds the words, and the kept test stays the code that checks them.
+`atlas tests kept` runs the kept runner, reads its JUnit report and writes
+the evidence. The design: `docs/design-docs/kept-tests/` in the Atlas repo.
+
+1. Send the `atlas:code-explorer` crew to map the area. Ask for each kept
+   test file and its test case names. Ask for the runner command and the env
+   variables it reads. Ask for each outside address that a kept test uses.
+2. Run the kept tests once as they are. Record the count. Each one must be
+   green before you start, and stays green to the end.
+3. Run `atlas tests write --root <repo>`.
+4. Find the JUnit form of the runner. Examples: `node --test
+   --test-reporter=junit --test-reporter-destination={report} {file}`,
+   `pytest --junitxml={report} {file}`, and a TestNG suite file with the
+   JUnit reporter.
+5. Draft `atlas/tests.yaml` with one way in of driver `kept`. Give it the
+   `command` with `{file}` and `{report}`, the `files` patterns, and the
+   `env` the kept tests need. Give it a local address in the env, or
+   `offline: true` when the tests reach no network.
+6. For each kept file, write one scenario with `through: [kept]` and
+   `kept-tests: <file>`. Write the assertions: what the file proves, in
+   plain words. Never claim more than its test cases check.
+7. Write the `## Kept checks` section. Map each assertion to the test cases
+   that check it: `- e1: "case name"`. Name each case of the file once.
+   List a case that proves no promise as `- not mapped: "case name" (why)`.
+8. Review the words (the section below). Give the crew the kept test file
+   too, so it can check each assertion against the code of its cases.
+9. Run `atlas tests kept --root <repo>`, then `atlas tests check --root <repo>`.
+10. Plant a fault in the product for each assertion, in a throwaway copy.
+    Run `atlas tests kept` there. Each assertion must go red once.
+11. Run the kept tests as they are once more. The count and the result
+    must match step 2.
+12. Add a CI step that runs `node test/atlas/kept.mjs --root .` from the
+    kit copy of the main branch, with `ATLAS_GUARDS_FROM` set.
+13. Open one pull request. List the outside addresses that step 1 found.
+    Give the guard parts in plain words, and one scenario with its proof
+    table. A person merges it.
+
+Worked run, on a CLI area with 14 kept test files:
+
+1. Input: the map finds `test/status.test.mjs` with 4 test cases. They start
+   the tool as a child process. No test reaches a network.
+2. Process: one kept way in with `offline: true`. The scenario
+   `thread-status` has two assertions. e1 maps 2 cases and e2 maps 1 case.
+   The help-text case is not mapped.
+3. Output: `atlas tests kept` prints `thread-status  pass`. A planted fault
+   in the state code turns e1 red, and e2 stays green.
+
 ## Write assertions
 
 Use these steps when you write or change an assertion.

package/tests/drivers/index.mjs

@@ -64,6 +64,8 @@ export async function makeDriver(ctx) {
         env: Object.fromEntries(Object.entries(spec.env ?? {}).map(([k, v]) => [k, fill(String(v))])),
         offline: spec.offline === true, hosts, root, productRoot, runId, redactor, record,
       });
+    case 'kept':
+      throw new Blocked(`the way in "${way}" is a kept runner; its scenarios run with atlas tests kept, not in a scenario test`);
     case 'function': {
       let target = adapter.functions?.[way] ?? adapter.functions?.function ?? adapter.functions;
       // No module from the adapter: the `module` of the way in is a path under the

package/tests/junit.mjs

@@ -0,0 +1,64 @@
+// The JUnit report reader, for path 1 of onboarding: an area whose kept tests
+// run on their own runner (design docs/design-docs/kept-tests). Most runners
+// write this report: node --test (--test-reporter=junit), pytest
+// (--junitxml), TestNG, JUnit, Vitest and Jest.
+//
+//   readJunit(xml) -> [{ name, classname, file, suite, result, message }]
+//   result: "pass", "fail" or "skipped". An <error> is a fail too.
+//
+// The reader has no dependency. It reads only the parts it needs: the
+// <testsuite> nesting, each <testcase> and its <failure>, <error> and
+// <skipped> children. It refuses text that holds no <testsuites> or
+// <testsuite> element, so a broken report is not read as "no tests".
+export class ReportError extends Error {}
+
+const ENTITIES = { lt: '<', gt: '>', amp: '&', quot: '"', apos: "'" };
+export const unescapeXml = (text) => String(text).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, code) => {
+  if (code[0] === '#') return String.fromCodePoint(code[1].toLowerCase() === 'x' ? parseInt(code.slice(2), 16) : Number(code.slice(1)));
+  return ENTITIES[code.toLowerCase()] ?? whole;
+});
+
+function attributes(text) {
+  const out = {};
+  for (const m of String(text).matchAll(/([A-Za-z_:][\w:.-]*)\s*=\s*("([^"]*)"|'([^']*)')/g)) out[m[1]] = unescapeXml(m[3] ?? m[4]);
+  return out;
+}
+
+// One tag at a time: comments, the XML head and CDATA are skipped.
+const TAG = /<!--[\s\S]*?-->|<\?[\s\S]*?\?>|<!\[CDATA\[[\s\S]*?\]\]>|<(\/?)([A-Za-z_][\w:.-]*)((?:\s+[^\s=/>]+\s*=\s*(?:"[^"]*"|'[^']*'))*)\s*(\/?)>/g;
+
+export function readJunit(xml) {
+  const text = String(xml ?? '');
+  if (!/<testsuites?[\s>/]/.test(text)) throw new ReportError('the report holds no <testsuites> or <testsuite> element');
+  const cases = [];
+  const suites = [];
+  let current = null;
+  for (const m of text.matchAll(TAG)) {
+    const [, closing, tag, attrs, selfClosing] = m;
+    if (!tag) continue;
+    if (tag === 'testsuite') {
+      if (closing) suites.pop();
+      else if (!selfClosing) suites.push(attributes(attrs));
+      continue;
+    }
+    if (tag === 'testcase') {
+      if (closing) { current = null; continue; }
+      const a = attributes(attrs);
+      const suite = suites.map((s) => s.name).filter(Boolean).join(' > ');
+      const file = a.file ?? suites.map((s) => s.file).filter(Boolean).at(-1) ?? null;
+      const one = { name: a.name ?? '', classname: a.classname ?? null, file, suite: suite || null, result: 'pass', message: null };
+      cases.push(one);
+      current = selfClosing ? null : one;
+      continue;
+    }
+    if (!current || closing) continue;
+    if (tag === 'failure' || tag === 'error') {
+      current.result = 'fail';
+      current.message ??= attributes(attrs).message ?? tag;
+    } else if (tag === 'skipped' && current.result !== 'fail') {
+      current.result = 'skipped';
+      current.message ??= attributes(attrs).message ?? null;
+    }
+  }
+  return cases;
+}

package/tests/kept.mjs

@@ -0,0 +1,185 @@
+#!/usr/bin/env node
+// Path 1 of onboarding: the tests an area already keeps (design
+// docs/design-docs/kept-tests). The kept tests are never edited. For each
+// scenario with `kept-tests: <file>` in its front matter, this:
+//   1. runs the kept runner of the scenario's `kept` way in, once, for that
+//      file. The way in's command holds {file} and {report}. The run goes
+//      through the command driver, so the runner gets the allow-list env, the
+//      env of the way in, a throwaway HOME, and the address guards;
+//   2. reads the JUnit report that the runner wrote (junit.mjs);
+//   3. sets the result of each assertion from the test cases that its
+//      "Kept checks" lines name: every case passed is `pass`; a failed case is
+//      `fail`; a skipped case, a case that is not in the report, or an
+//      assertion that names no case is `not checked`;
+//   4. writes one evidence file for the scenario, in the same schema and run
+//      folder as a scenario test. `atlas tests check`, `proof`, `verdict`,
+//      `covers` and `sweep` read it as they read any other.
+// A case of the report that no line maps, and that no "not mapped" line lists,
+// is a finding: every kept test case is named once.
+//
+// Usage:
+//   node kept.mjs --root <area> [--evidence <dir>] [--scenario <id>]
+// ATLAS_ENV, ATLAS_RUN_ID, ATLAS_EVIDENCE_DIR and ATLAS_GUARDS_FROM work as for a scenario test.
+// Exit 0 when every assertion passed and there is no finding, 1 otherwise, 2 for a bad call.
+import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, relative, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { Blocked } from './errors.mjs';
+import { Redactor } from './redact.mjs';
+import { readTestsYaml, checkTestsYaml, compareGuardParts } from './tests-yaml.mjs';
+import { readScenario, addFingerprints, normalize } from './scenario.mjs';
+import { Evidence } from './evidence.mjs';
+import { makeRunId } from './fresh.mjs';
+import { hostsOf } from './guards.mjs';
+import { parseLimit } from './wait.mjs';
+import { makeDriver } from './drivers/index.mjs';
+import { shellWord } from './drivers/command.mjs';
+import { readJunit } from './junit.mjs';
+
+function scenarioFiles(dir) {
+  if (!existsSync(dir)) return [];
+  const out = [];
+  for (const name of readdirSync(dir).sort()) {
+    const full = join(dir, name);
+    if (statSync(full).isDirectory()) { if (name !== 'node_modules') out.push(...scenarioFiles(full)); continue; }
+    if (name.endsWith('.md') && name !== 'map.md' && name !== 'README.md') out.push(full);
+  }
+  return out;
+}
+
+// The result of one assertion from the cases it names.
+export function resultOf(named, byName) {
+  if (!named.length) return { result: 'not checked', cases: [], why: 'the assertion names no kept test case' };
+  const cases = named.map(({ name }) => {
+    const found = byName.get(normalize(name)) ?? [];
+    if (!found.length) return { name, result: 'missing', message: 'not in the report' };
+    const failed = found.find((c) => c.result === 'fail');
+    const skipped = found.find((c) => c.result === 'skipped');
+    return { name, result: failed ? 'fail' : skipped ? 'skipped' : 'pass', message: (failed ?? skipped)?.message ?? null };
+  });
+  if (cases.some((c) => c.result === 'fail')) return { result: 'fail', cases };
+  if (cases.some((c) => c.result !== 'pass')) return { result: 'not checked', cases, why: 'a case was skipped or is not in the report' };
+  return { result: 'pass', cases };
+}
+
+export async function runKept({ root, evidenceDir, environment, runId = makeRunId(), guardsFrom = process.env.ATLAS_GUARDS_FROM, only = null, adapter = {}, write = () => {} } = {}) {
+  root = resolve(root);
+  evidenceDir = resolve(evidenceDir ?? process.env.ATLAS_EVIDENCE_DIR ?? join(root, 'test', 'evidence'));
+  environment ??= process.env.ATLAS_ENV ?? (process.env.CI ? 'ci' : 'local');
+  if (!existsSync(join(root, 'atlas', 'tests.yaml'))) {
+    write(`there is no atlas/tests.yaml under ${root}; path 1 needs a kept way in there\n`);
+    return { runId, results: [], findings: ['no atlas/tests.yaml'], exit: 1 };
+  }
+  const yaml = readTestsYaml(join(root, 'atlas', 'tests.yaml'));
+  const bad = checkTestsYaml(yaml);
+  if (bad.length) throw new Error(`atlas/tests.yaml: ${bad.map((f) => `line ${f.line}: ${f.message}`).join('; ')}`);
+  const tests = yaml.value;
+  let refusal = null;
+  if (guardsFrom) {
+    try {
+      const changes = compareGuardParts(readFileSync(guardsFrom, 'utf8'), tests);
+      if (changes.length) refusal = `the guard parts of atlas/tests.yaml differ from the main branch copy (${changes.map((c) => c.part).join(', ')}); the run is refused`;
+    } catch (error) { refusal = `ATLAS_GUARDS_FROM names a file that cannot be read as tests.yaml (${error.code ?? error.message}); the run is refused`; }
+  }
+  const findings = [];
+  const results = [];
+  for (const file of scenarioFiles(join(root, 'scenarios'))) {
+    const sc = readScenario(file);
+    if (!sc.keptFile || (only && sc.id !== only)) continue;
+    const rel = relative(root, file);
+    if (sc.problems.length) { findings.push(`${rel}: ${sc.problems.map((p) => `line ${p.line}: ${p.message}`).join('; ')}`); continue; }
+    addFingerprints(sc, tests);
+    const way = sc.through.find((w) => tests['ways-in']?.[w]?.driver === 'kept');
+    if (!way) { findings.push(`${rel}: through names no way in with driver kept`); continue; }
+    const spec = tests['ways-in'][way];
+    const redactor = new Redactor();
+    const evidence = new Evidence({ evidenceDir, runId, environment, scenario: sc, root, redactor });
+    const one = { scenario: sc.id, way, verdict: null, assertions: [] };
+    results.push(one);
+    const started = new Date().toISOString();
+    const block = (reason) => {
+      evidence.setWay(way, { verdict: 'blocked', reason, started, ended: new Date().toISOString() });
+      for (const a of sc.assertions) one.assertions.push({ id: a.fullId, result: 'blocked' });
+      one.verdict = 'blocked';
+      findings.push(`${rel}: blocked: ${reason}`);
+    };
+    if (refusal) { block(refusal); continue; }
+    if ([].concat(sc.fields['runs-in'] ?? []).map(String).includes(environment) === false) {
+      for (const a of sc.assertions) for (const w of a.ways ?? sc.through) evidence.setResult(a.fullId, w, 'not here', { why: `runs-in does not list ${environment}` });
+      evidence.setWay(way, { verdict: 'not here', reason: null, started, ended: new Date().toISOString() });
+      one.verdict = 'not here';
+      continue;
+    }
+    const reportDir = mkdtempSync(join(tmpdir(), `atlas-${runId}-kept-`));
+    const report = join(reportDir, 'report.xml');
+    const command = String(spec.command).replaceAll('{file}', shellWord(sc.keptFile)).replaceAll('{report}', shellWord(report));
+    let driver = null;
+    try {
+      driver = await makeDriver({
+        way, spec: { driver: 'command', command, env: spec.env, offline: spec.offline, cwd: spec.cwd },
+        env: { vars: {}, standIns: {} }, root, hosts: hostsOf(tests), redactor, runId, adapter,
+        record: (call) => evidence.addCall(call),
+      });
+      const answer = await driver.call({ timeoutMs: Math.max(1, Math.round(parseLimit(sc.fields.limit ?? '5m'))) });
+      if (answer.body.timedOut) { block(`the kept runner did not end within ${sc.fields.limit ?? '5m'}`); continue; }
+      if (!existsSync(report)) { block(`the kept runner wrote no report (exit ${answer.body.code}). Last output: ${redactor.text(String(answer.body.stderr || answer.body.stdout).trim().slice(-300))}`); continue; }
+      let cases;
+      try { cases = readJunit(readFileSync(report, 'utf8')); } catch (error) { block(`the report of the kept runner cannot be read: ${error.message}`); continue; }
+      const byName = new Map();
+      for (const c of cases) { const k = normalize(c.name); if (!byName.has(k)) byName.set(k, []); byName.get(k).push(c); }
+      for (const a of sc.assertions) {
+        const r = resultOf(sc.keptChecks.get(a.id) ?? [], byName);
+        for (const w of a.ways ?? sc.through) evidence.setResult(a.fullId, w, r.result, { kept: sc.keptFile, cases: r.cases, ...(r.why ? { why: r.why } : {}) });
+        one.assertions.push({ id: a.fullId, result: r.result });
+        for (const c of r.cases) if (c.result === 'missing') findings.push(`${rel}: ${a.id} names "${c.name}", which the report of ${sc.keptFile} does not hold`);
+      }
+      const named = new Set([...[...sc.keptChecks.values()].flat(), ...sc.notMapped].map((k) => normalize(k.name)));
+      for (const k of [...byName.keys()].filter((k) => !named.has(k))) {
+        findings.push(`${rel}: the case "${byName.get(k)[0].name}" of ${sc.keptFile} is not mapped; map it to an assertion, or list it under not mapped with a reason`);
+        evidence.note(`case not mapped: ${byName.get(k)[0].name}`);
+      }
+      const all = one.assertions.map((x) => x.result);
+      one.verdict = all.every((x) => x === 'pass') ? 'pass' : 'fail';
+      evidence.setWay(way, { verdict: one.verdict, reason: one.verdict === 'pass' ? null : 'an assertion did not pass', started, ended: new Date().toISOString() });
+    } catch (error) {
+      if (!(error instanceof Blocked)) throw error;
+      block(error.message);
+    } finally {
+      await driver?.close();
+      rmSync(reportDir, { recursive: true, force: true });
+    }
+  }
+  for (const one of results) {
+    write(`${one.scenario}  ${one.verdict}\n`);
+    for (const a of one.assertions) write(`  ${a.id}  ${a.result}\n`);
+  }
+  for (const f of findings) write(`finding  ${f}\n`);
+  if (!results.length) write('no scenario names kept-tests\n');
+  const green = !findings.length && results.every((r) => r.verdict === 'pass' || r.verdict === 'not here');
+  return { runId, results, findings, exit: green ? 0 : 1 };
+}
+
+export function parseArgs(argv) {
+  const out = {};
+  for (let i = 0; i < argv.length; i += 1) {
+    const m = /^--(root|evidence|scenario)(?:=(.*))?$/.exec(argv[i]);
+    if (!m) throw new Error(`unknown argument ${argv[i]}`);
+    out[m[1]] = m[2] ?? argv[++i];
+    if (out[m[1]] === undefined) throw new Error(`--${m[1]} needs a value`);
+  }
+  if (!out.root) throw new Error('--root <area> is needed');
+  return out;
+}
+
+export async function main(argv, write = (text) => process.stdout.write(text)) {
+  let args;
+  try { args = parseArgs(argv); } catch (error) { write(`${error.message}\nUse: node kept.mjs --root <area> [--evidence <dir>] [--scenario <id>]\n`); return 2; }
+  const { exit, runId } = await runKept({ root: args.root, evidenceDir: args.evidence, only: args.scenario ?? null, write });
+  write(`run ${runId}\n`);
+  return exit;
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  main(process.argv.slice(2)).then((code) => { process.exitCode = code; }, (error) => { process.stderr.write(`${error.message}\n`); process.exitCode = 2; });
+}

package/tests/link-check.mjs

@@ -289,11 +289,73 @@ function checkLinks(scenarios, root, testsDir, findings, rel) {
   for (const s of scenarios) {
     for (const a of s.assertions) {
       const named = (by.get(`${s.id}/${a.id}`) ?? []).filter((r) => r.kind !== 'waitFor');
+      // Path 1: the kept test cases of the "Kept checks" section are the test of the assertion.
+      if (!named.length && s.keptFile) continue;
       if (!named.length) findings.push({ file: rel(s.file), line: a.line, code: 'no-test', message: `no test names ${s.id}/${a.id}#${a.fp ?? '??????'}` });
     }
   }
 }
 
+// Path 1 (docs/design-docs/kept-tests). A scenario with kept-tests names a kept file that
+// exists, goes through a kept way in, maps each assertion to at least one test case, and
+// names each case once. A kept way in with `files` has a scenario for each file it matches.
+const keptCodes = { file: 'kept-file-missing', way: 'kept-no-way', unmapped: 'kept-not-mapped', twice: 'kept-mapped-twice' };
+function checkKept(s, tests, root, findings, rel) {
+  if (!s.keptFile) return;
+  const at = s.fieldLine('kept-tests');
+  if (!existsSync(join(root, s.keptFile))) findings.push({ file: rel(s.file), line: at, code: keptCodes.file, message: `kept-tests names ${s.keptFile}, which does not exist under the area` });
+  if (tests && !s.through.some((w) => tests['ways-in']?.[w]?.driver === 'kept')) findings.push({ file: rel(s.file), line: s.fieldLine('through'), code: keptCodes.way, message: 'a scenario with kept-tests goes through a way in with driver kept' });
+  for (const a of s.assertions) {
+    if (!(s.keptChecks.get(a.id) ?? []).length) findings.push({ file: rel(s.file), line: a.line, code: keptCodes.unmapped, message: `${s.id}/${a.id} names no kept test case in "Kept checks"` });
+  }
+  const seen = new Map();
+  for (const [id, list] of s.keptChecks) {
+    if (!s.assertions.some((a) => a.id === id)) findings.push({ file: rel(s.file), line: list[0].line, code: 'unknown-assertion', message: `a kept check names ${id}, which ${s.id} does not have` });
+    for (const k of list) {
+      const key = k.name.normalize('NFC').replace(/\s+/g, ' ').trim();
+      if (seen.has(key)) findings.push({ file: rel(s.file), line: k.line, code: keptCodes.twice, message: `the case "${k.name}" is named twice; name each kept case once` });
+      seen.set(key, k.line);
+    }
+  }
+  for (const k of s.notMapped) {
+    const key = k.name.normalize('NFC').replace(/\s+/g, ' ').trim();
+    if (seen.has(key)) findings.push({ file: rel(s.file), line: k.line, code: keptCodes.twice, message: `the case "${k.name}" is mapped and also listed as not mapped` });
+    seen.set(key, k.line);
+  }
+}
+
+// The files that a pattern of a kept way in matches: "*" stays inside one folder, "**" crosses folders.
+export function globToRegExp(pattern) {
+  let out = '';
+  const text = String(pattern).replace(/^\.\//, '');
+  for (let i = 0; i < text.length; i += 1) {
+    const c = text[i];
+    if (c === '*' && text[i + 1] === '*') { out += text[i + 2] === '/' ? '(?:.+/)?' : '.*'; i += text[i + 2] === '/' ? 2 : 1; }
+    else if (c === '*') out += '[^/]*';
+    else if (c === '?') out += '[^/]';
+    else out += c.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+  }
+  return new RegExp(`^${out}$`);
+}
+
+export function keptFilesOf(root, pattern) {
+  const re = globToRegExp(pattern);
+  const relOf = (f) => relative(root, f).split('\\').join('/');
+  return walk(root, { accept: (f) => re.test(relOf(f)) }).map(relOf);
+}
+
+function checkKeptFiles(scenarios, tests, root, findings, rel, yamlPath) {
+  const mapped = new Set(scenarios.filter((s) => s.keptFile).map((s) => s.keptFile.replace(/^\.\//, '')));
+  for (const [name, way] of Object.entries(tests?.['ways-in'] ?? {})) {
+    if (way?.driver !== 'kept' || !Array.isArray(way.files)) continue;
+    for (const pattern of way.files) {
+      for (const file of keptFilesOf(root, pattern)) {
+        if (!mapped.has(file)) findings.push({ file: rel(yamlPath), line: 1, code: 'kept-file-no-scenario', message: `${file} matches files of the kept way in "${name}", and no scenario names it in kept-tests` });
+      }
+    }
+  }
+}
+
 export function checkStatic({ root, testsDir, guardsFrom, rel, findings }) {
   const yamlPath = join(root, 'atlas', 'tests.yaml');
   let tests = null;
@@ -329,6 +391,8 @@ export function checkStatic({ root, testsDir, guardsFrom, rel, findings }) {
     }
   }
   for (const s of scenarios) checkWaitsAndWhens(s, findings, rel);
+  for (const s of scenarios) checkKept(s, tests, root, findings, rel);
+  checkKeptFiles(scenarios, tests, root, findings, rel, yamlPath);
   checkLinks(scenarios, root, testsDir, findings, rel);
   return { tests, scenarios };
 }

package/tests/scenario.mjs

@@ -271,6 +271,31 @@ export function parseScenario(source, file = 'scenario.md') {
     seen.add(m[1]);
     out.assertions.push({ id: m[1], kind: m[2], gate: m[3] === 'gate', words: m[4].trim(), line: item.line, names: names(m[4]), fp: null });
   }
+  // Path 1 (docs/design-docs/kept-tests): `kept-tests` names one kept test file, and the
+  // "Kept checks" section maps each assertion to the test cases of that file that check it.
+  //   - e1: "status prints one JSON line"
+  //   - not mapped: "the help text lists status" (why it maps to no assertion)
+  out.keptFile = typeof f['kept-tests'] === 'string' && f['kept-tests'].trim() ? f['kept-tests'].trim() : null;
+  if (f['kept-tests'] !== undefined && !out.keptFile) problem(out.fieldLine('kept-tests'), 'kept-tests names one kept test file, as a path under the area');
+  out.keptChecks = new Map();
+  out.notMapped = [];
+  const keptLines = sectionLines.get('kept checks') ?? [];
+  if (keptLines.length && !out.keptFile) problem(out.sections.get('kept checks'), 'a "Kept checks" section needs kept-tests in the front matter');
+  if (out.keptFile && !out.sections.has('kept checks')) problem(out.fieldLine('kept-tests'), 'a scenario with kept-tests needs a "## Kept checks" section');
+  for (const { text: raw, number } of keptLines) {
+    if (!raw.trim()) continue;
+    const m = /^-\s+(e\d+|not mapped):\s*"((?:[^"\\]|\\.)+)"\s*(.*)$/.exec(raw);
+    if (!m) { problem(number, `a kept check reads - eN: "test case name", or - not mapped: "test case name" (why); found "${raw.trim().slice(0, 60)}"`); continue; }
+    const name = m[2].replace(/\\(.)/g, '$1');
+    if (m[1] === 'not mapped') {
+      if (!m[3].trim()) problem(number, `"${name}" is not mapped and says no reason; add it in brackets`);
+      out.notMapped.push({ name, why: m[3].trim().replace(/^\(|\)$/g, ''), line: number });
+      continue;
+    }
+    if (!out.keptChecks.has(m[1])) out.keptChecks.set(m[1], []);
+    out.keptChecks.get(m[1]).push({ name, line: number });
+  }
+
   if (!out.assertions.length) problem(out.sections.get('assertions') ?? 1, 'the scenario has no assertions');
   else if (!out.sections.has('assertions')) problem(1, 'the scenario has no "## Assertions" section');
   return out;
@@ -345,6 +370,9 @@ export function addFingerprints(scenario, tests) {
     lines.push(`through=${[...scenario.through].sort().join(',')}`, `runs-in=${runsIn(scenario).sort().join(',')}`);
     // A weaker pass rule is a weaker test, so `repeat` is in the print. It adds a line only when set.
     if (scenario.repeat) lines.push(`repeat=${scenario.repeat.pass}/${scenario.repeat.runs}`);
+    // The kept test cases that check the assertion are part of what it means (path 1).
+    const keptNames = scenario.keptChecks?.get(a.id);
+    if (scenario.keptFile && keptNames?.length) lines.push(`kept=${scenario.keptFile}:${keptNames.map((k) => normalize(k.name)).sort().join('|')}`);
     a.fp = fingerprintOf([a.words, ...lines].join('\n'));
     a.fullId = `${scenario.id}/${a.id}#${a.fp}`;
     a.ways = waysOf(a, scenario, kept);

package/tests/tests-yaml.mjs

@@ -38,7 +38,7 @@ const SCHEMA = {
   runner: ['language', 'command'],
   environment: ENV_KEYS,
   ready: ['get', 'status', 'limit'],
-  wayIn: ['driver', 'base', 'command', 'module', 'env', 'cwd', 'model', 'key', 'tools-from', 'tools', 'system-prompt', 'offline', 'why'],
+  wayIn: ['driver', 'base', 'command', 'module', 'env', 'cwd', 'model', 'key', 'tools-from', 'tools', 'system-prompt', 'offline', 'files', 'why'],
   standIn: ['start', 'why', 'ready'],
   persona: ['role', 'secret', 'same-identity', 'sign-in', 'make', 'of', 'why'],
   setup: ['with', 'steps', 'gives', 'why'],
@@ -170,6 +170,12 @@ export function checkTestsYaml(parsed) {
     if (!way.driver) say('tests-yaml-missing-field', [...path, 'driver'], `the field "${[...path, 'driver'].join('.')}" is needed`);
     if (way.key !== undefined && !secretNamed(way.key)) say('unknown-name', [...path, 'key'], `"key" names ${way.key}, which "secrets" does not list`);
     if (way.driver === 'command' && way.command === undefined) say('tests-yaml-missing-field', [...path, 'command'], 'a command way in needs "command"');
+    // Path 1: the kept runner. Its command holds {file} and {report} (docs/design-docs/kept-tests).
+    if (way.driver === 'kept') {
+      if (typeof way.command !== 'string') say('tests-yaml-missing-field', [...path, 'command'], 'a kept way in needs "command": the kept runner, with {file} and {report}');
+      else for (const hole of ['{file}', '{report}']) if (!way.command.includes(hole)) say('tests-yaml', [...path, 'command'], `the command of a kept way in needs ${hole}`);
+      if (way.files !== undefined && (!Array.isArray(way.files) || !way.files.length || way.files.some((f) => typeof f !== 'string'))) say('tests-yaml', [...path, 'files'], '"files" is a list of paths or patterns, such as "test/*.test.mjs"');
+    } else if (way.files !== undefined) say('tests-yaml', [...path, 'files'], '"files" belongs to a kept way in');
     if (way.offline !== undefined && way.offline !== true) say('tests-yaml', [...path, 'offline'], '"offline" is true, or it is left out');
     if (way.driver === 'agent') {
       for (const key of ['model', 'tools-from', 'tools']) if (way[key] === undefined) say('tests-yaml-missing-field', [...path, key], `an agent way in needs "${key}"`);