@arjunkhera/atlas 0.3.41 → 0.3.43

package/tests/drivers/index.mjs

@@ -37,6 +37,9 @@ export function commandFor(command, root, productRoot) {
   return command.split(root).join(productRoot);
 }
 
+// The drivers that the kit ships: the cases of makeDriver, and the kept runner (kept.mjs).
+export const SHIPPED_DRIVERS = Object.freeze(['http', 'browser', 'mcp-stdio', 'function', 'agent', 'command', 'kept']);
+
 export async function makeDriver(ctx) {
   const { way, spec, env, root, hosts, redactor, record, adapter = {}, runId } = ctx;
   const productRoot = productRootOf(root);

package/tests/link-check.mjs

@@ -5,8 +5,13 @@
 //
 //   static  reads text: every scenario, atlas/tests.yaml, the tests and the
 //           support code. It fails on the nine cases below.
-//   lint    the JavaScript half. It fails on a skip, an "only", a todo, a
-//           retry, a soft check, or a test with no assert statement.
+//   lint    one lint for each language of the tests. JavaScript (.js, .mjs,
+//           .cjs): it fails on a skip, an "only", a todo, a retry, a soft
+//           check, or a test with no assert statement. Python (.py): it fails
+//           on a skip, an expected fail, a focus mark, a retry, a soft check
+//           (a swallowed assert), an exit, an assert that cannot fail, a check
+//           that is not a with block, and a check or a test with no assert
+//           statement.
 //   run     reads the evidence files of ONE run (--run <id>). With no --run,
 //           it reads the newest run of each scenario, one run per scenario. It fails on an assertion
 //           that the run did not check, on evidence of a changed scenario file
@@ -48,7 +53,7 @@ const FULL_ID = /^([a-z0-9][a-z0-9-]*)\/(e\d+)#([0-9a-f]{6})$/;
 // Folders skipped by name, at any depth. The kit folder and the evidence folder
 // are skipped by exact path only (<area>/test/atlas, <area>/test/evidence): a
 // folder that merely carries the name "atlas" or "evidence" is read like any other.
-const SKIP_DIRS = new Set(['node_modules', '.git']);
+const SKIP_DIRS = new Set(['node_modules', '.git', '__pycache__']);
 
 // ---------------------------------------------------------------- files
 
@@ -187,8 +192,10 @@ function checkBudget(scenario, tests, findings, rel) {
   }
 }
 
-// The ids that tests name. Only a literal first argument can be read.
-const CALL = /\brun\.(check|gate|judge|notExercised|waitFor)\s*\(/g;
+// The ids that tests name. Only a literal first argument can be read. A Python
+// test names the same parts in snake case (not_exercised, wait_for).
+const CALL = /\brun\.(check|gate|judge|notExercised|waitFor|not_exercised|wait_for)\s*\(/g;
+const KIND = { not_exercised: 'notExercised', wait_for: 'waitFor' };
 
 export function readRefs(file, code, rel, findings) {
   const refs = [];
@@ -202,15 +209,17 @@ export function readRefs(file, code, rel, findings) {
     }
     const id = FULL_ID.exec(literal[2]);
     if (!id) { findings?.push({ file: rel(file), line, code: 'bad-id', message: `"${literal[2]}" is not a full assertion id like <scenario>/e1#abc123` }); continue; }
-    refs.push({ file, line, kind: m[1], scenario: id[1], n: id[2], fp: id[3], full: literal[2] });
+    refs.push({ file, line, kind: KIND[m[1]] ?? m[1], scenario: id[1], n: id[2], fp: id[3], full: literal[2] });
   }
   return refs;
 }
 
 function scenarioTestFiles(root, testsDir) {
-  return walk(join(root, testsDir), { skipPaths: [join(root, testsDir, 'atlas'), join(root, testsDir, 'evidence')], accept: (f) => /\.(m?js|cjs)$/.test(f) });
+  return walk(join(root, testsDir), { skipPaths: [join(root, testsDir, 'atlas'), join(root, testsDir, 'evidence')], accept: (f) => /\.(m?js|cjs|py)$/.test(f) });
 }
 
+const isPython = (file) => file.endsWith('.py');
+
 // ---------------------------------------------------------------- the kit
 
 // The hash of a kit: the sha256 of its files, as sorted "path hash" lines.
@@ -256,6 +265,7 @@ export function checkKit({ root, testsDir, rel, findings, kitReleases = null })
   for (const one of onDisk) {
     const full = join(dir, one.path);
     if (one.link) { say(full, 'kit-extra', `${one.path} in the kit folder is a link; the kit holds plain files`); continue; }
+    if (one.path.split('/').includes('__pycache__')) { say(full, 'kit-extra', `${one.path} is Python bytecode in the kit folder; delete it. A Python test sets sys.dont_write_bytecode = True before it imports contract`); continue; }
     if (!Object.hasOwn(listed, one.path)) { say(full, 'kit-extra', `${one.path} is in the kit folder and not in manifest.json; the kit folder holds only what "atlas tests write" put there`); continue; }
     if (sha256Of(readFileSync(one.full)) !== listed[one.path]) say(full, 'kit-changed', `${one.path} differs from manifest.json; it was edited. Restore it with "atlas tests write"`);
   }
@@ -356,6 +366,27 @@ function checkKeptFiles(scenarios, tests, root, findings, rel, yamlPath) {
   }
 }
 
+// Python loads bytecode and binary modules before (or instead of) the source, so a
+// committed .pyc could run code that the lint and the reviewer never read. The test
+// folder holds none, at any depth: delete it, and run Python with -B.
+const BINARY = /\.(?:pyc|pyo|so|pyd)$/;
+function checkBinaries(root, testsDir, findings, rel) {
+  const found = [];
+  const look = (dir) => {
+    if (!existsSync(dir)) return;
+    for (const name of readdirSync(dir).sort()) {
+      const full = join(dir, name);
+      const stat = lstatSync(full);
+      // A link could point Python at bytecode outside the folder, so a link of any kind is a finding.
+      if (name === '__pycache__' || stat.isSymbolicLink()) found.push(full);
+      else if (stat.isDirectory()) { if (!SKIP_DIRS.has(name)) look(full); }
+      else if (BINARY.test(name)) found.push(full);
+    }
+  };
+  look(join(root, testsDir));
+  for (const full of found) findings.push({ file: rel(full), line: 1, code: 'python-bytecode', message: 'Python bytecode, a binary module or a link in the test folder can run code that no one reads; delete it, and run Python with -B' });
+}
+
 export function checkStatic({ root, testsDir, guardsFrom, rel, findings }) {
   const yamlPath = join(root, 'atlas', 'tests.yaml');
   let tests = null;
@@ -394,6 +425,7 @@ export function checkStatic({ root, testsDir, guardsFrom, rel, findings }) {
   for (const s of scenarios) checkKept(s, tests, root, findings, rel);
   checkKeptFiles(scenarios, tests, root, findings, rel, yamlPath);
   checkLinks(scenarios, root, testsDir, findings, rel);
+  checkBinaries(root, testsDir, findings, rel);
   return { tests, scenarios };
 }
 
@@ -441,6 +473,9 @@ function assertingNames(code) {
   return names;
 }
 
+// The test files and the support files that the lint reads: each file that
+// holds a scenario, each file under support/ or scenarios/, and each file of the
+// test folder that one of them imports.
 function lintFiles(root, testsDir) {
   const base = join(root, testsDir);
   const all = scenarioTestFiles(root, testsDir);
@@ -450,16 +485,55 @@ function lintFiles(root, testsDir) {
   const queue = [...seeds];
   while (queue.length) {
     const file = queue.pop();
-    for (const m of text.get(file).matchAll(/\bfrom\s+['"](\.{1,2}\/[^'"]+)['"]|\bimport\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]/g)) {
-      const target = resolve(dirname(file), m[1] ?? m[2]);
+    const targets = isPython(file) ? pythonImports(file, text.get(file), base) : [...text.get(file).matchAll(/\bfrom\s+['"](\.{1,2}\/[^'"]+)['"]|\bimport\s*\(\s*['"](\.{1,2}\/[^'"]+)['"]/g)].map((m) => resolve(dirname(file), m[1] ?? m[2]));
+    for (const target of targets) {
       if (text.has(target) && !picked.has(target)) { picked.add(target); queue.push(target); }
     }
   }
   return { files: [...picked].sort(), text, all };
 }
 
+// The files that a Python file may import: "from a.b import c" gives a/b.py, a/b/__init__.py
+// and a/b/c.py, from the folder of the file and from the test folder ("from .x" from the folder only).
+function pythonImports(file, code, base) {
+  const out = [];
+  const add = (dir, dotted) => {
+    const path = join(dir, ...dotted.split('.').filter(Boolean));
+    out.push(`${path}.py`, join(path, '__init__.py'));
+  };
+  const stripped = stripPy(code);
+  for (const m of stripped.matchAll(/^[ \t]*from[ \t]+(\.*)([\w.]*)[ \t]+import[ \t]+\(?([\w, \t\n]+)\)?/gm)) {
+    const names = m[3].split(',').map((n) => n.trim().split(/\s+/)[0]).filter(Boolean);
+    let dirs;
+    if (m[1]) {
+      let dir = dirname(file);
+      for (let i = 1; i < m[1].length; i += 1) dir = dirname(dir);
+      dirs = [dir];
+    } else dirs = [dirname(file), base];
+    for (const dir of dirs) {
+      if (m[2]) add(dir, m[2]);
+      for (const n of names) add(dir, m[2] ? `${m[2]}.${n}` : n);
+    }
+  }
+  for (const m of stripped.matchAll(/^[ \t]*import[ \t]+([\w., \t]+)$/gm)) {
+    for (const one of m[1].split(',').map((n) => n.trim().split(/\s+/)[0]).filter(Boolean)) {
+      for (const dir of [dirname(file), base]) add(dir, one);
+    }
+  }
+  return out;
+}
+
 export function checkLint({ root, testsDir, rel, findings }) {
   const { files, text, all } = lintFiles(root, testsDir);
+  const js = files.filter((f) => !isPython(f));
+  const py = files.filter(isPython);
+  if (js.length) lintJs({ files: js, all: all.filter((f) => !isPython(f)), text, rel, findings });
+  if (py.length) lintPy({ files: py, all: all.filter(isPython), text, rel, findings });
+}
+
+// ---------------------------------------------------------------- the JavaScript lint
+
+function lintJs({ files, all, text, rel, findings }) {
   const asserting = new Set();
   for (const f of all) for (const n of assertingNames(stripJs(text.get(f)))) asserting.add(n);
   const holds = (body) => ASSERTS.test(body) || [...asserting].some((n) => new RegExp(`(?<![\\w$.])${n.replace(/\$/g, '\\$')}\\s*\\(|\\.${n.replace(/\$/g, '\\$')}\\s*\\(`).test(body));
@@ -485,6 +559,144 @@ export function checkLint({ root, testsDir, rel, findings }) {
   }
 }
 
+// ---------------------------------------------------------------- the Python lint
+
+// Replaces comments, and the insides of strings, with spaces, as stripJs does.
+// It knows the four quote forms and the string prefixes (r, b, f, u). The fields
+// of an f-string ({...}) are code, and may hold strings of the same quote
+// (Python 3.12), so they are read as code and kept.
+export function stripPy(code) {
+  const blank = (t) => t.replace(/[^\n]/g, ' ');
+  // Reads code from i until `stop` (a closing brace of a field) at depth 0, or the end.
+  const readCode = (i, stop) => {
+    let out = '';
+    let depth = 0;
+    while (i < code.length) {
+      const c = code[i];
+      if (stop && depth === 0 && (c === '}' || c === '!' && code[i + 1] !== '=' || c === ':' )) return { out, i };
+      if (c === '#') { const end = code.indexOf('\n', i); const at = end < 0 ? code.length : end; out += blank(code.slice(i, at)); i = at; continue; }
+      if (c === '"' || c === "'") {
+        const prefix = /[A-Za-z]*$/.exec(code.slice(Math.max(0, i - 3), i))[0].toLowerCase();
+        const r = readString(i, prefix.includes('f'));
+        out += r.out; i = r.i; continue;
+      }
+      if (c === '(' || c === '[' || c === '{') depth += 1;
+      else if (c === ')' || c === ']' || c === '}') depth -= 1;
+      out += c; i += 1;
+    }
+    return { out, i };
+  };
+  const readString = (i, isF) => {
+    const c = code[i];
+    const quote = code.startsWith(c.repeat(3), i) ? c.repeat(3) : c;
+    let out = quote;
+    let j = i + quote.length;
+    while (j < code.length && !code.startsWith(quote, j) && (quote.length === 3 || code[j] !== '\n')) {
+      if (code[j] === '\\') { out += blank(code.slice(j, j + 2)); j += 2; continue; }
+      if (isF && code[j] === '{') {
+        if (code[j + 1] === '{') { out += '  '; j += 2; continue; }
+        out += '{';
+        const field = readCode(j + 1, true);
+        out += field.out; j = field.i;
+        // A format spec or a conversion, up to the closing brace: blank text, but nested fields are code.
+        while (j < code.length && code[j] !== '}' && code[j] !== '\n') {
+          if (code[j] === '{') { const inner = readCode(j + 1, true); out += `{${inner.out}`; j = inner.i; if (code[j] === '}') { out += '}'; j += 1; } continue; }
+          out += ' '; j += 1;
+        }
+        if (code[j] === '}') { out += '}'; j += 1; }
+        continue;
+      }
+      out += code[j] === '\n' ? '\n' : ' ';
+      j += 1;
+    }
+    if (code.startsWith(quote, j)) { out += quote; j += quote.length; }
+    return { out, i: j };
+  };
+  return readCode(0, false).out;
+}
+
+const PY_BANNED = [
+  [/@\s*(?:unittest\s*\.\s*)?(?:skip|skipIf|skipUnless|expectedFailure)\b/, 'a skipped or expected-fail test'],
+  [/\bmark\s*\.\s*(?:skip|skipif|xfail)\b/, 'a skipped or expected-fail test'],
+  [/\bmark\s*\.\s*(?:only|focus)\b/, 'a focused test'],
+  [/\.\s*skipTest\s*\(|\bpytest\s*\.\s*(?:skip|xfail|importorskip)\s*\(|\bSkipTest\b/, 'a test that skips itself'],
+  [/\bmark\s*\.\s*flaky\b|@\s*flaky\b|\breruns\s*=|@\s*(?:\w+\s*\.\s*)?retry\b|\b(?:import|from)\s+(?:tenacity|backoff|flaky)\b/, 'a retry'],
+  [/\bpytest_check\b|\bsoft_assertions\b|\bsoftest\b/, 'a soft check'],
+  [/\bexcept\s*(?:\(\s*[\w.,\s]*)?\b(?:AssertionError|BaseException|Exception)\b|\bexcept\s*:/, 'a soft check: an except that swallows a failed assert'],
+  [/\bsuppress\b|\bExitStack\b|\bdef\s+__exit__\b/, 'a soft check: a context manager that can swallow a failed assert'],
+  [/=\s*AssertionError\b|\bAssertionError\s+as\b|\bas\s+AssertionError\b/, 'a soft check: a new name for AssertionError'],
+  [/\bsys\s*\.\s*exit\s*\(|\bos\s*\.\s*_exit\s*\(|(?<![\w.])(?:exit|quit)\s*\(/, 'an exit from inside a test'],
+  [/(?<![\w.])assert\s*\(?\s*(?:not\s+(?:False|None|0|""|'')|True|[1-9][\d_.]*|"\s+"|'\s+')\s*\)?\s*(?:,|;|$)/m, 'an assert that cannot fail'],
+  [/\bassert(?:True|Truthy)\s*\(\s*(?:True|[1-9][\d_.]*)\s*[,)]|\bassert(?:False|IsNone)\s*\(\s*(?:False|None|0)\s*[,)]/, 'an assert that cannot fail'],
+  [/(?<![\w.])assert\s*\([^()\n]*,[^()\n]*\)\s*$/m, 'an assert that cannot fail: it asserts a tuple'],
+];
+const PY_ASSERTS = /(?<![\w.])assert\b|\bself\s*\.\s*(?:assert\w*|fail)\s*\(|\.\s*assert_\w+\s*\(|\bpytest\s*\.\s*(?:raises|fail)\s*\(/;
+
+// The block of a statement that starts at `index` (a def, a with): the rest of its
+// header line after the colon, or else the lines below it that are indented more.
+function pyBlock(code, index) {
+  let depth = 0;
+  let colon = -1;
+  for (let i = index; i < code.length; i += 1) {
+    const c = code[i];
+    if (c === '(' || c === '[' || c === '{') depth += 1;
+    else if (c === ')' || c === ']' || c === '}') depth -= 1;
+    else if (c === ':' && depth === 0) { colon = i; break; }
+  }
+  if (colon < 0) return '';
+  const lineStart = code.lastIndexOf('\n', index) + 1;
+  const indent = /^[ \t]*/.exec(code.slice(lineStart))[0].length;
+  const end = code.indexOf('\n', colon);
+  const rest = code.slice(colon + 1, end < 0 ? undefined : end);
+  if (rest.trim()) return rest;
+  const lines = [];
+  for (const line of code.slice(end + 1).split('\n')) {
+    if (!line.trim()) { lines.push(line); continue; }
+    if (/^[ \t]*/.exec(line)[0].length <= indent) break;
+    lines.push(line);
+  }
+  return lines.join('\n');
+}
+
+function pyAssertingNames(code) {
+  const names = new Set();
+  for (const m of code.matchAll(/^[ \t]*(?:async[ \t]+)?def[ \t]+([A-Za-z_]\w*)\s*\(/gm)) {
+    if (PY_ASSERTS.test(pyBlock(code, m.index))) names.add(m[1]);
+  }
+  return names;
+}
+
+function lintPy({ files, all, text, rel, findings }) {
+  const asserting = new Set();
+  for (const f of all) for (const n of pyAssertingNames(stripPy(text.get(f)))) asserting.add(n);
+  const holds = (body) => PY_ASSERTS.test(body) || [...asserting].some((n) => new RegExp(`(?<![\\w.])${n}\\s*\\(|\\.${n}\\s*\\(`).test(body));
+  for (const file of files) {
+    const code = stripPy(text.get(file));
+    const say = (index, message) => findings.push({ file: rel(file), line: lineAt(code, index), code: 'lint', message });
+    for (const [pattern, what] of PY_BANNED) {
+      const re = new RegExp(pattern.source, pattern.flags.includes('m') ? 'gm' : 'g');
+      for (const m of code.matchAll(re)) say(m.index, what);
+    }
+    // A check or a gate is a with block, and its block holds an assert statement.
+    for (const m of code.matchAll(/\brun\s*\.\s*(check|gate)\s*\(/g)) {
+      const lineStart = code.lastIndexOf('\n', m.index) + 1;
+      const head = code.slice(lineStart, m.index);
+      if (!/^[ \t]*(?:async[ \t]+)?with\b/.test(head)) { say(m.index, `the run.${m[1]} is not a with block, so it checks nothing`); continue; }
+      const block = pyBlock(code, m.index);
+      if (!holds(block)) say(m.index, `the run.${m[1]} holds no assert statement`);
+      // The block holds plain asserts: no try, and no context manager of its own, on its line or inside it.
+      const call = balanced(code, code.indexOf('(', m.index));
+      const after = code.slice(code.indexOf('(', m.index) + call.length, code.indexOf(':', code.indexOf('(', m.index) + call.length));
+      if (/,/.test(after.replace(/\bas\s+\w+/, ''))) say(m.index, `the run.${m[1]} shares its with line with another context manager, which can swallow a failed assert`);
+      if (/(?:^|\n)[ \t]*(?:try\s*:|(?:async[ \t]+)?with\b)/.test(block)) say(m.index, `the run.${m[1]} holds a try or a with block, which can swallow a failed assert`);
+    }
+    // An ordinary test (a def whose name starts with test) must hold one too.
+    for (const m of code.matchAll(/^[ \t]*(?:async[ \t]+)?def[ \t]+(test\w*)\s*\(/gm)) {
+      if (!holds(pyBlock(code, m.index))) say(m.index + m[0].indexOf('def'), 'a test with no assert statement');
+    }
+  }
+}
+
 // ---------------------------------------------------------------- the run half
 
 // The run ids in an evidence folder, newest first. A run is a sub-folder that

package/tests/tests-yaml.mjs

@@ -150,10 +150,19 @@ export function checkTestsYaml(parsed) {
   known(value, SCHEMA.top, []);
   for (const key of REQUIRED) if (!(key in value)) say('tests-yaml-missing-field', [key], `the field "${key}" is needed`);
   if ('spec' in value && value.spec !== SPEC) say('tests-yaml-spec', ['spec'], `spec must be ${SPEC}; this check reads spec ${SPEC} only`);
-  if (isMap(value.runner)) {
-    known(value.runner, SCHEMA.runner, ['runner']);
-    for (const key of SCHEMA.runner) if (!(key in value.runner)) say('tests-yaml-missing-field', ['runner', key], `the field "runner.${key}" is needed`);
-  } else if ('runner' in value) say('tests-yaml', ['runner'], '"runner" must be a map');
+  // One runner, or a list of runners: one for each language of the tests (a JavaScript runner and a Python runner, say).
+  const runners = isMap(value.runner) ? [[value.runner, ['runner']]] : Array.isArray(value.runner) ? value.runner.map((r, i) => [r, ['runner', i]]) : null;
+  if (runners) {
+    if (!runners.length) say('tests-yaml', ['runner'], '"runner" lists no runner');
+    const languages = new Set();
+    for (const [one, path] of runners) {
+      if (!isMap(one)) { say('tests-yaml', path, `"${path.join('.')}" must be a map`); continue; }
+      known(one, SCHEMA.runner, path);
+      for (const key of SCHEMA.runner) if (!(key in one)) say('tests-yaml-missing-field', [...path, key], `the field "${[...path, key].join('.')}" is needed`);
+      if (languages.has(one.language)) say('tests-yaml', [...path, 'language'], `two runners name the language "${one.language}"`);
+      languages.add(one.language);
+    }
+  } else if ('runner' in value) say('tests-yaml', ['runner'], '"runner" must be a map, or a list of maps');
 
   each(value.environments, SCHEMA.environment, ['environments'], (env, path) => {
     if (isMap(env.ready)) known(env.ready, SCHEMA.ready, [...path, 'ready']);

package/tests/verdict.mjs

@@ -311,7 +311,11 @@ const ENDED_WELL = new Set(['pass', 'not here']);
 // not pass or not here. A file whose scenario wrote no evidence at all runs again. So does a file that
 // names no scenario of the area. Returns the test files, as paths from the area, in order.
 // When nothing qualifies, it returns every test file: a run failed, and the evidence does not say where.
-export function rerunFiles({ root, run, evidence = null, tests = 'test' }) {
+// `language` picks the test files of one runner: javascript (*.test.mjs) or python (*_test.py).
+export const TEST_FILE = Object.freeze({ javascript: (n) => n.endsWith('.test.mjs'), python: (n) => n.endsWith('_test.py') });
+export function rerunFiles({ root, run, evidence = null, tests = 'test', language = 'javascript' }) {
+  const isTest = Object.hasOwn(TEST_FILE, language) ? TEST_FILE[language] : null;
+  if (!isTest) throw new Error(`there is no language "${language}"; use ${Object.keys(TEST_FILE).join(' or ')}`);
   const area = resolve(root);
   const dir = join(area, tests, 'scenarios');
   if (!existsSync(dir)) throw new Error(`there is no folder ${dir}`);
@@ -322,7 +326,7 @@ export function rerunFiles({ root, run, evidence = null, tests = 'test' }) {
   // A scenario with a model in it (repeat, or a judged assertion) never runs a second time: its subject
   // varies by design, so a second try would weaken its pass rule and spend the money twice.
   const varying = new Set(loaded.filter((s) => s.repeat || s.assertions.some((a) => a.kind === 'judged')).map((s) => s.id));
-  const files = readdirSync(dir).filter((n) => n.endsWith('.test.mjs')).filter((n) => {
+  const files = readdirSync(dir).filter(isTest).filter((n) => {
     const ids = [...readFileSync(join(dir, n), 'utf8').matchAll(/scenarios\/([A-Za-z0-9._-]+\.md)/g)].map((m) => byFile.get(m[1])).filter(Boolean);
     return !(ids.length && ids.every((id) => varying.has(id)));
   }).sort();
@@ -339,10 +343,10 @@ export function rerunFiles({ root, run, evidence = null, tests = 'test' }) {
   return pick.length ? pick : files.map((n) => join(tests, 'scenarios', n));
 }
 
-export function rerunCommand({ run, evidence, tests = 'test' }, root, line) {
+export function rerunCommand({ run, evidence, tests = 'test', language = 'javascript' }, root, line) {
   try {
     if (!run) throw Object.assign(new Error('give the run: --run <id>'), { usage: true });
-    for (const f of rerunFiles({ root, run, evidence, tests: tests ?? 'test' })) line(f);
+    for (const f of rerunFiles({ root, run, evidence, tests: tests ?? 'test', language: language ?? 'javascript' })) line(f);
     return 0;
   } catch (error) {
     process.stderr.write(`rerun: ${error.message}\n`);