@arjunkhera/atlas 0.3.13 → 0.3.15

package/tests/link-check.mjs

@@ -466,6 +466,25 @@ export function newestRunPerScenario(evidenceDir) {
   return new Map([...best].map(([scenario, b]) => [scenario, b.name]));
 }
 
+// The newest run of the whole folder, by the start time of its evidence (a name breaks a tie).
+export function newestRunId(evidenceDir) {
+  let best = null;
+  if (!existsSync(evidenceDir)) return null;
+  for (const name of readdirSync(evidenceDir)) {
+    const dir = join(evidenceDir, name);
+    if (!statSync(dir).isDirectory()) continue;
+    for (const file of readdirSync(dir)) {
+      if (!file.endsWith('.json')) continue;
+      let data;
+      try { data = JSON.parse(readFileSync(join(dir, file), 'utf8')); } catch { continue; }
+      if (data?.evidence !== 1 || data.fault_run || !data.scenario) continue;
+      const started = String(data.started);
+      if (!best || started > best.started || (started === best.started && name > best.name)) best = { started, name };
+    }
+  }
+  return best ? best.name : null;
+}
+
 // The run half reads the files of one run only, and only evidence of the
 // scenario file as it is now (source_sha256).
 export function checkRun({ evidenceDir, runId = null, environment = null, scenarios, rel, findings }) {
@@ -488,8 +507,8 @@ export function checkRun({ evidenceDir, runId = null, environment = null, scenar
       runOf.set(data.scenario, id);
     }
   }
-  const reached = new Set(['pass', 'fail', 'not exercised', 'not here']);
-  const ran = new Set(['pass', 'fail', 'not exercised']);
+  const reached = new Set(['pass', 'fail', 'not exercised', 'not here', 'flaky']);
+  const ran = new Set(['pass', 'fail', 'not exercised', 'flaky']);
   for (const s of scenarios) {
     if (!s.id) continue;
     const all = records.filter((r) => r.data.scenario === s.id);

package/tests/sweep.mjs

@@ -0,0 +1,408 @@
+// The daily sweep and the coverage view (design sections 11.3 and 11.4).
+//
+//   atlas tests sweep  --root <area> --evidence <dir> [--since <ISO date>] [--base <ref>] [--out <file>] [--text]
+//   atlas tests covers <item-id> --root <area> [--evidence <dir>] [--json]
+//
+// Pure code: no network, no key, no model. The sweep reads the evidence folders of the runs in
+// <dir> (one folder for each run; the folders may come from several CI runs) and the git history of
+// the repo that holds the area. It returns findings for the five checks of section 11.3:
+//
+//   1  flaky           each scenario/assertion with a `flaky` result, and on how many commits
+//   2  stand-ins       each contract scenario whose newest result is `blocked` (no test instance)
+//   3  no scenario     a pull request (a first-parent change of the base) changed a path that a scenario `covers`,
+//                      and no scenario that covers that path changed in the same pull request
+//   4  quarantine      NOT BUILT. The kit has no quarantine field and no `quarantined` result yet
+//                      (RESULTS in evidence.mjs), so this check reports none and says so. No format is invented.
+//   5  map facts       NOT BUILT. link-check.mjs reads no map format, so the check is skipped with a note
+//
+// The sweep never files an item. Each finding carries an `issue` block (title, body, labels, a
+// dedupe marker). A scheduled workflow opens the issues with `atlas tests issues`. That verb is in
+// the package (door/lib/issues.mjs), not in this kit, because it reaches the network. The lead
+// then adopts each issue into the tracker.
+//
+// The evidence is data from a run of pull request code, so the sweep does not trust it. It keeps
+// an assertion only when its id has the shape `scenario/eN`, its scenario exists in the area on
+// disk, and its way in is one that the area names. It keeps a run folder only when the name is a
+// CI run id or the kit's own run id. It counts what it drops in `notes`. Free text from the
+// evidence goes into an issue only in a fenced block, cut to 200 characters.
+import { spawnSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
+import { basename, dirname, join, relative, resolve } from 'node:path';
+import { readAttempts, settleRecords, loadAreaScenarios, clean, touches, keyOf } from './verdict.mjs';
+
+export const LABEL = 'atlas-sweep';
+const DAY = 24 * 60 * 60 * 1000;
+// A pull request is found by the next sweep too, so two days make a sweep that missed a day lose nothing.
+// The marker of a finding keeps a second sweep from opening a second issue.
+const WINDOW = 2 * DAY;
+const SCENARIO_FILE = /\.md$/;
+const NOT_SCENARIO = /(^|\/)(map|README)\.md$/;
+
+const usage = (message) => Object.assign(new Error(message), { usage: true });
+const list = (value) => (Array.isArray(value) ? value.map(String) : value === undefined || value === null ? [] : [String(value)]);
+
+function git(cwd, args, { allowFail = false } = {}) {
+  const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024 });
+  if (r.error) throw new Error(`git cannot run: ${r.error.message}`);
+  if (r.status !== 0) {
+    if (allowFail) return null;
+    throw new Error(`git ${args.join(' ')} failed: ${(r.stderr || r.stdout || '').trim().split('\n')[0]}`);
+  }
+  return r.stdout;
+}
+
+// ---------------------------------------------------------------- the evidence of many runs
+
+// A run folder is named for a CI run (ci<run id>-<attempt>, with an optional rerun count) or by the
+// kit's own run id (run<6 hex>, fresh.mjs). Any other name is dropped by the sweep.
+const RUN_NAME = /^(ci\d+-\d+(-\d+)?|run[0-9a-f]{6})$/;
+const ASSERTION_ID = /^[a-z0-9-]+\/e[0-9]+$/;
+const COMMIT = /^[0-9a-f]{40}$/;
+const hasJson = (dir) => readdirSync(dir).some((n) => n.endsWith('.json'));
+const dirsOf = (dir) => readdirSync(dir).sort().filter((n) => statSync(join(dir, n)).isDirectory());
+
+// Reads every run folder of the evidence folder. Returns [{ runId, scenarios: Map(id -> settled record) }].
+// A folder with no JSON of its own may hold run folders one level down (a sweep downloads each CI run
+// into its own folder). A run with no evidence file that counts is left out. A scenario with more than
+// two attempts keeps the settled record of its first and last attempt (settleRecords), as `verdict` does.
+//
+// With `filter` ({ scenarios: Set of ids, ways: Set of names }) the evidence is not trusted: see the
+// header. The array then has a `dropped` property that counts what was left out.
+export function readRuns(evidenceDir, filter = null) {
+  const dir = resolve(evidenceDir);
+  if (!existsSync(dir) || !statSync(dir).isDirectory()) throw usage(`the evidence folder ${dir} does not exist`);
+  const dropped = { folders: 0, scenarios: 0, assertions: 0, ways: 0 };
+  const candidates = [];
+  for (const name of dirsOf(dir)) {
+    if (hasJson(join(dir, name))) candidates.push({ parent: dir, name });
+    else for (const inner of dirsOf(join(dir, name))) candidates.push({ parent: join(dir, name), name: inner });
+  }
+  const runs = [];
+  for (const { parent, name } of candidates) {
+    if (filter && !RUN_NAME.test(name)) { dropped.folders += 1; continue; }
+    let attempts;
+    try { attempts = readAttempts(parent, name); } catch { continue; }
+    if (!attempts.size) continue;
+    const scenarios = new Map();
+    for (const [id, records] of attempts) {
+      const { record } = settleRecords(records);
+      if (!filter) { scenarios.set(id, record); continue; }
+      if (!filter.scenarios.has(id)) { dropped.scenarios += 1; continue; }
+      const assertions = record.assertions.filter((a) => {
+        const ok = ASSERTION_ID.test(keyOf(a.id)) && keyOf(a.id).startsWith(`${id}/`) && filter.ways.has(a.way);
+        if (!ok) dropped.assertions += 1;
+        return ok;
+      });
+      const ways = {};
+      for (const [way, w] of Object.entries(record.ways ?? {})) { if (filter.ways.has(way)) ways[way] = w; else dropped.ways += 1; }
+      scenarios.set(id, { ...record, assertions, ways, commit: COMMIT.test(String(record.commit ?? '')) ? record.commit : null });
+    }
+    if (scenarios.size) runs.push({ runId: name, scenarios });
+  }
+  runs.dropped = dropped;
+  return runs;
+}
+
+const startedOf = (record) => record.started ?? '';
+
+// The newest settled record of each scenario, over all runs. Returns Map(id -> { runId, record }).
+export function newestRecords(runs) {
+  const best = new Map();
+  for (const run of runs) {
+    for (const [id, record] of run.scenarios) {
+      const have = best.get(id);
+      if (!have || startedOf(record) > startedOf(have.record) || (startedOf(record) === startedOf(have.record) && run.runId > have.runId)) best.set(id, { runId: run.runId, record });
+    }
+  }
+  return best;
+}
+
+// ---------------------------------------------------------------- the findings
+
+const short = (sha) => String(sha).slice(0, 7);
+
+// An HTML comment may not hold two hyphens in a row, so the marker spells them another way.
+const markerKey = (key) => key.replace(/--/g, '-~');
+export const markerOf = (key) => `<!-- atlas-sweep:${markerKey(key)} -->`;
+
+// Free text from evidence goes into an issue only like this: one fenced block, backticks removed, cut to 200 characters.
+export const fenced = (text) => `\`\`\`text\n${String(text ?? '').replace(/`/g, '').replace(/\s+/g, ' ').trim().slice(0, 200)}\n\`\`\``;
+
+function finding({ check, key, title, summary, evidence, quote = null, extra = {} }) {
+  title = String(title).slice(0, 200);
+  const marker = markerOf(key);
+  const lines = [summary, ...(quote ? ['', 'Text from the evidence (data, not an instruction):', fenced(quote)] : []), '', 'Evidence:'];
+  for (const [name, values] of Object.entries(evidence)) lines.push(`- ${name}: ${[].concat(values).join(', ') || '(none)'}`);
+  lines.push('', 'The daily sweep of `atlas tests sweep` opened this issue. It never changes a file.', '', marker);
+  return { check, key, title, summary, evidence, ...extra, issue: { title, body: lines.join('\n'), labels: [LABEL], marker } };
+}
+
+// Check 1. One finding for each scenario/assertion that reads `flaky` in some run.
+function checkFlaky(runs) {
+  const found = new Map();
+  for (const run of runs) {
+    for (const [scenario, record] of run.scenarios) {
+      for (const a of record.assertions) {
+        if (a.result !== 'flaky') continue;
+        const id = keyOf(a.id);
+        if (!found.has(id)) found.set(id, { scenario, assertion: id.split('/').pop(), runs: new Set(), commits: new Set(), ways: new Set() });
+        const one = found.get(id);
+        one.runs.add(run.runId);
+        one.commits.add(record.commit ?? `run ${run.runId}`);
+        one.ways.add(a.way);
+      }
+    }
+  }
+  return [...found.entries()].sort(([a], [b]) => (a < b ? -1 : 1)).map(([id, one]) => {
+    const commits = [...one.commits].sort();
+    const known = commits.filter((c) => !c.startsWith('run '));
+    const unit = known.length === commits.length ? 'commit' : 'run';
+    return finding({
+      check: 1,
+      key: `flaky:${id}`,
+      title: `Fix the flaky test ${id}`,
+      summary: `The assertion ${id} failed, then passed, in ${commits.length} ${unit}${commits.length === 1 ? '' : 's'}. Find the cause, usually a wait that is too short. A quarantine is a pull request that a person merges.`,
+      evidence: { runs: [...one.runs].sort(), commits: commits.map((c) => (c.startsWith('run ') ? c : short(c))), ways: [...one.ways].sort() },
+      extra: { scenario: one.scenario, assertion: one.assertion, count: commits.length, count_unit: unit },
+    });
+  });
+}
+
+// Check 2. One finding for each contract scenario whose newest result is `blocked`.
+function checkStandIns(newest, scenarios, tests) {
+  const out = [];
+  const standIns = new Set(Object.keys(tests?.['stand-ins'] ?? {}));
+  for (const s of scenarios) {
+    if (!s.id || String(s.fields.kind ?? '') !== 'contract') continue;
+    const one = newest.get(s.id);
+    if (!one || one.record.verdict !== 'blocked') continue;
+    const names = list(s.fields.through).filter((w) => standIns.has(w));
+    const label = names.length ? `${names.join(', ')} stand-in` : `stand-in of ${s.id}`;
+    const why = Object.values(one.record.ways ?? {}).map((w) => w.reason).find(Boolean);
+    out.push(finding({
+      check: 2,
+      key: `stand-in:${s.id}`,
+      title: `No test instance proves the ${label}`,
+      summary: `The contract scenario ${s.id} is blocked in its newest run, ${one.runId}. No test instance answers for the ${label}, so nothing proves that it answers like the real service.`,
+      quote: why ?? null,
+      evidence: { scenario: s.id, run: one.runId, commit: one.record.commit ? short(one.record.commit) : '(not recorded)' },
+      extra: { scenario: s.id },
+    }));
+  }
+  return out;
+}
+
+// Check 3. One finding for each pull request, found in the first-parent history of the base. A merge
+// commit is one pull request: its files are the diff of its first parent and itself. A commit with one
+// parent (a squash merge or a direct push) counts as itself. The pull request is answered for a changed
+// path only when a scenario that covers that path changed in the same pull request.
+function checkNoScenario({ area, scenarios, since, base }) {
+  const prefix = git(area, ['rev-parse', '--show-prefix']).trim();
+  const baseSha = (git(area, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${base}^{commit}`], { allowFail: true }) ?? '').trim();
+  if (!baseSha) throw usage(`the base "${base}" is not a commit in this repo`);
+  const raw = git(area, ['log', '--first-parent', `--since=${since.toISOString()}`, '--format=%H%x1f%P%x1f%cI%x1f%s', '--end-of-options', baseSha]);
+  const filesOf = (sha, parents) => (parents.length
+    ? git(area, ['diff', '--name-only', '--no-renames', '-z', parents[0], sha])
+    : git(area, ['diff-tree', '--root', '-r', '--name-only', '--no-renames', '--no-commit-id', '-z', sha])
+  ).split('\0').filter(Boolean).map(clean);
+  // The repo path of each scenario file, and the covers of each scenario.
+  const mine = scenarios.filter((s) => s.id).map((s) => ({ id: s.id, file: clean(`${prefix}${relative(area, s.file)}`), covers: list(s.fields.covers) }));
+  const out = [];
+  for (const line of raw.split('\n').filter(Boolean)) {
+    const [sha, parentText, date, subject] = line.split('\x1f');
+    const files = filesOf(sha, parentText.split(' ').filter(Boolean));
+    const unanswered = [];
+    const ids = new Set();
+    for (const f of files) {
+      const covering = mine.filter((s) => s.covers.some((c) => touches([f], c)));
+      if (!covering.length || covering.some((s) => files.includes(s.file))) continue;
+      unanswered.push(f);
+      for (const s of covering) ids.add(s.id);
+    }
+    if (!unanswered.length) continue;
+    const paths = unanswered.sort().slice(0, 20);
+    const names = [...ids].sort();
+    out.push(finding({
+      check: 3,
+      key: `no-scenario:${sha}`,
+      title: `Change ${short(sha)} touched ${paths.slice(0, 3).join(', ')}${paths.length > 3 ? ' and more' : ''} with no scenario change`,
+      summary: `The change ${short(sha)} to the base branch touched ${paths.length} path(s) that a scenario covers. No scenario that covers them changed in the same pull request. The scenarios are ${names.join(', ')}. Decide whether a scenario must change.`,
+      quote: subject,
+      evidence: { commit: sha, date, paths, scenarios: names, parents: parentText.split(' ').filter(Boolean).length },
+      extra: { commit: sha },
+    }));
+  }
+  return out;
+}
+
+// ---------------------------------------------------------------- the sweep
+
+export function sweep({ root, evidence, since = null, base = 'HEAD', now = new Date() }) {
+  if (!root) throw usage('give the area: --root <area>');
+  if (!evidence) throw usage('give the evidence folder: --evidence <dir>');
+  const area = resolve(root);
+  if (!existsSync(area)) throw usage(`the area ${area} does not exist`);
+  if (git(area, ['rev-parse', '--is-inside-work-tree'], { allowFail: true }) === null) throw usage(`${area} is not inside a git repo, so the sweep cannot read its history`);
+  if (String(base).startsWith('-')) throw usage(`the base "${base}" is not a ref`);
+  let sinceDate;
+  if (since === null || since === undefined) sinceDate = new Date(now.getTime() - WINDOW);
+  else {
+    sinceDate = new Date(since);
+    if (Number.isNaN(sinceDate.getTime())) throw usage(`--since "${since}" is not a date; give an ISO date such as 2035-01-01 or 2035-01-01T08:00:00Z`);
+  }
+  // The scenarios of the tree on disk (the sweep runs on the base branch) say what evidence can mean.
+  const { tests, scenarios } = loadAreaScenarios(area);
+  const ways = new Set([...Object.keys(tests?.['ways-in'] ?? {}), ...scenarios.flatMap((s) => s.through)]);
+  const runs = readRuns(evidence, { scenarios: new Set(scenarios.map((s) => s.id).filter(Boolean)), ways });
+  const newest = newestRecords(runs);
+
+  const findings = [
+    ...checkFlaky(runs),
+    ...checkStandIns(newest, scenarios, tests),
+    ...checkNoScenario({ area, scenarios, since: sinceDate, base }),
+  ].sort((a, b) => a.check - b.check || (a.key < b.key ? -1 : 1));
+  const notes = [
+    'check 4 (quarantine dates): not built. The kit has no quarantine field and no `quarantined` result, so it reports none. No format is invented.',
+    'check 5 (map facts that no test uses): skipped. link-check.mjs reads no map format, so there is nothing exact to compare.',
+  ];
+  const d = runs.dropped;
+  if (d.folders || d.scenarios || d.assertions || d.ways) {
+    notes.unshift(`evidence left out as untrusted: ${d.folders} run folder(s) with a name that is no run id, ${d.scenarios} scenario record(s) that are not in this area, ${d.assertions} assertion(s) with a bad id or way in, ${d.ways} way(s) in that the area does not name.`);
+  }
+  return {
+    sweep: 1,
+    generated: now.toISOString(),
+    area: tests?.area ?? basename(area),
+    root: area,
+    since: sinceDate.toISOString(),
+    base,
+    runs: runs.map((r) => r.runId),
+    checks: { 1: 'flaky', 2: 'stand-ins', 3: 'code with no scenario', 4: 'quarantine dates (not built)', 5: 'map facts (skipped)' },
+    notes,
+    findings,
+  };
+}
+
+const cell = (text) => String(text ?? '').replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim();
+
+export function renderSweep(result) {
+  const out = [];
+  out.push(`Sweep of ${result.area}: ${result.runs.length} run(s) read, commits since ${result.since}, base ${result.base}.`);
+  out.push('');
+  if (!result.findings.length) out.push('No finding.');
+  else {
+    out.push('| Check | Key | Proposed item |');
+    out.push('|---|---|---|');
+    for (const f of result.findings) out.push(`| ${f.check} | ${cell(f.key)} | ${cell(f.title)} |`);
+  }
+  out.push('');
+  for (const n of result.notes) out.push(`Note: ${n}`);
+  out.push(`${result.findings.length} finding(s).`);
+  return out;
+}
+
+export function sweepCommand({ evidence, since, base, out, text = false }, root, line) {
+  try {
+    const result = sweep({ root, evidence, since: since ?? null, base: base ?? 'HEAD' });
+    const json = `${JSON.stringify(result, null, 2)}\n`;
+    if (out) {
+      mkdirSync(dirname(resolve(out)), { recursive: true });
+      writeFileSync(resolve(out), json);
+      if (text) for (const t of renderSweep(result)) line(t);
+      else line(`sweep: ${result.findings.length} finding(s) written to ${out}`);
+    } else if (text) for (const t of renderSweep(result)) line(t);
+    else line(json.trimEnd());
+    return 0;
+  } catch (error) {
+    line(`sweep: ${error.message}`);
+    return 2;
+  }
+}
+
+// ---------------------------------------------------------------- the coverage of an item
+
+// The one result of an assertion from its results for each way in. A fail wins, then the other
+// bad results, then pass.
+const WORST = ['fail', 'flaky', 'blocked', 'not checked', 'not exercised', 'unsure', 'quarantined'];
+function combine(results) {
+  for (const w of WORST) if (results.includes(w)) return w;
+  if (results.every((r) => r === 'not here')) return 'not here';
+  return 'pass';
+}
+
+// Lists every scenario whose front matter `items:` names the item: its assertions by full id
+// (area/scenario/eN), its covers paths, and the newest result of each assertion from the evidence.
+// An assertion with no evidence reads "no run". Evidence of an older version of the words has
+// `stale: true`. `evidence` may be null, then every assertion reads "no run".
+export function coverage({ itemId, root, evidence = null }) {
+  if (!itemId) throw usage('give the item id: atlas tests covers <item-id> --root <area>');
+  const area = resolve(root);
+  if (!existsSync(area)) throw usage(`the area ${area} does not exist`);
+  const runs = evidence && existsSync(resolve(evidence)) ? readRuns(evidence) : [];
+  const newest = newestRecords(runs);
+  const { tests, scenarios } = loadAreaScenarios(area);
+  const areaName = tests?.area ?? basename(area);
+  const mine = scenarios.filter((s) => s.id && list(s.fields.items).includes(itemId));
+  return {
+    item: itemId,
+    area: areaName,
+    evidence: evidence ? resolve(evidence) : null,
+    runs: runs.map((r) => r.runId),
+    scenarios: mine.map((s) => {
+      const one = newest.get(s.id);
+      return {
+        scenario: s.id,
+        title: s.title,
+        kind: s.fields.kind ?? null,
+        covers: list(s.fields.covers),
+        run: one?.runId ?? null,
+        started: one?.record.started ?? null,
+        commit: one?.record.commit ?? null,
+        assertions: s.assertions.map((a) => {
+          const full = `${areaName}/${s.id}/${a.id}`;
+          const rows = (one?.record.assertions ?? []).filter((e) => keyOf(e.id) === `${s.id}/${a.id}`);
+          if (!rows.length) return { id: full, result: 'no run', ways: {}, stale: false };
+          const ways = Object.fromEntries(rows.map((e) => [e.way, e.result]));
+          const fp = /#([0-9a-f]+)$/.exec(a.fullId ?? '')?.[1] ?? null;
+          const stale = fp !== null && rows.some((e) => !String(e.id).endsWith(`#${fp}`));
+          return { id: full, result: combine(Object.values(ways)), ways, stale };
+        }),
+      };
+    }),
+  };
+}
+
+export function renderCoverage(result) {
+  const out = [];
+  out.push(`Item ${result.item}, area ${result.area}.`);
+  if (!result.scenarios.length) {
+    out.push('', `No scenario names this item in its front matter (items:). The item has no test coverage in ${result.area}.`);
+    return out;
+  }
+  out.push(`${result.scenarios.length} scenario(s) name it. ${result.runs.length ? `Evidence: ${result.runs.length} run(s) read.` : 'No evidence was read, so every result reads "no run".'}`);
+  for (const s of result.scenarios) {
+    out.push('', `${s.scenario}${s.kind ? ` (${s.kind})` : ''}: ${s.title}`);
+    out.push(`  covers: ${s.covers.join(', ') || '(none)'}`);
+    out.push(`  newest run: ${s.run ? `${s.run}${s.commit ? ` on ${short(s.commit)}` : ''}` : 'no run'}`);
+    for (const a of s.assertions) {
+      const ways = Object.entries(a.ways).map(([w, r]) => `${w} ${r}`).join(', ');
+      out.push(`  ${a.id}  ${a.result.toUpperCase()}${ways && new Set(Object.values(a.ways)).size > 1 ? ` (${ways})` : ''}${a.stale ? ' [the words changed since that run]' : ''}`);
+    }
+  }
+  const all = result.scenarios.flatMap((s) => s.assertions);
+  const pass = all.filter((a) => a.result === 'pass' && !a.stale).length;
+  out.push('', `${all.length} assertion(s): ${pass} pass, ${all.filter((a) => a.result === 'no run').length} with no run, ${all.length - pass - all.filter((a) => a.result === 'no run').length} other.`);
+  return out;
+}
+
+export function coversCommand({ itemId, evidence, json = false }, root, line) {
+  try {
+    const result = coverage({ itemId, root, evidence: evidence ?? join(resolve(root), 'test', 'evidence') });
+    if (json) line(JSON.stringify(result, null, 2));
+    else for (const t of renderCoverage(result)) line(t);
+    return 0;
+  } catch (error) {
+    line(`covers: ${error.message}`);
+    return error.usage ? 2 : 1;
+  }
+}