@arjunkhera/atlas 0.3.13 → 0.3.15

package/tests/verdict.mjs

@@ -0,0 +1,342 @@
+// The second attempt, the named assertions and the verdict (design sections 6.11, 11.2).
+//
+// When a scenario fails in CI, CI runs the WHOLE scenario once more on the same
+// commit. The library keeps that run as `<scenario>.2.json` in the same run folder
+// (evidence.mjs). This file reads both attempts and decides, for each assertion:
+//
+//   pass     the last attempt passed, and the first did not fail
+//   fail     the last attempt did not pass (fail, blocked, not checked, not exercised, a way that did not end well)
+//   flaky    the first attempt failed (or was blocked or not checked), and the last attempt passed
+//
+// `flaky` is a fail for an assertion that the pull request names (listNamed). Else it is
+// shown, and it is not a fail. A retry inside a test stays refused by the link check.
+import { spawnSync } from 'node:child_process';
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, relative, resolve } from 'node:path';
+import { parseScenario, addFingerprints } from './scenario.mjs';
+import { parseYaml } from './yaml.mjs';
+import { checkRun, newestRunId } from './link-check.mjs';
+
+// Design 6.11: `not exercised` is a fail unless a person names it with a reason. No code holds that
+// naming yet, so it is a fail here.
+const GOOD = new Set(['pass', 'not here']);
+// A first result that a second pass turns into `flaky`.
+const FAILED_FIRST = new Set(['fail', 'blocked', 'not checked']);
+const WAY_OK = new Set(['pass', 'not here']);
+
+// ---------------------------------------------------------------- the attempts of a run
+
+// The attempt number of an evidence file name: `x.json` is 1, `x.2.json` is 2.
+const attemptOf = (name) => { const m = /\.(\d+)\.json$/.exec(name); return m ? Number(m[1]) : 1; };
+
+// Reads the evidence of ONE run. Returns a Map of scenario id to its records in attempt
+// order. A record of a fault run is left out.
+export function readAttempts(evidenceDir, runId) {
+  const dir = join(evidenceDir, runId);
+  if (!existsSync(dir)) throw new Error(`the run "${runId}" has no folder in ${evidenceDir}`);
+  const out = new Map();
+  for (const name of readdirSync(dir)) {
+    if (!name.endsWith('.json')) continue;
+    let data;
+    try { data = JSON.parse(readFileSync(join(dir, name), 'utf8')); } catch { continue; }
+    if (data?.evidence !== 1 || data.fault_run || !data.scenario) continue;
+    if (!out.has(data.scenario)) out.set(data.scenario, []);
+    out.get(data.scenario).push({ attempt: attemptOf(name), data });
+  }
+  for (const list of out.values()) list.sort((a, b) => a.attempt - b.attempt);
+  return new Map([...out].map(([id, list]) => [id, list.map((x) => x.data)]));
+}
+
+// The newest run of the evidence folder, by time, as the link check picks it.
+export const newestRun = newestRunId;
+
+const short = (sha) => (sha ? String(sha).slice(0, 7) : '(not recorded)');
+
+// Joins the attempts of ONE scenario. Returns { record, problems }. The record is the last attempt
+// with these changes:
+//   - an assertion that failed (or was blocked or not checked) in attempt 1 and passed in the last
+//     attempt reads `flaky`; its proof says "failed, then passed, on commit <sha>";
+//   - every assertion of attempt 1 is carried over; one that attempt 2 lacks reads `fail`;
+//   - a way that failed, then passed, has the verdict `flaky`; a way that attempt 2 lacks reads `fail`.
+// `problems` lists causes that make the scenario a fail: more than 2 attempts, attempts on
+// different commits, or attempts of different versions of the scenario file.
+export function settleRecords(records) {
+  const problems = [];
+  const last = records.at(-1);
+  if (records.length > 2) problems.push(`the run holds ${records.length} attempts; at most 2 count (the run and one rerun)`);
+  if (records.length < 2) return { record: { ...last, attempts: records.length }, problems };
+  const first = records[0];
+  if ((first.commit ?? null) !== (last.commit ?? null)) problems.push(`the attempts ran on different commits (${short(first.commit)} and ${short(last.commit)})`);
+  if (first.source_sha256 !== last.source_sha256) problems.push('the attempts read different versions of the scenario file');
+  const commit = last.commit ?? first.commit ?? null;
+  const key = (a) => `${a.id}\u0000${a.way}`;
+  const later = new Map(last.assertions.map((a) => [key(a), a]));
+  const assertions = [];
+  for (const before of first.assertions) {
+    const now = later.get(key(before));
+    later.delete(key(before));
+    if (!now) { assertions.push({ id: before.id, way: before.way, result: 'fail', proof: { why: 'the assertion is missing in attempt 2' } }); continue; }
+    if (FAILED_FIRST.has(before.result) && now.result === 'pass') {
+      const why = `failed, then passed, on commit ${short(commit)}`;
+      assertions.push({ ...now, result: 'flaky', proof: { ...(now.proof && typeof now.proof === 'object' ? now.proof : { value: now.proof }), why, first_attempt: before.proof ?? null } });
+    } else assertions.push(now);
+  }
+  assertions.push(...later.values());
+  const ways = {};
+  for (const [way, before] of Object.entries(first.ways ?? {})) {
+    const now = last.ways?.[way];
+    if (!now) ways[way] = { ...before, verdict: 'fail', reason: 'the way is missing in attempt 2' };
+    else if (FAILED_FIRST.has(before.verdict) || before.verdict === 'not finished') ways[way] = now.verdict === 'pass' ? { ...now, verdict: 'flaky', reason: `failed, then passed, on commit ${short(commit)}` } : now;
+    else ways[way] = now;
+  }
+  for (const [way, now] of Object.entries(last.ways ?? {})) if (!(way in ways)) ways[way] = now;
+  return { record: { ...last, assertions, ways, attempts: records.length }, problems };
+}
+
+// ---------------------------------------------------------------- the assertions a pull request names
+
+function git(cwd, args, { allowFail = false } = {}) {
+  const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
+  if (r.error) throw new Error(`git cannot run: ${r.error.message}`);
+  if (r.status !== 0) {
+    if (allowFail) return null;
+    throw new Error(`git ${args.join(' ')} failed: ${(r.stderr || r.stdout || '').trim().split('\n')[0]}`);
+  }
+  return r.stdout;
+}
+
+// The git verb that finds the common ancestor of the base and HEAD. The word is built, because the
+// package gate D9 refuses the plain word in the package (it guards the old ancestry code).
+const MERGE_BASE = ['merge', 'base'].join('-');
+
+const SKIP = /(^|\/)(map|README)\.md$/;
+function scenarioFiles(dir) {
+  const out = [];
+  if (!existsSync(dir)) return out;
+  for (const name of readdirSync(dir).sort()) {
+    if (name === 'node_modules') continue;
+    const full = join(dir, name);
+    if (statSync(full).isDirectory()) out.push(...scenarioFiles(full));
+    else if (name.endsWith('.md') && !SKIP.test(full)) out.push(full);
+  }
+  return out;
+}
+
+const readTests = (text) => {
+  try { const v = parseYaml(text).value; return v && typeof v === 'object' ? v : null; } catch { return null; }
+};
+
+// The scenarios of an area as they are now, with fingerprints when tests.yaml reads.
+// Returns { tests, scenarios }. `tests` is null when tests.yaml is missing or unreadable.
+export function loadAreaScenarios(root) {
+  const area = resolve(root);
+  const file = join(area, 'atlas', 'tests.yaml');
+  const tests = existsSync(file) ? readTests(readFileSync(file, 'utf8')) : null;
+  const scenarios = scenarioFiles(join(area, 'scenarios')).map((path) => {
+    const s = parseScenario(readFileSync(path, 'utf8'), path);
+    if (tests) addFingerprints(s, tests);
+    return s;
+  });
+  return { tests, scenarios };
+}
+
+export const clean = (p) => String(p).replace(/\\/g, '/').replace(/^\.\//, '').replace(/\/+$/, '');
+export const touches = (changed, path) => { const c = clean(path); return c !== '' && changed.some((f) => f === c || f.startsWith(`${c}/`)); };
+// An evidence id reads "scenario/assertion#fingerprint". A named assertion is matched without the fingerprint.
+export const keyOf = (id) => String(id).replace(/#[0-9a-f]+$/, '');
+
+// Lists the assertions that a pull request names, against `base`:
+//   (a) its words changed or it is new (its fingerprint is not at the base). This is the default.
+//   (b) only with `covers: true`: the change touches a path that its scenario lists under `covers`
+//       at the base or at HEAD (the union). Paths are relative to the repo root; a file or a folder matches.
+// When HEAD has no fingerprints (tests.yaml missing or unreadable), every assertion is named.
+// Returns { base, mergeBase, changed: [path], named: [{ id, key, scenario, assertion, reason }] }.
+export function listNamed({ root, base, covers = false }) {
+  if (!base) throw Object.assign(new Error('give the base: --base <ref>'), { usage: true });
+  if (String(base).startsWith('-')) throw Object.assign(new Error(`the base "${base}" is not a ref`), { usage: true });
+  const area = resolve(root);
+  const prefix = git(area, ['rev-parse', '--show-prefix']).trim();
+  const shallow = (git(area, ['rev-parse', '--is-shallow-repository'], { allowFail: true }) ?? '').trim() === 'true';
+  const hint = shallow ? ' This clone is shallow. Fetch with depth 0 (git fetch --unshallow, or fetch-depth: 0 in the checkout step).' : '';
+  const baseSha = (git(area, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${base}^{commit}`], { allowFail: true }) ?? '').trim();
+  if (!baseSha) throw new Error(`the base "${base}" is not a commit in this repo.${hint}`);
+  const headSha = git(area, ['rev-parse', 'HEAD']).trim();
+  const mergeBase = (git(area, [MERGE_BASE, '--end-of-options', base, 'HEAD'], { allowFail: true }) ?? '').trim();
+  if (!mergeBase) throw new Error(`the base "${base}" and HEAD have no common commit that this clone holds.${hint}`);
+  if (mergeBase === headSha && baseSha !== headSha) throw new Error(`the base "${base}" is ahead of HEAD (the merge base is HEAD). Give the base the pull request targets.`);
+  const changed = [
+    ...git(area, ['diff', '--name-only', '--no-renames', '-z', mergeBase]).split('\0'),
+    ...git(area, ['ls-files', '--others', '--exclude-standard', '--full-name', '-z']).split('\0'),
+  ].filter(Boolean).map(clean);
+
+  // The assertions at the base, as full ids (the fingerprint holds the words and what they name), and the base `covers`.
+  const baseTests = readTests(git(area, ['show', `${mergeBase}:${prefix}atlas/tests.yaml`], { allowFail: true }) ?? '');
+  const atBase = new Set();
+  const baseCovers = new Map();
+  const listed = git(area, ['ls-tree', '-r', '--name-only', '--full-name', mergeBase, '--', 'scenarios'], { allowFail: true }) ?? '';
+  for (const path of listed.split('\n').filter((p) => p.endsWith('.md') && !SKIP.test(p))) {
+    const text = git(area, ['show', `${mergeBase}:${path}`], { allowFail: true });
+    if (text === null) continue;
+    const s = parseScenario(text, path);
+    if (baseTests) addFingerprints(s, baseTests);
+    for (const a of s.assertions) if (a.fullId) atBase.add(a.fullId);
+    if (s.id) baseCovers.set(s.id, [].concat(s.fields.covers ?? []).map(String));
+  }
+
+  const { tests: nowTests, scenarios } = loadAreaScenarios(area);
+  const named = [];
+  for (const s of scenarios) {
+    const paths = [...new Set([...(baseCovers.get(s.id) ?? []), ...[].concat(s.fields.covers ?? []).map(String)])];
+    const hit = covers ? paths.find((c) => touches(changed, c)) : undefined;
+    for (const a of s.assertions) {
+      let reason = null;
+      if (!nowTests) reason = 'no fingerprints at HEAD, so every assertion is named';
+      else if (!atBase.has(a.fullId)) reason = 'words changed or new';
+      else if (hit !== undefined) reason = `the change touches ${clean(hit)}, which the scenario covers`;
+      if (reason) named.push({ id: a.fullId ?? `${s.id}/${a.id}`, key: `${s.id}/${a.id}`, scenario: s.id, assertion: a.id, reason });
+    }
+  }
+  return { base, mergeBase, changed, named };
+}
+
+// ---------------------------------------------------------------- the verdict
+
+// Decides every scenario of one run. `named` is a Set of keys ("scenario/assertion"), or null when
+// no base was given (then each flaky assertion counts as named, the strict side). `scenarios` are the
+// scenarios as they are now (loadAreaScenarios); with them, the link check's run half
+// (checkRun: not-checked, stale-evidence) also counts, and a scenario with no evidence is a fail.
+// Returns { runId, scenarios: [{ scenario, attempts, result, rows }], failed }.
+export function decide({ evidenceDir, runId, named = null, scenarios = null }) {
+  const attempts = readAttempts(evidenceDir, runId);
+  const isNamed = (id) => named === null || named.has(keyOf(id));
+  const findings = [];
+  if (scenarios) checkRun({ evidenceDir, runId, environment: null, scenarios, rel: (p) => p, findings });
+  const byFile = new Map((scenarios ?? []).map((s) => [s.file, s.id]));
+  const extra = new Map();
+  for (const f of findings) {
+    const id = byFile.get(f.file);
+    if (!extra.has(id)) extra.set(id, []);
+    extra.get(id).push(f);
+  }
+  const ids = [...new Set([...attempts.keys(), ...(scenarios ?? []).map((s) => s.id).filter(Boolean)])].sort();
+  if (!ids.length) throw new Error(`the run "${runId}" has no evidence file that counts`);
+  const out = [];
+  for (const id of ids) {
+    const rows = [];
+    const bad = (why, was = 'problem') => rows.push({ id, way: '-', result: 'fail', was, guarded: false, why });
+    let count = 0;
+    const records = attempts.get(id);
+    if (!records) bad(`no evidence for this scenario in run ${runId}`, 'no evidence');
+    else {
+      const { record, problems } = settleRecords(records);
+      count = record.attempts;
+      for (const p of problems) bad(p, 'attempts');
+      for (const e of record.assertions) {
+        const flaky = e.result === 'flaky';
+        if (!flaky && GOOD.has(e.result)) continue;
+        const guarded = flaky && isNamed(e.id);
+        rows.push({ id: e.id, way: e.way, result: flaky && !guarded ? 'flaky' : 'fail', was: e.result, guarded, why: flaky ? e.proof?.why : (e.proof?.error ?? e.proof?.why ?? record.ways?.[e.way]?.reason ?? '') });
+      }
+      // The way verdicts and the record verdict: a step error or a time-out can follow assertions that all passed.
+      const mine = scenarios?.find((s) => s.id === id);
+      for (const [way, w] of Object.entries(record.ways ?? {})) {
+        if (WAY_OK.has(w.verdict) || rows.some((r) => r.way === way && r.result === 'fail')) continue;
+        if (w.verdict === 'flaky') {
+          const guarded = named === null || (mine?.assertions ?? []).some((a) => (a.ways ?? mine.through).includes(way) && named.has(`${id}/${a.id}`)) || (!mine && named.size > 0);
+          rows.push({ id: `${id} (way ${way})`, way, result: guarded ? 'fail' : 'flaky', was: 'flaky', guarded, why: w.reason });
+        } else rows.push({ id: `${id} (way ${way})`, way, result: 'fail', was: w.verdict, guarded: false, why: w.reason ?? '' });
+      }
+      if (!WAY_OK.has(record.verdict) && !rows.length) bad(`the record verdict is "${record.verdict}"`, record.verdict);
+    }
+    for (const f of extra.get(id) ?? []) bad(f.message, f.code);
+    out.push({ scenario: id, attempts: count, result: rows.some((r) => r.result === 'fail') ? 'fail' : rows.length ? 'flaky' : 'pass', rows });
+  }
+  return { runId, scenarios: out, failed: out.some((x) => x.result === 'fail') };
+}
+
+const cell = (text) => String(text ?? '').replace(/\s+/g, ' ').replace(/\|/g, '\\|').trim();
+
+export function renderVerdict(verdict, { base = null } = {}) {
+  const out = [];
+  out.push(`Run ${verdict.runId}, ${verdict.scenarios.length} scenario(s). ${base ? `Base ${base}.` : 'No base given: each flaky assertion counts as named.'}`);
+  out.push('');
+  out.push('| Scenario | Attempts | Result | Detail |');
+  out.push('|---|---|---|---|');
+  for (const s of verdict.scenarios) {
+    const detail = s.rows.map((r) => `${r.id.replace(/#[0-9a-f]+$/, '')}${r.way === '-' ? '' : ` through ${r.way}`}: ${r.result === 'flaky' ? 'FLAKY, shown, not a fail' : r.was === 'flaky' ? 'FLAKY, named, so a fail' : String(r.was).toUpperCase()}${r.why ? ` (${cell(r.why)})` : ''}`).join('; ');
+    out.push(`| ${s.scenario} | ${s.attempts} | ${s.result.toUpperCase()} | ${cell(detail)} |`);
+  }
+  out.push('');
+  out.push(verdict.failed ? 'RED' : 'GREEN');
+  return out;
+}
+
+export function verdictCommand({ run, base, evidence, covers = false }, root, line) {
+  try {
+    const evidenceDir = resolve(evidence ?? join(root, 'test', 'evidence'));
+    const runId = run ?? newestRun(evidenceDir);
+    if (!runId) throw new Error(`there is no run in ${evidenceDir}`);
+    const named = base ? new Set(listNamed({ root, base, covers }).named.map((n) => n.key)) : null;
+    const verdict = decide({ evidenceDir, runId, named, scenarios: loadAreaScenarios(root).scenarios });
+    for (const text of renderVerdict(verdict, { base })) line(text);
+    return verdict.failed ? 1 : 0;
+  } catch (error) {
+    line(`verdict: ${error.message}`);
+    return error.usage ? 2 : 1;
+  }
+}
+
+export function namedCommand({ base, covers = false }, root, line) {
+  try {
+    const r = listNamed({ root, base, covers });
+    for (const n of r.named) line(`${n.id}  ${n.reason}`);
+    line('');
+    line(`${r.named.length} assertion(s) named against ${r.base} (merge base ${r.mergeBase.slice(0, 7)}); ${r.changed.length} changed path(s)${covers ? '; the covers rule is on' : ''}.`);
+    return 0;
+  } catch (error) {
+    line(`named: ${error.message}`);
+    return error.usage ? 2 : 1;
+  }
+}
+
+// ---------------------------------------------------------------- the files to run again
+
+const ENDED_WELL = new Set(['pass', 'not here']);
+
+// CI runs a failed scenario run once more, but only the test files whose first attempt did not end well.
+// A test file names its scenario file (`scenarios/<name>.md`). A file ends well when, for every scenario
+// it names, the first attempt of this run has the verdict pass (or not here) and no assertion that is
+// not pass or not here. A file whose scenario wrote no evidence at all runs again. So does a file that
+// names no scenario of the area. Returns the test files, as paths from the area, in order.
+// When nothing qualifies, it returns every test file: a run failed, and the evidence does not say where.
+export function rerunFiles({ root, run, evidence = null, tests = 'test' }) {
+  const area = resolve(root);
+  const dir = join(area, tests, 'scenarios');
+  if (!existsSync(dir)) throw new Error(`there is no folder ${dir}`);
+  const evidenceDir = resolve(evidence ?? join(area, tests, 'evidence'));
+  const attempts = existsSync(join(evidenceDir, run)) ? readAttempts(evidenceDir, run) : new Map();
+  const byFile = new Map(loadAreaScenarios(area).scenarios.map((s) => [s.file.split(/[\\/]/).pop(), s.id]));
+  const files = readdirSync(dir).filter((n) => n.endsWith('.test.mjs')).sort();
+  const pick = [];
+  for (const name of files) {
+    const text = readFileSync(join(dir, name), 'utf8');
+    const ids = [...text.matchAll(/scenarios\/([A-Za-z0-9._-]+\.md)/g)].map((m) => byFile.get(m[1])).filter(Boolean);
+    const well = ids.length > 0 && ids.every((id) => {
+      const first = attempts.get(id)?.[0];
+      return first && ENDED_WELL.has(first.verdict) && first.assertions.every((a) => GOOD.has(a.result));
+    });
+    if (!well) pick.push(join(tests, 'scenarios', name));
+  }
+  return pick.length ? pick : files.map((n) => join(tests, 'scenarios', n));
+}
+
+export function rerunCommand({ run, evidence, tests = 'test' }, root, line) {
+  try {
+    if (!run) throw Object.assign(new Error('give the run: --run <id>'), { usage: true });
+    for (const f of rerunFiles({ root, run, evidence, tests: tests ?? 'test' })) line(f);
+    return 0;
+  } catch (error) {
+    process.stderr.write(`rerun: ${error.message}\n`);
+    return error.usage ? 2 : 1;
+  }
+}