@assuredloop/cli 0.0.0-stage → 0.1.0

package/src/tests.js

@@ -0,0 +1,267 @@
+// Tests and their results ([LNK-3]), observed and never judged: test files
+// by common path patterns or the `tests:` paths of .assuredloop; assertion
+// counts for JS/TS, Python and Go; the result files `results:` names, each
+// classed by the revision it names; the tests linked to the changed code,
+// each with its reason; and [HNT-2]'s four test notes.
+import { basename } from 'node:path';
+import { git, hasCommits, isShallow } from './git.js';
+import { configured } from './spec.js';
+import { line } from './commands.js';
+import { fileLinks } from './views.js';
+import { adrFolders } from './adrs.js';
+import { changedWith, idNear, idsOn, paths } from './links.js';
+
+const DIR = /(^|\/)(test|tests|__tests__|spec)\//;
+const NAME = /^test_|\.(test|spec)\.|_test\.|Tests?\.[^.]+$/;
+const MARK = /^test_|\.(test|spec)(?=\.)|_test(?=\.)|Tests?(?=\.[^.]+$)/g;
+const JS = /\.[cm]?[jt]sx?$/;
+const ASSERT = /(?<![\w$.])(?:t\.)?assert(?:\.\w+)*\s*\(|(?<![\w$.])expect\s*\(/g;
+// Python and Go: a line that starts with an assertion, no tokenizer (#130).
+const LINE_ASSERT = [
+  [/^test_.*\.py$|_test\.py$/, /^(?:assert |self\.assert\w*\(|(?:with )?pytest\.raises\()/],
+  [/_test\.go$/, /^(?:t\.(?:Errorf?|Fatalf?)|(?:assert|require)\.\w+)\(/],
+];
+const REVISION = /\brevision:[ \t]*([0-9a-f]{7,40})\b/i;
+const stem = (p) => basename(p).replace(MARK, '').replace(/\..*$/, '');
+const COMMENT = /\/\/[^\n]*|\/\*[\s\S]*?\*\//y;
+const STRING = /'(?:\\.|[^'\\\n])*'|"(?:\\.|[^"\\\n])*"/y;
+const TEXT = /(?:\\[\s\S]|[^`\\$]|\$(?!\{))*(?:`|\$\{)?/y;
+const REGEX = /\/(?![*/])(?:\\.|\[(?:\\.|[^\]\\\n])*\]|[^/\\\n[])+\/[a-z]*/y;
+// A `/` starts a regex after these, not after a value (where it divides).
+const OPERAND = /(?:^|[(,=:[!&|?{};+\-*%<>~^]|\b(?:return|typeof|case|do|else|in|of|new|delete|void|throw|yield|await)) ?$/;
+
+// The code of JS/TS text: comments, string and regex contents and template
+// text blanked, lines kept, so only code holds calls; a template's `${…}` stays code.
+function codeOf(t) {
+  const blank = (s) => s.replace(/[^\n]/g, ' ');
+  const open = []; // one per `{`: a plain brace, or a template's `${`
+  let [out, i, prev] = ['', 0, '']; // prev: the last code characters, spaces squeezed, a value as x
+  const take = (re) => { re.lastIndex = i; return re.exec(t)?.[0] || null; };
+  const skip = (m, value) => { out += blank(m); i += m.length; if (value) prev = `${prev}x`.slice(-8); };
+  // Template text up to its closing backtick, or up to a `${`, whose code follows.
+  const text = () => { const m = take(TEXT) ?? ''; skip(m, m.endsWith('`')); if (m.endsWith('${')) open.push('template'); };
+  while (i < t.length) {
+    const m = take(COMMENT);
+    if (m) { skip(m); continue; }
+    const v = take(STRING) ?? (t[i] === '/' && OPERAND.test(prev) ? take(REGEX) : null);
+    if (v) { skip(v, true); continue; }
+    const c = t[i++];
+    if (c === '`') { out += ' '; text(); continue; }
+    if (c === '}' && open.pop() === 'template') { out += ' '; text(); continue; }
+    if (c === '{') open.push('brace');
+    out += c;
+    if (!/\s/.test(c) || !prev.endsWith(' ')) prev = `${prev}${/\s/.test(c) ? ' ' : c}`.slice(-8);
+  }
+  return out;
+}
+
+// Whether a path is a test file; never one under the baseline root or requests/.
+export const testMatcher = (root, named) => (p) => !p.startsWith(`${root}/`) && !p.startsWith('requests/') &&
+  (DIR.test(p) || NAME.test(basename(p)) || named.some((n) => p === n || p.startsWith(`${n}/`)));
+
+// A test file's assertions from `before` to `after` (null when absent): the
+// counts, and the assertion lines added and removed; other syntax is skipped.
+function observe(path, before, after) {
+  const starts = LINE_ASSERT.find(([name]) => name.test(basename(path)))?.[1];
+  if (!JS.test(path) && !starts) return { path, skipped: 'not JS/TS, Python or Go, the syntax al counts' };
+  const lines = starts ? (t) => (t ?? '').split('\n').map((l) => l.trim()).filter((l) => starts.test(l))
+    : (t) => { const src = (t ?? '').split('\n'); return codeOf(t ?? '').split('\n').flatMap((l, i) => (l.match(ASSERT) ? [src[i].trim()] : [])); };
+  const minus = (x, y) => { const left = [...y]; return x.filter((l) => { const i = left.indexOf(l); return i < 0 || !left.splice(i, 1); }); };
+  const count = starts ? (t) => lines(t).length : (t) => codeOf(t ?? '').match(ASSERT)?.length ?? 0;
+  const [a, b] = [lines(before), lines(after)];
+  return { path, was: count(before), now: count(after), added: minus(b, a).length, removed: minus(a, b).length };
+}
+const moved = (o) => !o.skipped && (o.added || o.removed);
+const observation = (o) => (o.skipped ? `${o.path}  skipped (${o.skipped})` : `${o.path}  ${o.was} → ${o.now} assertions (+${o.added} -${o.removed} lines)`);
+
+// A result file's counts and its passing and failing test names: JUnit XML's
+// testcases, or TAP's points (a SKIP or TODO is neither), each named under
+// its `# Subtest:` parents, and whether the TAP is incomplete or invalid: it
+// bails out, or a stream (the root, or a subtest's by its indent) has no plan
+// or a count of points other than its plan; null when it is neither.
+function parse(text) {
+  const r = { passed: 0, failed: 0, failing: new Set(), passing: new Set() };
+  const note = (name, failed) => (failed ? (r.failed++, r.failing.add(name)) : (r.passed++, r.passing.add(name)));
+  if (/<testcase\b/.test(text)) {
+    for (const [, attrs, inner = ''] of text.matchAll(/<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/g)) {
+      if (!/<skipped\b/.test(inner)) note(attrs.match(/\bname="([^"]*)"/)?.[1] ?? '', /<(failure|error)\b/.test(inner));
+    }
+    return r;
+  }
+  const [parents, streams] = [[], []];
+  let [points, bail, bad] = [0, false, null];
+  // The streams deeper than `at` end: each needs its plan met.
+  const close = (at) => {
+    while (streams.at(-1)?.at > at) {
+      const { plan, n } = streams.pop();
+      bad ??= plan === null ? 'incomplete: no plan' : n < plan ? `incomplete: ${n} of a plan of ${plan}` : n > plan ? `invalid: ${n} points for a plan of ${plan}` : null;
+    }
+  };
+  const stream = (at) => {
+    close(at);
+    if (streams.at(-1)?.at !== at) streams.push({ at, plan: null, n: 0 });
+    return streams.at(-1);
+  };
+  for (const l of text.split('\n')) {
+    const at = l.match(/^[ \t]*/)[0].length;
+    const sub = l.match(/^[ \t]*# Subtest: (.*)$/);
+    const point = l.match(/^[ \t]*(not )?ok\b[ \t]*\d*[ \t]*(?:-[ \t]*)?(.*)$/);
+    if (sub) {
+      while (parents.at(-1)?.at >= at) parents.pop();
+      parents.push({ at, name: sub[1].trim() });
+    } else if (/^[ \t]*Bail out!/.test(l)) bail = true;
+    else if (/^[ \t]*1\.\.\d+/.test(l)) stream(at).plan = Number(l.match(/1\.\.(\d+)/)[1]);
+    else if (point) {
+      points++;
+      stream(at).n++;
+      const name = [...parents.filter((p) => p.at < at).map((p) => p.name), point[2].replace(/\s+#\s*(skip|todo)\b.*$/i, '').trim()].join(' > ');
+      if (!/(^|\s)#\s*(skip|todo)\b/i.test(point[2])) note(name, Boolean(point[1]));
+    }
+  }
+  close(-1);
+  r.incomplete = bail ? 'incomplete: bail out' : bad;
+  return points ? r : null;
+}
+
+// The files `results:` names, read from `tree` (the working tree, or the
+// commit's under --at), each classed by its revision line against `head` and
+// `base`: head, base, older or unknown.
+function results(top, tree, at, head, base) {
+  const out = [];
+  for (const p of configured(top, tree, at, 'results')) {
+    const files = tree.read(p) !== null ? [p] : tree.walk(p);
+    if (!files.length) out.push({ path: p, missing: true });
+    for (const path of files) {
+      const text = tree.read(path).toString('utf8');
+      const sha = text.match(REVISION)?.[1].toLowerCase();
+      const rev = sha && git(top, ['rev-parse', '--verify', '--quiet', `${sha}^{commit}`], { allowFail: true });
+      const cls = !rev ? 'unknown' : rev === head ? 'head' : rev === base ? 'base'
+        : git(top, ['merge-base', '--is-ancestor', rev, head], { allowFail: true }) !== null ? 'older' : 'unknown';
+      out.push({ path, rev: rev || sha, cls, tests: parse(text) });
+    }
+  }
+  return out;
+}
+
+// What check and the views read of branch `b` (readBranch's), once: the changed tests and
+// their assertions, the tests linked to each changed code file, the sections
+// near the changed code, and the named results.
+export function testFacts(top, b) {
+  if (b.testFacts) return b.testFacts;
+  // [HNT-3]: a repo with no commits yet has no tests or results to read against.
+  if (!b.at && !hasCommits(top)) return (b.testFacts = { head: null, tests: [], names: new Map(), observed: [], links: new Map(), near: new Map(), results: [] });
+  const head = git(top, ['rev-parse', b.at ?? 'HEAD']);
+  const isTest = testMatcher(b.root, configured(top, b.tree, b.at, 'tests'));
+  // --diff reads the working tree's results unless --at names a commit ([VW-8]).
+  const r = b.readResults ?? b;
+  const found = results(top, r.tree, r.at, head, b.base);
+  const adr = adrFolders(top, b.tree, b.at);
+  const text = (p) => b.tree.read(p)?.toString('utf8') ?? null;
+  const tracked = new Set(paths(b.at ? git(top, ['ls-tree', '-r', '-z', '--name-only', head]) : git(top, ['ls-files', '-z'], { worktree: true })));
+  const tests = [...new Set([...tracked, ...b.changed])].filter((p) => isTest(p) && text(p) !== null);
+  const names = new Map(tests.map((t) => [t, new Set(idsOn(text(t)))]));
+  const observed = b.changed.filter(isTest).map((p) => observe(p, b.base && git(top, ['show', `${b.base}:${p}`], { allowFail: true }), text(p)));
+  const isAdr = (p) => adr.some((d) => p.startsWith(`${d}/`));
+  const code = b.changed.filter((p) => !isTest(p) && !p.startsWith('requests/') && !p.startsWith(`${b.root}/`)
+    && !isAdr(p) && !found.some((r) => r.path === p));
+  const ctx = { root: b.root, requests: b.requests, seen: b.seen, shallow: isShallow(top), headings: new Map() };
+  // The tests linked to changed file `c`, each with its reason and, as `how`,
+  // the reason without its commit, by which the note groups them.
+  const linked = (c) => {
+    // check reads the working tree; an untracked file has no history, only its own [ID] lines.
+    const lines = (text(c) ?? '').split('\n');
+    const reached = tracked.has(c) ? fileLinks(top, b.base, b.at ? head : null, c, ctx).ids
+      : new Map(idNear(lines, 1, lines.map((_, i) => i + 1)).map((x) => [x.id, `${c}  ${x.reason}`]));
+    const together = tracked.has(c) ? changedWith(top, head, c).commits : [];
+    const ls = tests.flatMap((t) => {
+      const id = [...reached.keys()].find((i) => names.get(t).has(i));
+      const commit = together.find((x) => x.files.includes(t));
+      const [reason, how] = id ? [`names [${id}], near ${c}`, `naming [${id}] (${reached.get(id).replace(/ {2,}/, ': ')})`]
+        : commit ? [`changed together with ${c} (${commit.sha.slice(0, 7)})`, `changed together with ${c}`]
+          : stem(t) === stem(c) ? [`the same stem as ${c}`, `with the same stem as ${c}`] : [];
+      return reason ? [{ test: t, reason, how }] : [];
+    });
+    return { reached: [...reached.keys()], ls };
+  };
+  const links = new Map();
+  const near = new Map();
+  for (const c of b.base ? code : []) {
+    const { reached, ls } = linked(c);
+    near.set(c, reached);
+    links.set(c, ls);
+  }
+  // A changed ADR counts as a linked change for a test's assertions, as code does.
+  const adrLinks = new Map((b.base ? b.changed.filter((p) => isAdr(p) && !isTest(p)) : []).map((p) => [p, linked(p).ls]));
+  b.testFacts = { head, tests, names, observed, links, adrLinks, near, results: found };
+  return b.testFacts;
+}
+
+// Linked tests grouped by how they are linked, with `cap` paths named in all
+// and the rest counted ([HNT-1]): `8 naming [X] (<how c reaches X>): a, b, c; and 5 more; …`.
+function grouped(ls, cap) {
+  const groups = new Map();
+  for (const l of ls) groups.set(l.how, [...(groups.get(l.how) ?? []), l.test]);
+  let left = cap;
+  const out = [...groups].map(([how, ts]) => {
+    const named = ts.slice(0, left);
+    left -= named.length;
+    return `${ts.length} ${how}${named.length ? `: ${named.join(', ')}` : ''}`;
+  });
+  return `${out.join('; ')}${ls.length > cap ? `; and ${ls.length - cap} more; al check --all lists them` : ''}`;
+}
+
+// [HNT-2]'s test notes for branch `b`.
+export function testHints(top, b) {
+  const f = testFacts(top, b);
+  const out = [];
+  const add = (rank, text, command) => out.push({ kind: 'note', rank, owners: [], text, command });
+  const diff = `al context --diff ${b.range}`;
+  for (const [c, ls] of f.links) {
+    if (ls.length && !ls.some((l) => b.changed.includes(l.test))) add(15, `${c} changed, but its linked tests did not: ${grouped(ls, b.all ? ls.length : 3)}`, diff);
+  }
+  const nearIds = new Set([...f.near.values()].flat());
+  for (const o of f.observed.filter(moved)) {
+    const code = [...f.links].filter(([, ls]) => ls.some((l) => l.test === o.path)).map(([c]) => c);
+    const adrs = [...f.adrLinks].filter(([, ls]) => ls.some((l) => l.test === o.path));
+    const named = [...(f.names.get(o.path) ?? [])];
+    if (!code.length && !adrs.length && !named.some((id) => b.changedIds.includes(id))) add(16, `${o.path} changed its assertions (${o.was} → ${o.now}) with no linked code or spec change`, diff);
+    const ids = [...new Set([...named.filter((id) => nearIds.has(id)), ...code.flatMap((c) => f.near.get(c))])];
+    if (/^0\b/.test(b.tier ?? '') && ids.length) add(21, `the claim is tier 0, but ${o.path}, a test of ${ids.map((i) => `[${i}]`).join(', ')} near the changed code, changed its assertions`, `${diff} --for review`);
+  }
+  const head = f.results.some((r) => r.cls === 'head');
+  for (const r of f.results.filter((x) => !x.missing && (['older', 'unknown'].includes(x.cls) || (x.cls === 'base' && !head)))) {
+    add(20, `${r.path} is at ${r.rev ? `${r.cls} revision ${r.rev.slice(0, 7)}` : 'an unknown revision'}: not evidence for this change`,
+      `run the tests at ${f.head.slice(0, 7)} with a "revision: <sha>" line in ${r.path}, then al check`);
+  }
+  return out;
+}
+
+// The Tests lines: each changed test's assertions; with `linked`, each test
+// linked to the changed code or naming a changed section, with its reason.
+export function testLines(top, b, linked) {
+  const f = testFacts(top, b);
+  const out = f.observed.map(observation);
+  if (linked) {
+    for (const [c, ls] of f.links) out.push(...ls.map((l) => `${l.test}  linked to ${c}: ${l.reason}`));
+    for (const t of f.tests) out.push(...[...f.names.get(t)].filter((id) => b.changedIds.includes(id)).map((id) => `${t}  names [${id}], changed on this branch`));
+  }
+  return (out.length ? out : ['no test file changed']).map((t, i) => line(i ? '' : 'Tests', t));
+}
+
+// The Results lines: each named file with its revision, class and counts;
+// then, with both a head and a base result, the failures compared by name.
+export function resultLines(top, b) {
+  const rs = testFacts(top, b).results;
+  const out = rs.map((r) => (r.missing ? `${r.path}  not found` : !r.tests ? `${r.path}  not a format al reads (TAP, JUnit XML): skipped`
+    : `${r.path}  at ${r.rev ? r.rev.slice(0, 7) : 'no revision'} (${r.cls}${['older', 'unknown'].includes(r.cls) ? ', not evidence for this change' : ''}): ${r.tests.passed} passed, ${r.tests.failed} failed${r.tests.incomplete ? `; ${r.tests.incomplete}` : ''}`));
+  const names = (cls, kind) => (rs.some((r) => r.cls === cls && r.tests) ? new Set(rs.filter((r) => r.cls === cls && r.tests).flatMap((r) => [...r.tests[kind]])) : null);
+  const [h, was, passing] = [names('head', 'failing'), names('base', 'failing'), names('head', 'passing')];
+  const list = (xs) => xs.join(', ') || 'none';
+  if (h && was) out.push(`new failures: ${list([...h].filter((n) => !was.has(n)))} · already failing at base: ${list([...h].filter((n) => was.has(n)))} · fixed: ${list([...was].filter((n) => passing.has(n) && !h.has(n)))}`);
+  return (out.length ? out : ['no result files named (add results: <path> to .assuredloop)']).map((t, i) => line(i ? '' : 'Results', t));
+}
+
+// Not known, when a head result is shown.
+export const headNote = (top, b) => (testFacts(top, b).results.some((r) => r.cls === 'head') ? ['whether the head results covered uncommitted changes'] : []);
+export const assertionsChanged = (top, b) => testFacts(top, b).observed.filter(moved).map((o) => o.path);
+export const nearIds = (top, b) => [...new Set([...testFacts(top, b).near.values()].flat())];

package/src/tree.js

@@ -0,0 +1,113 @@
+// Every read goes through a tree: the working tree, or a commit's tree read
+// through git ([VW-8]). Paths are relative to the repo's top level.
+import { lstatSync, readFileSync, readdirSync } from 'node:fs';
+import { join, relative } from 'node:path';
+import { git, resolveCommit } from './git.js';
+
+export function openTree(top, at) {
+  if (!at) {
+    return {
+      label: 'working tree',
+      // No read goes through a symlink anywhere on its path, and a symlink is
+      // never listed, so no read leaves the repo through one ([SPC-1]).
+      read(path) {
+        if (!noSymlinkOn(top, path)) return null;
+        try { return readFileSync(join(top, path)); } catch { return null; }
+      },
+      list(dir) {
+        if (!noSymlinkOn(top, dir)) return null;
+        try { return readdirSync(join(top, dir), { withFileTypes: true }).filter((d) => !d.isSymbolicLink()).map((d) => d.name).sort(); } catch { return null; }
+      },
+      // Every file under `dir`, as repo-relative paths in path order, going
+      // down real folders only.
+      walk(dir) {
+        const out = [];
+        if (!noSymlinkOn(top, dir)) return out;
+        const down = (d) => {
+          let entries;
+          try { entries = readdirSync(join(top, d), { withFileTypes: true }); } catch { return; }
+          for (const e of entries) {
+            if (e.isDirectory()) down(`${d}/${e.name}`);
+            else if (e.isFile()) out.push(relative(top, join(top, d, e.name)));
+          }
+        };
+        down(dir);
+        return out.sort();
+      },
+    };
+  }
+  const sha = resolveCommit(top, at);
+  return {
+    label: `commit ${sha.slice(0, 7)}`,
+    read(path) {
+      return git(top, ['cat-file', 'blob', `${sha}:${path}`], { text: false, allowFail: true });
+    },
+    list(dir) {
+      // -z: names as stored, never quoted (a non-ASCII or quoted name included).
+      const out = git(top, ['ls-tree', '-z', '--name-only', sha, '--', `${dir}/`], { allowFail: true });
+      if (!out) return null;
+      return out.split('\0').filter(Boolean).map((p) => p.slice(dir.length + 1)).sort();
+    },
+    walk(dir) {
+      const out = git(top, ['ls-tree', '-z', '-r', '--name-only', sha, '--', `${dir}/`], { allowFail: true });
+      return out ? out.split('\0').filter(Boolean).sort() : [];
+    },
+    // Whether any part of `path` is a symlink (mode 120000) in this commit.
+    linkOn(path) {
+      const parts = path.split('/');
+      return parts.some((_, i) => git(top, ['ls-tree', sha, '--', parts.slice(0, i + 1).join('/')], { allowFail: true })?.startsWith('120000'));
+    },
+  };
+}
+
+// A request's name, its permanent ID ([REC-1]): lowercase letters, digits and
+// hyphens. It is never a path, so no alias reaches a request by another way.
+export const NAME = /^[a-z0-9][a-z0-9-]*$/;
+export const isName = (name) => typeof name === 'string' && NAME.test(name) && name !== 'archive';
+
+// Every request folder in `tree` whose name is a request name, as { name,
+// dir, open }; a name both open and archived is read open, as findRequest
+// reads it. A symlink is never a request folder (list leaves it out).
+export function requestDirs(tree) {
+  const dirs = [];
+  for (const [at, open] of [['requests', true], ['requests/archive', false]]) {
+    for (const name of (tree.list(at) ?? []).filter(isName)) {
+      if (!dirs.some((d) => d.name === name) && tree.read(`${at}/${name}/request.md`) !== null) dirs.push({ name, dir: `${at}/${name}`, open });
+    }
+  }
+  return dirs;
+}
+
+// The folder under requests/ (or requests/archive/) that a path is in, whatever its name.
+export const folderName = (path) => path.match(/^requests\/archive\/([^/]+)\//)?.[1] ?? path.match(/^requests\/(?!archive\/)([^/]+)\//)?.[1];
+
+// The request a path under requests/ belongs to, when its folder is a request name.
+export const folderOf = (path) => {
+  const name = folderName(path);
+  return isName(name) ? name : undefined;
+};
+
+// Where a request lives in this tree: requests/<name> or requests/archive/<name>.
+export function findRequest(tree, name) {
+  if (!isName(name)) return null;
+  for (const at of ['requests', 'requests/archive']) {
+    if ((tree.list(at) ?? []).includes(name) && tree.read(`${at}/${name}/request.md`) !== null) return `${at}/${name}`;
+  }
+  return null;
+}
+
+// Whether the tool may write `path` (repo-relative): it stays inside the repo
+// (no `..`), and no part of it below the repo top, from the first
+// folder down to the file, is a symlink, wherever the link points. Parts that
+// do not exist yet are created as real folders.
+export function noSymlinkOn(top, path) {
+  if (path.split('/').includes('..')) return false;
+  let at = top;
+  for (const part of path.split('/')) {
+    at = join(at, part);
+    try {
+      if (lstatSync(at).isSymbolicLink()) return false;
+    } catch { return true; }
+  }
+  return true;
+}