@portll/cobolwork 0.2.140 → 0.3.0

package/lib/dataflow.mjs

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { inScope, isProgram, isBms, readSource, relPath } from './sources.mjs';
+import { inScope, isProgram, isBms, relPath } from './sources.mjs';
 import { parseBms, symbolicNames } from './bms.mjs';
 import { dirname, join } from 'node:path';
 import { parseSource, buildFileIndex } from './parser.mjs';
@@ -15,7 +15,7 @@ import { eachWithinMemory, watchMemoryBuffer } from './kernel/memory.mjs';
 // A flow finding is located by source and sink rather than by one path and line, so it keeps its
 // own comparator. The text comparison underneath it is the shared one.
 import { byText } from './kernel/findings.mjs';
-import { buildControl, creditOf, hasFact, within } from './control.mjs';
+import { buildControl, creditOf, hasFact, topOf, within, wholeTop } from './control.mjs';
 import { directoryTree } from './kernel/source-tree.mjs';
 
 const OS_COMMAND_ROUTINE = /^(SYSTEM|C\$SYSTEM|CBL_EXEC_RUN_UNIT|CBL_GC_HOSTED|BXPSYSTM)$/i;
@@ -197,6 +197,7 @@ export function analyze(root, opts = {}) {
   const nodes = [];
   const programs = [];
   const jclPending = [];
+  const jclTrees = new Map();
   // CICS definitions, from CSD extracts and from DFHCSDUP job input, and the estate's own statement
   // of which DDs and queues reach the internal reader.
   const csd = { tdqueues: new Map(), transactions: new Map(), urimaps: new Map() };
@@ -209,7 +210,7 @@ export function analyze(root, opts = {}) {
   const jobSteps = [];
   // The maps of the repository being read, by MAPSET/MAP and by MAP alone.
   const bmsMaps = new Map();
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const relOf = new Map();
   const rel = (p) => { let r = relOf.get(p); if (r === undefined) { r = relPath(root, p); relOf.set(p, r); } return r; };
   // `at` is the statement an edge that is not a statement's own is read at: the CALL or LINK that
@@ -405,6 +406,13 @@ export function analyze(root, opts = {}) {
     // A whole number no sign is written on, which no value put in it can leave below 0.
     const unsignedWhole = (it) => it.level !== 88 && !it.index && !(it.children || []).length
       && /^9+$/.test(String(it.picture || '').toUpperCase().replace(/(\w)\((\d+)\)/g, (_, ch, n) => ch.repeat(Number(n))));
+    // Where a length's reference starts: the start's node, a constant it is moved by, and the span
+    // facts that keep start and length together inside the item.
+    const startOf = (from, len, size) => {
+      const s = from.tok ? itemOfToken(from.tok) : null;
+      const spans = ctl && ctl.spans && s && len ? ctl.spans.filter((f) => ((f.a === s && f.b === len) || (f.a === len && f.b === s)) && f.k + from.offset <= size + 1).map((f) => f.t) : [];
+      return { node: from.tok ? nodeOfToken(from.tok) : null, offset: from.offset, size, spans };
+    };
     const indexed = new Set();
     const subscripting = new Map();
     // Each subscript use of a name, with the other indices of the same reference: the loop-bound
@@ -437,8 +445,9 @@ export function analyze(root, opts = {}) {
         const use = `${key}|${point ?? `s${st.at}`}|${offset}`;
         if (indexed.has(use)) continue;
         indexed.add(use);
+        const start = kind === 'reference-modification' && x.from && base != null ? startOf(x.from, idx, base) : null;
         at.sinks.push({
-          kind, onlyFrom: FROM_OUTSIDE, file: st.file, line: st.line, point, group: `${pk}|${key}`, limit, ...(offset ? { offset } : {}), ...(offset > 0 && idx && unsignedWhole(idx) ? { nonNeg: true } : {}), ...(ssrange ? { ssrange } : {}),
+          kind, onlyFrom: FROM_OUTSIDE, file: st.file, line: st.line, point, group: `${pk}|${key}`, limit, ...(start ? { start } : {}), ...(offset ? { offset } : {}), ...(offset > 0 && idx && unsignedWhole(idx) ? { nonNeg: true } : {}), ...(ssrange ? { ssrange } : {}),
           detail: kind === 'subscript'
             ? `${at.name} subscripts ${host.name}, a table of ${table.occurs}`
             : `${at.name} sets the ${x.kind === 'refmod-offset' ? 'start' : 'length'} of a reference to ${host.name}, which is ${host.size} bytes`,
@@ -959,7 +968,7 @@ export function analyze(root, opts = {}) {
     const tree = (!repo && opts.tree) ? opts.tree : directoryTree(base, opts);
     const idx = tree.index;
     const files = tree.list().filter(isProgram).filter(inScope(opts));
-    for (const f of tree.list().filter(isJcl).filter(inScope(opts))) jclPending.push(f);
+    for (const f of tree.list().filter(isJcl).filter(inScope(opts))) { jclPending.push(f); jclTrees.set(f, tree); }
     for (const f of tree.list().filter((p) => /\.csd$/i.test(p)).filter(inScope(opts))) {
       try { addCsd(tree.text(f).text, f); } catch (e) { stats.unreadable++; unread.push(`${tree.rel(f)}: ${e.code || e.name}`); }
     }
@@ -1088,7 +1097,7 @@ export function analyze(root, opts = {}) {
   // ACCEPT FROM COMMAND-LINE is untrusted.
   for (const f of jclPending) {
     let src;
-    try { src = readSource(f).text; } catch (e) { stats.unreadable++; unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
+    try { src = jclTrees.get(f).text(f).text; } catch (e) { stats.unreadable++; unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
     let job;
     try { job = parseJcl(src, f); } catch (e) { stats.threw++; unparsed.push(`${rel(f)}: ${e.code || e.name}`); continue; }
     stats.jclFiles++;
@@ -1282,6 +1291,18 @@ export function analyze(root, opts = {}) {
     const bits = p.facts.get(point);
     return bits ? creditOf(n.checks, bits, kind, limit, offset, nonNeg) : NONE;
   }
+  // A length is judged with its start: the last byte, S + L - 1, stays inside the item where the
+  // start at its highest leaves room for the length, or where a fact bounds the two together.
+  function spanned(c, sink, n) {
+    if (c.level < 2 || !sink.start) return c;
+    const p = programs[n.pk];
+    const bits = p && p.ordered && sink.point != null ? p.facts.get(sink.point) : null;
+    const s = sink.start;
+    if (bits && s.spans.some((t) => hasFact(bits, t))) return c;
+    const top = bits && s.node ? topOf(s.node.checks, bits) : null;
+    const len = c.cons ? wholeTop(c.cons) : null;
+    return top != null && len != null && top + s.offset + len - 1 <= s.size ? c : { level: 1, check: c.check };
+  }
   // A loop bound cannot push its counter out of the table when the loop's own condition or the
   // body's checks keep every subscript the counter makes, and each index beside it, in range.
   // That holds whatever the bound is, so it does not depend on the route the input took.
@@ -1311,7 +1332,7 @@ export function analyze(root, opts = {}) {
     let point = sink.point;
     for (let h = hops.length - 1; h >= 0; h--) {
       const n = hops[h].node;
-      let c = creditAt(n, point, sink.kind, sink.limit, sink.offset, sink.nonNeg);
+      let c = spanned(creditAt(n, point, sink.kind, sink.limit, sink.offset, sink.nonNeg), sink, n);
       // A count checked against a number above the table's maximum has been checked, not kept in range.
       if (c.level === 2 && sink.top != null && !within(c.cons, sink.top, false)) c = { level: 1, check: c.check };
       if (c.level > best.level) { best = c; by = n; }
@@ -1333,7 +1354,10 @@ export function analyze(root, opts = {}) {
     const at = new Map();
     for (const c of want) { if (!at.has(c.node.id)) at.set(c.node.id, []); at.get(c.node.id).push(c); }
     // Along the way no sink's limit applies, so a bound that holds only against one blocks nothing.
-    const holds = (since, point, k, limit = null, offset = 0, nonNeg = false) => since.some((n) => creditAt(n, point, k, limit, offset, nonNeg).level >= level);
+    const holds = (since, point, k, limit = null, offset = 0, nonNeg = false, sink = null) => since.some((n) => {
+      const c = creditAt(n, point, k, limit, offset, nonNeg);
+      return (sink ? spanned(c, sink, n) : c).level >= level;
+    });
     const lost = new Set();
     const dirty = new Map();
     const parts = new Map();
@@ -1342,7 +1366,7 @@ export function analyze(root, opts = {}) {
     const began = edgesWalked;
     for (let i = 0; i < open.length; i++) {
       const st = open[i];
-      for (const c of at.get(st.node.id) || []) if (!holds(st.since, c.sink.point, c.sink.kind, c.sink.limit, c.sink.offset, c.sink.nonNeg)) lost.add(c);
+      for (const c of at.get(st.node.id) || []) if (!holds(st.since, c.sink.point, c.sink.kind, c.sink.limit, c.sink.offset, c.sink.nonNeg, c.sink)) lost.add(c);
       for (const e of st.node.edgesOut) {
         edgesWalked++;
         const to = e.to;

package/lib/diff.mjs

@@ -1,16 +1,15 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
-import { spawnSync } from 'node:child_process';
 import { tmpdir } from 'node:os';
-import { dirname, join, resolve, sep } from 'node:path';
-import { parseSource } from './parser.mjs';
-import { inScope, isProgram, readSource, relPath } from './sources.mjs';
+import { dirname, join, resolve } from 'node:path';
+import { git, revisionBlobs, treeOf, treePathParts } from './kernel/git.mjs';
+import { inScope, isProgram } from './sources.mjs';
 import { scanAll } from './scan.mjs';
 import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 // A diff finding is located by program rather than by line, so it orders on its own key. The text
 // comparison underneath it is the shared one.
 import { byText, evidenceMap } from './kernel/findings.mjs';
-import { directoryTree } from './kernel/source-tree.mjs';
+import { directoryTree, gitTree } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
 import { loadSite, SITE_FILE } from './site.mjs';
 import { loadBaseline, BASELINE_FILE, SUPPRESSING } from './baseline.mjs';
@@ -30,56 +29,18 @@ export const DIFF_RULES = {
 
 const MAX_LISTED = 8;
 
-// A reviewed repository's .git/config can name commands; with fsmonitor off and plumbing only, git runs none.
-const GIT_ENV = { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0' };
-const git = (repo, args, opts = {}) => spawnSync('git', ['-c', 'core.fsmonitor=false', '-C', repo, ...args], { env: GIT_ENV, maxBuffer: 256 * 1024 * 1024, ...opts });
-const refused = (what, ref, why) => Object.assign(new Error(`${what} ${ref} failed: ${why}`), { code: 'EDIFFREF' });
-
-function treeOf(repo, ref) {
-  if (!ref || String(ref).startsWith('-')) throw refused('git rev-parse', ref, 'a revision cannot be empty or start with "-"');
-  const r = git(repo, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{tree}`], { encoding: 'utf8' });
-  const oid = String(r.stdout || '').trim();
-  if (r.status !== 0 || !/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(oid)) throw refused('git rev-parse', ref, String(r.stderr || '').trim() || 'not a revision');
-  return oid;
-}
-
-// A tree path as a file under `dir`, or null if a part is empty, climbs, names a drive or stream, or is .git.
-function placeIn(dir, path) {
-  const parts = path.split('/');
-  if (parts.some((p) => !p || p === '.' || p === '..' || /[\\:\0]/.test(p) || p.toLowerCase() === '.git')) return null;
-  const to = resolve(dir, ...parts);
-  return to.startsWith(dir + sep) ? to : null;
-}
-
-// The revision as committed: blobs as stored, with no filter or line-ending conversion; links and submodules left out.
+// The revision as committed, written into a temporary directory, for a caller that hands the tree
+// to a program outside this process: build's compiler and ironwork read files, not blobs.
 function materialise(repo, ref) {
-  const tree = treeOf(repo, ref);
-  const listed = git(repo, ['ls-tree', '-r', '-z', '--full-tree', tree]);
-  if (listed.status !== 0) throw refused('git ls-tree', ref, String(listed.stderr || '').trim() || `exit ${listed.status}`);
-  const blobs = [];
-  for (const entry of listed.stdout.toString('utf8').split('\0')) {
-    const m = /^(\d{6}) blob ([0-9a-f]+)\t(.+)$/s.exec(entry);
-    if (m && m[1] !== '120000') blobs.push({ oid: m[2], path: m[3] });
-  }
+  const { blobs } = revisionBlobs(repo, treeOf(repo, ref), ref);
   const dir = realpathSync(mkdtempSync(join(tmpdir(), 'cobolwork-diff-')));
   try {
-    for (let i = 0; i < blobs.length; i += 500) {
-      const batch = blobs.slice(i, i + 500);
-      const r = git(repo, ['cat-file', '--batch'], { input: batch.map((b) => b.oid).join('\n') + '\n' });
-      if (r.status !== 0) throw refused('git cat-file', ref, String(r.stderr || '').trim() || `exit ${r.status}`);
-      let at = 0;
-      for (const b of batch) {
-        const nl = r.stdout.indexOf(0x0a, at);
-        const head = r.stdout.toString('utf8', at, nl).split(' ');
-        if (head[1] !== 'blob') throw refused('git cat-file', ref, `${b.oid} is ${head[1] || 'missing'}`);
-        const size = Number(head[2]);
-        const to = placeIn(dir, b.path);
-        if (to) {
-          mkdirSync(dirname(to), { recursive: true });
-          writeFileSync(to, r.stdout.subarray(nl + 1, nl + 1 + size));
-        }
-        at = nl + 1 + size + 1;
-      }
+    for (const b of blobs) {
+      const parts = treePathParts(b.path);
+      if (!parts) continue;
+      const to = resolve(dir, ...parts);
+      mkdirSync(dirname(to), { recursive: true });
+      writeFileSync(to, b.bytes);
     }
   } catch (e) {
     rmSync(dir, { recursive: true, force: true });
@@ -102,9 +63,9 @@ const qualified = (it) => { const names = []; for (let x = it; x; x = x.parent)
 
 // A program that cannot be read or parsed is named in `unread`, never dropped: dropped, it would
 // read as removed from the tree, and every change inside it would go unreported.
-function facts(root, opts = {}) {
-  // Each side of a diff is its own tree: the base is a revision materialised into a temporary
-  // directory, the head is the working tree, and they are walked separately.
+function facts(side, opts = {}) {
+  // Each side of a diff is its own tree: a revision read out of git, or the working tree, and they
+  // are walked separately.
   //
   // This used to build its own parse options with systemDirs: [], where every other caller passed
   // the caller's. Nobody decided that - it is what five copies of one object literal do. The
@@ -112,24 +73,24 @@ function facts(root, opts = {}) {
   // change inside one was invisible to the command whose whole job is what a change reaches. Both
   // sides now take the same configuration, which for a caller that passes no systemDirs is
   // exactly the old behaviour.
-  const tree = directoryTree(root, opts);
-  const idx = tree.index;
+  const tree = side.tree || directoryTree(side.root, opts);
+  const rel = (p) => tree.rel(p);
   const out = new Map();
   out.unread = [];
   for (const f of tree.list().filter(isProgram).filter(inScope(opts))) {
     let src;
-    try { src = readSource(f).text; } catch (e) { out.unread.push(`${relPath(root, f)}: ${e.code || e.name}`); continue; }
+    try { src = tree.text(f).text; } catch (e) { out.unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
     if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) continue;
     let r;
     try {
       r = tree.parse(f, src);
-    } catch (e) { out.unread.push(`${relPath(root, f)}: ${e.code || e.name}`); continue; }
-    const file = relPath(root, f);
+    } catch (e) { out.unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
+    const file = rel(f);
     for (const p of r.programs) {
       const items = new Map();
       for (const it of p.items) {
         if (!it.name || it.name === 'FILLER' || it.level === 88 || it.level === 66 || it.level === 78) continue;
-        items.set(qualified(it), { size: it.size, offset: it.offset, from: it.file ? relPath(root, it.file) : file, section: it.section, level: it.level });
+        items.set(qualified(it), { size: it.size, offset: it.offset, from: it.file ? rel(it.file) : file, section: it.section, level: it.level });
       }
       const calls = new Set();
       const dynamic = new Set();
@@ -149,7 +110,7 @@ function facts(root, opts = {}) {
         if (arg) commareas.add(arg.u);
       }
       const using = new Set(p.calls.flatMap(c => c.using.filter(a => a.word).map(a => a.word)));
-      const copied = new Set(r.copies.filter((c) => c.status === 'resolved' && c.path).map((c) => relPath(root, c.path)));
+      const copied = new Set(r.copies.filter((c) => c.status === 'resolved' && c.path).map((c) => rel(c.path)));
       out.set(`${file}#${p.id}`, { id: p.id, file, text: src, items, calls, dynamic, commareas, using, copied });
     }
   }
@@ -168,12 +129,17 @@ function countByKey(findings) {
   return m;
 }
 
-export function diffTrees(baseRoot, headRoot, opts = {}) {
-  // The allow list describes the working tree, so it never applies to a revision checked out into
-  // a temporary directory: applied there it would match nothing and the base would read as empty.
+// Each side is a directory, or a source tree: a git revision read without writing it out.
+const sideOf = (x) => (typeof x === 'string' ? { root: x, tree: null } : { root: x.root, tree: x });
+
+export function diffTrees(baseSide, headSide, opts = {}) {
+  const baseAt = sideOf(baseSide);
+  const headAt = sideOf(headSide);
+  // The allow list describes the working tree, so it never applies to a revision: applied there it
+  // would match nothing and the base would read as empty.
   const baseOpts = { ...opts, allow: null };
-  const base = facts(baseRoot, baseOpts);
-  const head = facts(headRoot, opts);
+  const base = facts(baseAt, baseOpts);
+  const head = facts(headAt, opts);
   const findings = [];
   const reach = { programsCompared: 0, programsAdded: 0, programsRemoved: 0, programsWithLayoutChange: 0, reachedOnlyThroughCopybooks: 0, copybooksImplicated: new Set() };
 
@@ -214,8 +180,9 @@ export function diffTrees(baseRoot, headRoot, opts = {}) {
 
   const listing = { listSinks: opts.listSinks === true, listSources: opts.listSources === true };
   const shared = { only: opts.only, fullTrace: opts.fullTrace, systemDirs: opts.systemDirs || [], ...listing };
-  const before = scanAll(baseRoot, shared);
-  const after = scanAll(headRoot, { ...shared, allow: opts.allow });
+  const withTree = (at) => (at.tree ? { tree: at.tree } : {});
+  const before = scanAll(baseAt.root, { ...shared, ...withTree(baseAt) });
+  const after = scanAll(headAt.root, { ...shared, ...withTree(headAt), allow: opts.allow });
   // A file neither tree could read cannot be compared. Its findings are neither introduced nor
   // resolved, and calling them resolved would report a scan failure as a fix.
   const unreadable = new Set([...base.unread, ...head.unread].map(u => u.split(': ')[0]));
@@ -237,7 +204,7 @@ export function diffTrees(baseRoot, headRoot, opts = {}) {
     return n > (afterCounts.get(k) || 0);
   });
   const uncomparable = [...new Set([...before.findings, ...after.findings].filter(f => !comparable(f)).map(f => f.path))];
-  const configurationChanged = configurationChanges(baseRoot, headRoot);
+  const configurationChanged = configurationChanges(baseSide, headSide);
 
   for (const f of findings) { const m = DIFF_RULES[f.rule]; f.sev = m.sev; f.cwe = m.cwe; f.evidence = m.evidence; }
   findings.sort((a, b) => byText(a.rule, b.rule) || byText(a.path, b.path) || byText(String(a.program), String(b.program)));
@@ -272,13 +239,16 @@ export function diffTrees(baseRoot, headRoot, opts = {}) {
 const SITE_LISTS = ['productionQualifiers', 'productionJobPaths', 'nonProductionJobPaths', 'systemNames', 'vendorPacks',
   'internalReaderDds', 'internalReaderQueues', 'compilerOptions', 'apfLibraries', 'restrictedDatasets', 'surrogateUsers'];
 
-function siteChange(baseRoot, headRoot) {
-  const was = existsSync(join(baseRoot, SITE_FILE));
-  const now = existsSync(join(headRoot, SITE_FILE));
+// Whether a side holds a file at its top: in the source tree it was read from, or on disk.
+const holds = (at, name) => (at.tree && at.tree.kind !== 'directory' ? at.tree.contains(resolve(at.tree.root, name)) : existsSync(join(at.root, name)));
+
+function siteChange(baseAt, headAt) {
+  const was = holds(baseAt, SITE_FILE);
+  const now = holds(headAt, SITE_FILE);
   if (!was && !now) return null;
   if (!now) return { file: SITE_FILE, change: 'removed, so the rules that need it do not run' };
-  const b = loadSite(baseRoot);
-  const h = loadSite(headRoot);
+  const b = loadSite(baseAt.root, null, baseAt.tree);
+  const h = loadSite(headAt.root, null, headAt.tree);
   const parts = was ? [] : ['added'];
   for (const k of SITE_LISTS) {
     const gone = b[k].filter((x) => !h[k].includes(x)).map((x) => `-${x}`);
@@ -291,16 +261,19 @@ function siteChange(baseRoot, headRoot) {
   return parts.length ? { file: SITE_FILE, change: printable(parts.join('; '), 400) } : null;
 }
 
-export const configurationChanges = (baseRoot, headRoot) =>
-  [siteChange(baseRoot, headRoot), baselineChange(baseRoot, headRoot)].filter(Boolean);
+export const configurationChanges = (baseSide, headSide) => {
+  const baseAt = sideOf(baseSide);
+  const headAt = sideOf(headSide);
+  return [siteChange(baseAt, headAt), baselineChange(baseAt, headAt)].filter(Boolean);
+};
 
-function baselineChange(baseRoot, headRoot) {
-  const was = existsSync(join(baseRoot, BASELINE_FILE));
-  const now = existsSync(join(headRoot, BASELINE_FILE));
+function baselineChange(baseAt, headAt) {
+  const was = holds(baseAt, BASELINE_FILE);
+  const now = holds(headAt, BASELINE_FILE);
   if (!was && !now) return null;
-  const held = (root, there) => new Map((there ? loadBaseline(root)?.entries || [] : []).map((e) => [`${e.fingerprint}|${e.rule}`, e]));
-  const b = held(baseRoot, was);
-  const h = held(headRoot, now);
+  const held = (at, there) => new Map((there ? loadBaseline(at.root, { tree: at.tree })?.entries || [] : []).map((e) => [`${e.fingerprint}|${e.rule}`, e]));
+  const b = held(baseAt, was);
+  const h = held(headAt, now);
   const added = [...h.keys()].filter((k) => !b.has(k));
   const suppressing = added.filter((k) => SUPPRESSING.includes(h.get(k).action)).length;
   const removed = [...b.keys()].filter((k) => !h.has(k)).length;
@@ -334,11 +307,14 @@ export function withRefs(repo, baseRef, headRef, opts, fn) {
   }
 }
 
+// Reads each revision out of git without writing it anywhere; the head is the working tree when
+// headRef is null.
 export function diffRefs(repo, baseRef, headRef = null, opts = {}) {
-  return withRefs(repo, baseRef, headRef, opts, (baseDir, headDir, scoped) => {
-    const res = diffTrees(baseDir, headDir, scoped);
-    res.summary.base = baseRef;
-    res.summary.head = headRef || WORKING_TREE;
-    return res;
-  });
+  const trees = { systemDirs: opts.systemDirs || [] };
+  const base = gitTree(repo, baseRef, trees);
+  const head = headRef ? gitTree(repo, headRef, trees) : repo;
+  const res = diffTrees(base, head, headRef ? opts : { ...opts, allow: gitScope(repo) });
+  res.summary.base = baseRef;
+  res.summary.head = headRef || WORKING_TREE;
+  return res;
 }

package/lib/equivalence.mjs

@@ -50,12 +50,18 @@ function closure(root, file, idx, systemDirs) {
   return out;
 }
 
-// Programs the change edits or adds, or whose copybooks it edits: [{ path, base, head, via }] with
-// each side's source digest and the copybooks that changed under an unchanged program.
+// Programs the change edits, adds or deletes, or whose copybooks it edits: [{ path, base, head, via }]
+// with each side's source digest (head null for a deleted program) and the copybooks that changed
+// under an unchanged program.
 export function changedPrograms(baseDir, headDir, allow = null, systemDirs = []) {
   const out = [];
   const headTree = directoryTree(headDir);
-  const baseIdx = directoryTree(baseDir).index;
+  const baseTree = directoryTree(baseDir);
+  const baseIdx = baseTree.index;
+  for (const base of baseTree.list().filter(isProgram).sort()) {
+    const path = relPath(baseDir, base);
+    if (!existsSync(join(headDir, path))) out.push({ path, base: sha256(readFileSync(base)), head: null, deleted: true });
+  }
   for (const head of headTree.list().filter((p) => isProgram(p) && (!allow || allow.has(p))).sort()) {
     const path = relPath(headDir, head);
     const h = sha256(readFileSync(head));
@@ -74,12 +80,17 @@ export function changedPrograms(baseDir, headDir, allow = null, systemDirs = [])
 // Whether a commit in base..head reads as authored by a tool, from its identities and trailers.
 const TOOL = /\[bot\]|(^|[^a-z])bot@|copilot|claude|anthropic|openai|codex|devin|cursor|gemini|aider|sweep-ai|dependabot|renovate/i;
 export function machineAuthored(repo, base, head, env = process.env) {
-  const r = spawnSync('git', ['-C', repo, 'rev-list', '--format=%x1e%an%x00%ae%x00%cn%x00%ce%x00%B', `${base}..${head}`], { encoding: 'utf8', timeout: 30000, maxBuffer: 16 << 20, env: cleanGitEnv(env) });
+  // NUL ends every field and every commit (-z), and a commit message cannot hold one.
+  const r = spawnSync('git', ['-C', repo, 'log', '-z', '--format=%an%x00%ae%x00%cn%x00%ce%x00%B', `${base}..${head}`], { encoding: 'utf8', timeout: 30000, maxBuffer: 16 << 20, env: cleanGitEnv(env) });
   if (r.status !== 0) return { known: false, commits: [] };
+  const fields = r.stdout.split('\x00');
+  if (fields.at(-1) === '') fields.pop();
+  if (fields.length % 5 !== 0) return { known: false, commits: [] };
   const commits = [];
-  for (const chunk of r.stdout.split('\x1e').slice(1)) {
-    const [an, ae, cn, ce, body = ''] = chunk.split('\x00');
-    const trailers = body.split('\n').filter((l) => /^(co-authored-by|generated-by|assisted-by):/i.test(l.trim()));
+  for (let i = 0; i < fields.length; i += 5) {
+    const [an, ae, cn, ce, body = ''] = fields.slice(i, i + 5);
+    // Control characters are dropped first, so one placed before a trailer does not hide it.
+    const trailers = body.split('\n').map((l) => l.replace(/[\x00-\x1f\x7f]/g, '').trim()).filter((l) => /^(co-authored-by|generated-by|assisted-by):/i.test(l));
     const signals = [an, ae, cn, ce, ...trailers, ...(/generated with/i.test(body) ? ['generated with'] : [])].filter((x) => x && TOOL.test(x));
     if (signals.length) commits.push({ signals: [...new Set(signals.map((s) => s.trim().slice(0, 80)))] });
   }
@@ -101,13 +112,20 @@ export function checkEquivalence({ baseDir, headDir, allow, files = [], allowed
       problems.push(`${s.file} names head ${h ? h.slice(0, 12) : 'nothing'} and base ${b ? b.slice(0, 12) : 'nothing'}, which match no program this change edits`);
     }
   }
+  // Every statement that names a program is judged, and the program passes only if each does.
   const programs = changed.map((c) => {
-    const s = statements.find((x) => x.statement && subjectDigest(x.statement, 'head') === c.head && subjectDigest(x.statement, 'base') === c.base);
+    if (c.deleted) return { path: c.path, deleted: true, statement: null, ok: false, because: 'the change deletes this program, and no equivalence statement can show what its callers do now' };
+    const matching = statements.filter((x) => x.statement && subjectDigest(x.statement, 'head') === c.head && subjectDigest(x.statement, 'base') === c.base);
+    const via = c.via ? { via: c.via } : {};
+    if (!matching.length) return { path: c.path, ...via, statement: null, ok: false, because: c.via ? `no equivalence statement names this program, whose copybook${c.via.length > 1 ? 's' : ''} ${c.via.join(', ')} the change edits` : 'no equivalence statement names this change' };
+    const judged = matching.map((s) => judge(c, s));
+    return judged.find((j) => !j.ok) || judged[0];
+  });
+  function judge(c, s) {
     const via = c.via ? { via: c.via } : {};
-    if (!s) return { path: c.path, ...via, statement: null, ok: false, because: c.via ? `no equivalence statement names this program, whose copybook${c.via.length > 1 ? 's' : ''} ${c.via.join(', ')} the change edits` : 'no equivalence statement names this change' };
     const p = s.statement.predicate || {};
     const verdict = p.verdict || null;
-    const covered = p.coverage !== null && p.coverage !== undefined;
+    const covered = !!p.coverage && typeof p.coverage === 'object' && Array.isArray(p.coverage.unreached) && p.coverage.unreached.every((u) => typeof u === 'string');
     const ranOn = new Set(((p.closure && p.closure.head) || []).map((x) => x.sha256));
     const missing = (c.viaDigests || []).filter((d) => !ranOn.has(d));
     const because = !PASSING.has(verdict) ? `the statement's verdict is ${verdict}${Array.isArray(p.inconclusive) && p.inconclusive.length ? ` (${p.inconclusive.join('; ')})` : ''}`
@@ -115,9 +133,9 @@ export function checkEquivalence({ baseDir, headDir, allow, files = [], allowed
       : !covered ? 'the statement measured no coverage of the changed paragraphs, so it is inconclusive'
         : allowed && !s.signed ? `the statement is ${s.reason}`
           : null;
-    const unreached = covered && Array.isArray(p.coverage.unreached) ? p.coverage.unreached : [];
+    const unreached = covered ? p.coverage.unreached : [];
     return { path: c.path, ...via, statement: s.file, verdict, signed: s.signed, signer: s.signer, coverage: covered, ...(unreached.length ? { unreached } : {}), ok: because === null && !unreached.length, ...(because || unreached.length ? { because: because || `the inputs never reached ${unreached.join(', ')}` } : {}) };
-  });
+  }
   const authorship = mode === 'machineAuthored' && repo && base ? machineAuthored(repo, base, head || 'HEAD', env) : null;
   const required = mode === 'always' ? true : mode === 'machineAuthored' ? (authorship && authorship.known ? authorship.commits.length > 0 : null) : false;
   return { mode, required, ...(authorship ? { machineAuthored: authorship } : {}), programs, problems };

package/lib/evidence/record.mjs

@@ -36,12 +36,14 @@ export const KINDS = {
   dd: { fields: { dd: str, event: (x) => ['open', 'close', 'end'].includes(x), mode: str, sha256: hex64, bytes: int }, required: ['dd', 'event'] },
   call: { fields: { program: str, from: str, sha256: hex64 }, required: ['program'] },
   abend: { fields: { code: str, file: str, line: int }, required: ['code'] },
+  step: { fields: { step: str, pgm: str, outcome: str }, required: ['step', 'pgm', 'outcome'] },
+  sink: { fields: { sink: str, file: str, line: int, marker: str, reached: (x) => typeof x === 'boolean' }, required: ['sink', 'file', 'line', 'marker', 'reached'] },
   genesis: { fields: { createdAt: str, rotatedFrom: any }, required: ['createdAt'] },
   run: { fields: { run: str, runChain: hex32, runLength: int, runTip: hex64 }, required: ['run', 'runChain', 'runLength', 'runTip'] },
   'lock-broken': { fields: { holderPid: int, ageMs: int }, required: ['holderPid', 'ageMs'] },
 };
 
-export const JOURNAL_KINDS = new Set(['open', 'input', 'finding', 'suppressed', 'baseline-write', 'witness', 'verdict', 'output', 'close', 'dd', 'call', 'abend']);
+export const JOURNAL_KINDS = new Set(['open', 'input', 'finding', 'suppressed', 'baseline-write', 'witness', 'verdict', 'output', 'close', 'dd', 'call', 'abend', 'step', 'sink']);
 export const LEDGER_KINDS = new Set(['genesis', 'run', 'lock-broken']);
 
 export const newChain = () => randomBytes(16).toString('hex');

package/lib/evidence/slsa.mjs

@@ -20,9 +20,24 @@ function named(path, root) {
   return abs.startsWith(r + sep) ? relative(r, abs).split(sep).join('/') : basename(abs);
 }
 
+// The compiler's arguments with every value replaced: an option keeps its name, a path or other value
+// becomes <value>. The digest of the arguments as run lets a holder of them check the statement.
+function redacted(argv) {
+  return argv.map((a) => {
+    const s = String(a);
+    if (!s.startsWith('-')) return '<value>';
+    const cut = s.search(/[=/\\]/);
+    return cut < 0 ? s : `${s.slice(0, cut)}${s[cut] === '=' ? '=' : ''}<value>`;
+  });
+}
+
 export function slsaStatement({ provenance, docBytes, root, artifacts = [], runId = null, runTip = null, builderId = null, startedOn = null, finishedOn = null }) {
   const subject = [{ name: 'build.json', digest: { sha256: sha256(docBytes) } },
-    ...artifacts.map((p) => ({ name: named(p, root), digest: { sha256: sha256(readFileSync(p)) } }))];
+    ...artifacts.map((p) => {
+      let bytes;
+      try { bytes = readFileSync(p); } catch (e) { throw new Error(`the artifact ${basename(p)} could not be read (${e.code || e.message})`); }
+      return { name: named(p, root), digest: { sha256: sha256(bytes) } };
+    })];
 
   const resolvedDependencies = (provenance.sources || []).map((s) => ({ uri: `file:${s.path}`, name: s.path, digest: { sha256: s.sha256 } }));
   if (provenance.compiler && provenance.compiler.sha256) {
@@ -36,7 +51,7 @@ export function slsaStatement({ provenance, docBytes, root, artifacts = [], runI
     revisions: provenance.revisions,
     policy: { sha256: provenance.policy.sha256, setBy: provenance.policy.setBy },
     ...(provenance.copylibs && provenance.copylibs.length ? { copylibs: provenance.copylibs.map((d) => basename(d)) } : {}),
-    ...(provenance.compiler && provenance.compiler.argv ? { compilerArguments: provenance.compiler.argv } : {}),
+    ...(provenance.compiler && provenance.compiler.argv ? { compilerArguments: redacted(provenance.compiler.argv), compilerArgumentsSha256: sha256(provenance.compiler.argv.map(String).join('\0')) } : {}),
   };
   const internalParameters = {
     tool: 'cobolwork', toolVersion: provenance.toolVersion, flowModel: provenance.flowModel,

package/lib/execution.mjs

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Execution: whether the estate's own runs entered the paragraph a finding is in, from the
+// reports `ironwork run --coverage` writes. A finding in a paragraph a run entered is code the tests
+// exercise; one in a paragraph no run entered is code they never reach. See docs/spec/evidence.md §13.4 (execution coverage).
+import { readFileSync } from 'node:fs';
+import { basename, delimiter, join } from 'node:path';
+
+export function executionFeedPaths(opts = {}) {
+  if (opts.executionFeeds) return opts.executionFeeds;
+  return (process.env.COBOLWORK_EXECUTION || '').split(delimiter).filter(Boolean);
+}
+
+// One report: per PROGRAM-ID, each paragraph's source line, name and how often a run entered it.
+// Shape (ironwork's): { programs: [{ program, detail: [{ name, line, entered }] }] }.
+export function loadExecutionFeed(path) {
+  const feed = { file: basename(path), programs: new Map(), problem: null };
+  let doc;
+  try { doc = JSON.parse(readFileSync(path, 'utf8')); } catch (e) { feed.problem = e.code ? `could not be read (${e.code})` : `is not JSON (${e.message})`; return feed; }
+  if (!doc || !Array.isArray(doc.programs)) { feed.problem = 'holds no programs'; return feed; }
+  for (const p of doc.programs) {
+    const detail = Array.isArray(p?.detail) ? p.detail : null;
+    if (typeof p?.program !== 'string' || !detail || !detail.every((d) => typeof d?.name === 'string' && Number.isInteger(d.line) && Number.isInteger(d.entered) && d.entered >= 0)) {
+      feed.problem = `a program's paragraphs are not { name, line, entered }`;
+      return feed;
+    }
+    feed.programs.set(p.program.toUpperCase(), detail.map((d) => ({ name: d.name, line: d.line, entered: d.entered })));
+  }
+  return feed;
+}
+
+const PROGRAM_ID = /^.{0,6}[ \d]?\s*PROGRAM-ID\.?\s+['"]?([A-Za-z0-9#$@-]+)/;
+
+// The programs a source holds, by PROGRAM-ID and the line it starts on.
+function programsOf(text) {
+  const out = [];
+  text.split(/\r?\n/).forEach((line, i) => {
+    const m = PROGRAM_ID.exec(line);
+    if (m) out.push({ id: m[1].toUpperCase(), line: i + 1 });
+  });
+  return out;
+}
+
+// Each finding inside a paragraph of a covered program gets `executed: { paragraph, entered }`,
+// entered summed over the feeds. A finding before the program's first paragraph, or in a program
+// no feed covers, is left as it is.
+export function applyExecution(findings, root, feeds) {
+  const good = feeds.filter((f) => !f.problem);
+  const byExecution = { entered: 0, 'never-entered': 0 };
+  if (!good.length) return { byExecution: null };
+  const sources = new Map();
+  const programs = (path) => {
+    if (!sources.has(path)) {
+      let text = null;
+      try { text = readFileSync(join(root, path), 'utf8'); } catch { /* not a file the scan can read again */ }
+      sources.set(path, text === null ? [] : programsOf(text));
+    }
+    return sources.get(path);
+  };
+  for (const f of findings) {
+    if (!f.path || !Number.isInteger(f.line)) continue;
+    const owner = programs(f.path).filter((p) => p.line <= f.line && good.some((g) => g.programs.has(p.id))).at(-1);
+    if (!owner) continue;
+    let paragraph = null;
+    let entered = 0;
+    for (const g of good) {
+      const detail = g.programs.get(owner.id);
+      const at = detail?.filter((d) => d.line >= owner.line && d.line <= f.line).at(-1);
+      if (!at) continue;
+      paragraph = at.name;
+      entered += at.entered;
+    }
+    if (paragraph === null) continue;
+    f.executed = { paragraph, entered };
+    byExecution[entered > 0 ? 'entered' : 'never-entered']++;
+  }
+  return { byExecution };
+}

package/lib/explain.mjs

@@ -7,7 +7,7 @@ import { EVIDENCE } from './kernel/findings.mjs';
 import { REGISTRY } from './kernel/registry.mjs';
 import { WHO_ACTS } from './tui/model.mjs';
 import { verificationPlan } from './verify.mjs';
-import { cobolCard, statementCard } from './card.mjs';
+import { cobolCard, statementCard } from './cards.mjs';
 
 const HIDDEN_RULES = new Set(Object.keys(REGISTRY.find((s) => s.name === 'hidden').rules));
 

package/lib/ftp.mjs

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { statementCard } from './card.mjs';
+import { statementCard } from './cards.mjs';
 
 // IBM's batch FTP client takes the host and its options on PARM, or the host as the first line of
 // its input, reads subcommands from INPUT, and logs on from NETRC or from its input. CardDemo's job