@portll/cobolwork 0.0.1 → 0.2.76

package/lib/kernel/identity.mjs

@@ -0,0 +1,216 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// What a finding is, as distinct from where it is printed today.
+//
+// A report that cannot say a finding is the one it reported yesterday cannot say fixed, new or
+// regressed, and a baseline has nothing to accept. The line number is the obvious key and the wrong
+// one: code added above a finding moves it, and commitwork measured a line-keyed identity reporting
+// eight findings fixed that had only shifted. So identity is built from names the source gives
+// itself: the program, then the section and paragraph a statement sits in, or the record a data
+// entry belongs to; the job, step and DD for JCL; and the statement's own text, with the sequence
+// and identification areas removed because renumbering rewrites them.
+//
+// No position enters it anywhere, including as a tiebreak. An ordinal among look-alikes would be a
+// line number under another name, unstable in exactly the cases it exists for. Two findings that
+// agree on rule, scope and text are the same statement flagged twice, and they share a fingerprint;
+// the report counts how many did, so the collapse is visible rather than silent.
+import { createHash } from 'node:crypto';
+import { join } from 'node:path';
+import { detectFormat, normalize, tokenize, VERBS, NOT_LABELS, SCOPE_TERMINATORS } from '../parser.mjs';
+import { parseJcl } from '../jcl.mjs';
+import { isJcl, isProgram, isCopybook, readSource } from '../sources.mjs';
+
+export const FINGERPRINT_VERSION = 'cobolwork/v1';
+
+const LEVEL_RECORD = /^(0?1|77)$/;
+
+// The named scope of every line of a COBOL source: program, then section.paragraph in the
+// procedure division, or the 01 or 77 record elsewhere. Read from tokens, so a comment or a
+// literal cannot pose as a label. A copybook has no divisions, so there a label and a record are
+// both read, whichever the copybook turns out to hold.
+export function cobolScopes(src, file = '') {
+  const norm = normalize(src, detectFormat(src), new Map());
+  const { tokens } = tokenize(norm, file);
+  const at = new Array(norm.entries.length + 1).fill(null);
+  let program = null, division = null, section = null, paragraph = null, record = null;
+  let sentenceStart = true;
+  const inProcedure = () => division === 'PROCEDURE' || (division === null && program === null);
+  const current = () => {
+    const where = division === 'PROCEDURE' || ((section || paragraph) && division === null)
+      ? [section, paragraph].filter(Boolean).join('.')
+      : record || division || '';
+    return program ? `${program}/${where}` : where;
+  };
+  for (let i = 0; i < tokens.length; i++) {
+    const t = tokens[i];
+    const next = tokens[i + 1];
+    if (t.t === 'period') { sentenceStart = true; continue; }
+    if (t.t === 'word') {
+      if (t.u === 'PROGRAM-ID' || t.u === 'FUNCTION-ID') {
+        let j = i + 1;
+        if (tokens[j] && tokens[j].t === 'period') j++;
+        if (tokens[j]) program = String(tokens[j].v).toUpperCase();
+        division = 'IDENTIFICATION'; section = paragraph = record = null;
+      } else if (next && next.t === 'word' && next.u === 'DIVISION') {
+        division = t.u === 'ID' ? 'IDENTIFICATION' : t.u;
+        section = paragraph = record = null;
+      } else if (sentenceStart && inProcedure() && next && next.t === 'word' && next.u === 'SECTION' && !NOT_LABELS.has(t.u)) {
+        section = t.u; paragraph = null;
+      } else if (sentenceStart && inProcedure() && next && next.t === 'period' && !NOT_LABELS.has(t.u) && !VERBS.has(t.u) && !SCOPE_TERMINATORS.has(t.u)) {
+        paragraph = t.u;
+      } else if (sentenceStart && division !== 'PROCEDURE' && (t.u === 'FD' || t.u === 'SD') && next && next.t === 'word') {
+        record = next.u;
+      } else if (sentenceStart && division !== 'PROCEDURE' && LEVEL_RECORD.test(t.v) && next && next.t === 'word') {
+        record = next.u;
+      }
+    }
+    sentenceStart = false;
+    if (at[t.line] == null) at[t.line] = current();
+  }
+  // A line holding no token of its own - a comment, a continuation, a line inside EXEC - is in
+  // whatever scope was in force above it.
+  let last = '';
+  for (let l = 1; l < at.length; l++) { if (at[l] == null) at[l] = last; else last = at[l]; }
+  return at;
+}
+
+// Job, step and DD for every line of a job. In-stream data belongs to the DD that opens it.
+export function jclScopes(src, file = '') {
+  const lines = String(src).split(/\r?\n/).length;
+  const at = new Array(lines + 1).fill('');
+  let job;
+  try { job = parseJcl(src, file); } catch { return at; }
+  const events = [
+    ...job.jobs.map((j) => ({ line: j.line, kind: 'job', name: j.name })),
+    ...job.steps.map((s) => ({ line: s.line, kind: 'step', name: s.name || `(${s.pgm || s.proc || 'step'})` })),
+    ...job.dds.map((d) => ({ line: d.line, kind: 'dd', name: d.name || '(concatenated)' })),
+  ].sort((a, b) => a.line - b.line);
+  let jobName = '', stepName = '', ddName = '';
+  let e = 0;
+  for (let l = 1; l <= lines; l++) {
+    while (e < events.length && events[e].line <= l) {
+      const ev = events[e++];
+      if (ev.kind === 'job') { jobName = ev.name || ''; stepName = ''; ddName = ''; }
+      else if (ev.kind === 'step') { stepName = ev.name; ddName = ''; }
+      else ddName = ev.name;
+    }
+    at[l] = [jobName, stepName, ddName].filter(Boolean).join('/');
+  }
+  return at;
+}
+
+// A fingerprint can be tested against guesses, so its line has credentials masked or is left out.
+const SECRET_LINE = new Set(['jcl-instream-credential', 'cd-signon-password', 'cd-snode-credentials']);
+// Every repetition is bounded or cannot overlap its neighbour, so a crafted line costs linear time.
+const maskSecrets = (s) => {
+  const out = s
+    .replace(/\b(PASSWORD|PASSWRD|PASSPHRASE|PHRASE|PWD|PASS)(\s*[=(]\s*)('[^']{0,256}'|"[^"]{0,256}"|[^\s,)'"]{1,256})/gi, '$1$2*')
+    .replace(/\b(USERID|SNODEID|PNODEID)(\s*=\s*\([^,)]{0,256},)[^)]{0,256}\)/gi, '$1$2*)');
+  const named = out.search(/PASS|PWD|PSWD/i);
+  return named < 0 ? out : out.replace(/\bVALUE(\s+)('[^']{0,256}'|"[^"]{0,256}")/gi,
+    (m, space, literal, at) => (at > named ? `${m.slice(0, m.length - literal.length)}'*'` : m));
+};
+// Strips trailing blanks and periods in linear time; a regular expression for it is quadratic on a long run.
+const trimEnd = (s) => {
+  let end = s.length;
+  while (end > 0 && /[\s.]/.test(s[end - 1])) end--;
+  return s.slice(0, end);
+};
+
+// A line as the compiler or the reader sees it: without the sequence area, without columns 73 to
+// 80, with its spacing collapsed. Case is folded for COBOL and JCL, which do not distinguish it, and
+// a COBOL line loses the period that ends its sentence, which moves when a statement is added after it.
+function codeText(line, kind, format) {
+  // A line past this is not code anyone reads, and every step below is linear in what is left.
+  let s = (line || '').slice(0, 4096);
+  if (kind === 'cobol') {
+    if (format === 'fixed') s = s.slice(7, 72);
+    else if (format === 'variable') s = s.slice(7, 250);
+    else if (format === 'terminal') s = s.slice(1);
+    s = trimEnd(s.replace(/\*>.*$/, '').toUpperCase());
+  } else if (kind === 'jcl') s = s.slice(0, 72).toUpperCase();
+  return maskSecrets(s).replace(/\s+/g, ' ').trim();
+}
+
+// Reads each file once, however many findings it holds.
+function fileFacts(root, path, cache) {
+  let facts = cache.get(path);
+  if (facts) return facts;
+  facts = { kind: 'other', lines: [], scopes: null, format: null };
+  cache.set(path, facts);
+  let src;
+  try { src = readSource(join(root, path)).text; } catch { return facts; }
+  facts.lines = src.split(/\r?\n/);
+  if (isJcl(join(root, path))) {
+    facts.kind = 'jcl';
+    facts.scopes = jclScopes(src, path);
+  } else if (isProgram(join(root, path)) || isCopybook(join(root, path))) {
+    facts.kind = 'cobol';
+    facts.format = detectFormat(src);
+    try { facts.scopes = cobolScopes(src, path); } catch { facts.scopes = null; }
+  }
+  return facts;
+}
+
+// What a finding's fingerprint is made of.
+function partsOf(f, root, cache) {
+  const facts = f.path ? fileFacts(root, f.path, cache) : { kind: 'other', lines: [], scopes: null };
+  const line = Number(f.line) || 0;
+  const scoped = facts.scopes && line > 0 && line < facts.scopes.length ? facts.scopes[line] : '';
+  // A program-level scope names the program, which survives the file being moved or renamed.
+  // Anything without one - a copybook's data, a Dockerfile, a site file - is known by its path.
+  const scope = facts.kind === 'cobol' && /\//.test(scoped) && !isCopybook(join(root, f.path)) ? scoped : `${f.path || ''}#${scoped}`;
+  // Line 1 is where a set reports a whole file or a whole program; its text is not what the
+  // finding is about.
+  const text = line > 1 && !SECRET_LINE.has(f.rule) ? codeText(facts.lines[line - 1], facts.kind, facts.format) : '';
+  const subject = [f.program, f.name, f.step].filter((x) => x != null && x !== '').join('|');
+  return { scope, subject, text };
+}
+
+const digest = (parts) => createHash('sha256').update(parts.join('\u0000')).digest('hex').slice(0, 32);
+
+// Stamps `fingerprint` on every finding and returns how many shared one with an earlier finding.
+// `repo` names the repository when one report holds several, where the same program id in two
+// repositories is two programs.
+export function stampFingerprints(findings, { root, repo = '' } = {}) {
+  const cache = new Map();
+  const seen = new Set();
+  let shared = 0;
+  for (const f of findings) {
+    const { scope, subject, text } = partsOf(f, root, cache);
+    const fp = digest([FINGERPRINT_VERSION, f.rule, repo, scope, subject, text]);
+    if (seen.has(fp)) shared++;
+    seen.add(fp);
+    f.fingerprint = fp;
+  }
+  return { version: FINGERPRINT_VERSION, shared };
+}
+
+// Looser keys for one finding seen in two trees whose line was edited between them: `scope` is the
+// fingerprint without the line's text, and `route` names a data-flow finding by the statement its
+// trace starts at. Null where a finding has no route.
+export function pairingKeys(findings, { root, repo = '' } = {}) {
+  const cache = new Map();
+  return findings.map((f) => {
+    const { scope, subject } = partsOf(f, root, cache);
+    const src = f.evidence === 'path' && f.related && f.related[0];
+    let route = null;
+    if (src && src.path) {
+      const facts = fileFacts(root, src.path, cache);
+      const text = codeText(facts.lines[(Number(src.line) || 0) - 1], facts.kind, facts.format);
+      route = digest([FINGERPRINT_VERSION, 'route', f.rule, repo, f.program || '', src.path, text]);
+    }
+    return { scope: digest([FINGERPRINT_VERSION, 'scope', f.rule, repo, scope, subject]), route };
+  });
+}
+
+// Every line of a file as the fingerprint reads it, led by a fixed-format line's indicator column so
+// that a line commented out reads as changed. Null when the file cannot be read.
+export function codeLines(root, path) {
+  const full = join(root, path);
+  let src;
+  try { src = readSource(full).text; } catch { return null; }
+  const kind = isJcl(full) ? 'jcl' : isProgram(full) || isCopybook(full) ? 'cobol' : 'other';
+  const format = kind === 'cobol' ? detectFormat(src) : null;
+  const indicator = format === 'fixed';
+  return src.split(/\r?\n/).map((l) => (indicator ? l.charAt(6) : '') + codeText(l, kind, format));
+}

package/lib/kernel/memory.mjs

@@ -0,0 +1,217 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// One place that decides how much more this process can afford to read, and one loop that every
+// rule set uses to stay inside it.
+//
+// Two different failures put a scan on the floor, and a budget in bytes of source only answers the
+// first:
+//
+//   volume        a repository holds more source than the heap can take. A byte budget answers
+//                 this: stop reading at N bytes, report the rest as unread.
+//
+//   accumulation  a rule set holds a structure whose size is not proportional to the source it has
+//                 read - a parse tree per program, a graph node per data item. The byte budget is
+//                 useless here, because the thing that grows is not the bytes. A 100,000-program
+//                 repository exhausted an 8GB heap with a 32MB source budget in force, because the
+//                 budget bounded the input and nothing bounded the structure.
+//
+// So this module measures the heap itself rather than trusting a proxy for it. `getAvailableMemory`
+// asks V8 what is left and asks the operating system whether it agrees; `watchMemoryBuffer` turns
+// that into a decision with hysteresis, so a scan does not stop on one unlucky sample taken before
+// a collection; and `eachWithinMemory` is the loop, so no rule set has to get this right on its own.
+//
+// What is never done is stopping quietly. Every function here reports what it did not read, and the
+// caller is expected to put that in `coverageIncomplete` - a finding count over files nobody opened
+// is not a clean result, and running out of memory is not an excuse to pretend otherwise.
+import { freemem } from 'node:os';
+import { getHeapStatistics } from 'node:v8';
+
+const MB = 1024 * 1024;
+
+// A floor below which we stop regardless of fractions. Enough to finish the statement in progress,
+// build a report and serialise it.
+export const MEMORY_FLOOR = 192 * MB;
+
+// How much of the heap to leave unspent. V8 needs headroom to collect at all: a mark-compact that
+// cannot find room is the "ineffective mark-compacts near heap limit" failure, which is a crash
+// rather than a slowdown.
+export const MEMORY_RESERVE = 0.2;
+
+let override = null;
+
+// The two readings this module takes from outside itself, and the only two. Both are injectable,
+// because a test that asks the real machine is a test whose verdict is a property of the machine.
+//
+// This was not hypothetical. Two sessions ran the same commit: one saw 206 of 206 pass with 502 MB
+// free, the other saw 17 fail with 330 MB free. The failing assertion was that a watcher reports
+// 'ok' when there is room, and whether there was room depended on what else the box happened to be
+// doing - other test workers, an editor, a second agent. `node --test` runs a process per file, so
+// the heaps were never shared; freemem() was, because it is the whole machine.
+//
+// Note which of the two is the dangerous one. `available` is the lesser of the heap's headroom and
+// the machine's, and on both boxes the machine term won - by 8x on one and 13x on the other. So
+// injecting a heap reader alone would leave the term that actually decides still wired to the
+// operating system, and the tests would look fixed while remaining a measurement of the hardware.
+
+// An operator's statement about the machine, in megabytes, for a process whose share of it is not
+// what `freemem` reports: a container with a memory limit, a CI runner sharing a host, a scan run
+// beside other work. `freemem` is the whole box, so without this the scan's coverage is decided by
+// what else the box happens to be doing - which is a real result, and usually not the one wanted.
+// It is honoured by a child process too, which is how a spawned scan inherits the same statement.
+export const FREE_MEMORY_ENV = 'COBOLWORK_FREE_MEMORY_MB';
+
+function envFree() {
+  const mb = Number(process.env[FREE_MEMORY_ENV]);
+  return Number.isFinite(mb) && mb > 0 ? () => mb * MB : null;
+}
+
+const defaultFree = () => (envFree() || freemem)();
+
+let readHeap = getHeapStatistics;
+let readFree = defaultFree;
+
+// Replace either reading. Called with nothing, or with a missing key, it restores the default one -
+// so a test can hand back the machine as easily as it took it. The default free reader honours
+// COBOLWORK_FREE_MEMORY_MB, so restoring does not discard an operator's statement.
+export function setMemoryReaders({ heap = null, free = null } = {}) {
+  readHeap = heap || getHeapStatistics;
+  readFree = free || defaultFree;
+}
+
+// An explicit ceiling, in bytes, for callers that know better than the heap does - a CI box sharing
+// a machine, a test that wants deterministic behaviour, an operator who has measured. Pass null to
+// go back to asking V8.
+export function setAvailableMemory(bytes) {
+  override = bytes == null ? null : Math.max(0, Number(bytes));
+}
+
+export function memoryStatus() {
+  const h = readHeap();
+  const used = h.used_heap_size;
+  const limit = override == null ? h.heap_size_limit : override;
+  const heapHeadroom = Math.max(0, limit - used);
+  // The heap limit is a promise V8 makes about itself, not about the machine. If the operating
+  // system has less free than V8 thinks it may take, the operating system wins.
+  const osHeadroom = readFree();
+  return {
+    limit, used, heapHeadroom, osHeadroom,
+    available: Math.max(0, Math.min(heapHeadroom, osHeadroom)),
+    fractionUsed: limit > 0 ? used / limit : 1,
+    overridden: override != null,
+  };
+}
+
+export const getAvailableMemory = () => memoryStatus().available;
+
+// Is there room to keep going? Returns 'ok', 'tight' (stop taking on new work) or 'exhausted'.
+//
+// Hysteresis matters: heapUsed sawtooths, and a single sample taken just before a collection would
+// stop a scan that had plenty of room. A watcher reports 'tight' only after `patience` consecutive
+// samples agree, so one unlucky reading is not a verdict.
+export function watchMemoryBuffer({ reserve = MEMORY_RESERVE, floor = MEMORY_FLOOR, patience = 3 } = {}) {
+  let consecutive = 0;
+  let peak = 0;
+  let samples = 0;
+
+  return {
+    check() {
+      const s = memoryStatus();
+      samples++;
+      peak = Math.max(peak, s.used);
+      const tight = s.available < floor || s.fractionUsed > 1 - reserve;
+      consecutive = tight ? consecutive + 1 : 0;
+      if (s.available < floor / 4) return 'exhausted';        // one sample is enough to believe this
+      return consecutive >= patience ? 'tight' : 'ok';
+    },
+    // A collection, if the process was started with --expose-gc. Never required: the loop is
+    // correct without it, and this only buys back headroom that was already garbage.
+    //
+    // It resets the patience counter because the readings before a collection describe a heap that
+    // no longer exists. That reset is why `stillTight` exists and why the caller must use it: on
+    // its own it would hand back a clean slate every time, and a loop that collects whenever it is
+    // told it is tight would never accumulate the consecutive readings needed to stop.
+    collect() {
+      if (typeof global.gc === 'function') { global.gc(); consecutive = 0; return true; }
+      return false;
+    },
+    // Is it still tight, right now, with no hysteresis?
+    //
+    // Hysteresis exists so that one unlucky sample taken just before a collection cannot stop a
+    // scan that had room. Immediately after a collection there is nothing left to be unlucky
+    // about, so the question is no longer "have several readings agreed" but the simpler one:
+    // did the collection actually help. Asking `check()` here instead is what defeated the guard -
+    // the reset inside `collect` guaranteed it answered 'ok', so a run started with --expose-gc
+    // ran on through sustained pressure until V8 aborted with ineffective mark-compacts. That is
+    // not hypothetical: it is how a 100,000-program repository killed a corpus run, on the guard
+    // written to stop exactly that.
+    stillTight() {
+      const s = memoryStatus();
+      return s.available < floor || s.fractionUsed > 1 - reserve;
+    },
+    get peak() { return peak; },
+    get samples() { return samples; },
+  };
+}
+
+const STOPPED_BECAUSE = {
+  'byte budget': 'the scan reached its source byte budget',
+  'memory reserve': 'the scan reached its memory reserve',
+  'memory exhausted': 'memory ran out',
+};
+export const stoppedBecause = (stoppedBy) => STOPPED_BECAUSE[stoppedBy] || `the scan stopped (${stoppedBy})`;
+
+// The loop every rule set uses instead of `for (const f of files)`.
+//
+// It stops when memory runs short or when a byte budget is spent, and it says which, and it names
+// everything it did not get to. `work` returns the number of bytes it took on, or nothing; a rule
+// set that reads a file should return its length so the byte budget means something.
+export function eachWithinMemory(items, work, {
+  maxBytes = Infinity,
+  every = 8,                 // sample the heap this often, not on every item
+  watcher = null,
+  label = 'scan',
+} = {}) {
+  const w = watcher || watchMemoryBuffer();
+  const skipped = [];
+  let processed = 0;
+  let bytes = 0;
+  let stoppedBy = null;
+
+  for (let i = 0; i < items.length; i++) {
+    if (stoppedBy) { skipped.push(items[i]); continue; }
+
+    if (bytes >= maxBytes) { stoppedBy = 'byte budget'; skipped.push(items[i]); continue; }
+
+    // Checking costs a syscall, so it is sampled rather than continuous - except once the heap is
+    // already past the reserve, when every item is checked because the next one may be the last.
+    if (i % every === 0 || w.check === undefined || memoryStatus().fractionUsed > 1 - MEMORY_RESERVE) {
+      const state = w.check();
+      if (state === 'exhausted' || state === 'tight') {
+        // Collect, and carry on only if the collection genuinely helped. Asking `check()` here
+        // instead asked a counter that `collect` had just reset, so the answer was always 'ok'.
+        if (!w.collect() || w.stillTight()) {
+          stoppedBy = state === 'exhausted' ? 'memory exhausted' : 'memory reserve';
+          skipped.push(items[i]);
+          continue;
+        }
+      }
+    }
+
+    const took = work(items[i], i);
+    if (typeof took === 'number') bytes += took;
+    processed++;
+  }
+
+  return {
+    label,
+    processed,
+    bytes,
+    skipped,
+    complete: skipped.length === 0,
+    stoppedBy,
+    peakHeapBytes: w.peak,
+    // The sentence a report should carry. Written here so every rule set says it the same way.
+    note: skipped.length === 0 ? null
+      : `${label}: ${skipped.length} of ${items.length} files were not read because ${stoppedBecause(stoppedBy)}. `
+        + 'The findings below are over what was read, and are not a result for this repository as a whole.',
+  };
+}

package/lib/kernel/printable.mjs

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Text bound for a report, terminal or log: control and bidirectional characters replaced, length capped.
+export const printable = (s, max = 200) => {
+  const t = String(s ?? '').replace(/[\u0000-\u001f\u007f-\u009f\u200e\u200f\u202a-\u202e\u2066-\u2069]/g, '?');
+  return t.length > max ? `${t.slice(0, max)}...` : t;
+};

package/lib/kernel/registry.mjs

@@ -0,0 +1,79 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Every rule set this tool has, in one list.
+//
+// Adding the opaque set cost nine edits across four files: an import, a spread into ALL_RULES, an
+// entry in RULE_SETS, a branch in scanAll, a line in the tool-name map, two lines in the feed
+// catalogue, the CLI usage string, and the key list test/cli.test.mjs enumerates by hand. Eight of
+// those nine are this file's job, and none of them is a decision - they are the same fact written
+// down nine times, which is eight chances to write it down wrong.
+//
+// One of those chances was taken. The flow set was renamed to `cobolwork-flow` while the map from
+// tool name to report key still said `cobolwork`, so that set's file counts went out under the
+// literal key `undefined` in every report for several commits. lib/scan.mjs grew an ESETNAME throw
+// to make the next rename fail loudly instead. The throw stays - see `reportKey` below - but the
+// map it guarded is gone, because a name derived from one place cannot disagree with itself.
+import { scan as scanFlow, RULES as FLOW_RULES } from '../sets/flow.mjs';
+import { scanCics, CICS_RULES } from '../sets/cics.mjs';
+import { scanHidden, HIDDEN_RULES } from '../sets/hidden.mjs';
+import { scanCopybooks, COPYBOOK_RULES } from '../sets/copybook.mjs';
+import { scanJcl, JCL_RULES } from '../sets/jcl.mjs';
+import { scanBuild, BUILD_RULES } from '../sets/build.mjs';
+import { scanRecon, RECON_RULES } from '../sets/recon.mjs';
+import { scanVendor, VENDOR_RULES } from '../sets/vendor.mjs';
+import { scanOpaque, OPAQUE_RULES } from '../sets/opaque.mjs';
+import { scanWeb, WEB_RULES } from '../sets/web.mjs';
+import { scanPriv, PRIV_RULES } from '../sets/priv.mjs';
+import { scanLog, LOG_RULES } from '../sets/log.mjs';
+import { scanCompile, COMPILE_RULES } from '../sets/compile.mjs';
+import { toolName } from './ruleset.mjs';
+
+// The order is the order a scan runs them and the order --only lists them. It is not significant
+// to correctness; it is significant to a report being comparable with yesterday's.
+export const REGISTRY = [
+  { name: 'flow', scan: scanFlow, rules: FLOW_RULES },
+  { name: 'cics', scan: scanCics, rules: CICS_RULES },
+  { name: 'hidden', scan: scanHidden, rules: HIDDEN_RULES },
+  { name: 'copybook', scan: scanCopybooks, rules: COPYBOOK_RULES },
+  { name: 'jcl', scan: scanJcl, rules: JCL_RULES },
+  { name: 'build', scan: scanBuild, rules: BUILD_RULES },
+  { name: 'recon', scan: scanRecon, rules: RECON_RULES },
+  { name: 'vendor', scan: scanVendor, rules: VENDOR_RULES },
+  { name: 'opaque', scan: scanOpaque, rules: OPAQUE_RULES },
+  { name: 'web', scan: scanWeb, rules: WEB_RULES },
+  { name: 'compile', scan: scanCompile, rules: COMPILE_RULES },
+  { name: 'priv', scan: scanPriv, rules: PRIV_RULES },
+  { name: 'log', scan: scanLog, rules: LOG_RULES },
+];
+
+// Re-exported from the kernel's ruleset module, which is where it has to live: this file imports
+// every rule set, and every rule set imports that one.
+export { toolName };
+
+export const RULE_SETS = REGISTRY.map((s) => s.name);
+
+export const ALL_RULES = Object.assign({}, ...REGISTRY.map((s) => s.rules));
+
+const BY_TOOL = new Map(REGISTRY.map((s) => [toolName(s.name), s.name]));
+
+// What a part's counts are filed under. An unregistered set still fails loudly rather than filing
+// under `undefined`, which is the property the ESETNAME throw was added for and the reason this
+// function exists instead of a bare lookup.
+export function reportKey(tool) {
+  const name = BY_TOOL.get(tool);
+  if (!name) throw Object.assign(new Error(`rule set ${tool} has no report key`), { code: 'ESETNAME', tool });
+  return name;
+}
+
+// Rule ids must be unique across sets: ALL_RULES is one flat object, so a duplicate would silently
+// take the last definition and a finding would carry another set's severity.
+export function duplicateRuleIds() {
+  const seen = new Map();
+  const dupes = [];
+  for (const s of REGISTRY) {
+    for (const id of Object.keys(s.rules)) {
+      if (seen.has(id)) dupes.push(`${id} is declared by both ${seen.get(id)} and ${s.name}`);
+      else seen.set(id, s.name);
+    }
+  }
+  return dupes;
+}

package/lib/kernel/ruleset.mjs

@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// What a rule set says about itself once it has finished looking.
+//
+// Eight sets wrote the same closing five lines: take the guarded loop's account of what it did not
+// reach, stamp the findings, tally them, and assemble a summary. The assembly varied, and none of
+// the variation was a decision - one set set coverageIncomplete inside its stats, another passed it
+// in the returned object, a third ORed it with a flag from the JCL parser, and one had a second
+// return site that skipped the whole thing and shipped a different shape.
+//
+// That last one matters more than tidiness. SECURITY.md treats a report that overstates coverage as
+// a security bug rather than a defect: a file the tool failed to read while still reporting
+// coverageIncomplete: false is the failure this project exists to prevent. Eight hand-written
+// copies of the code that makes that claim is eight places for it to be wrong, and the one that was
+// wrong was found by a test rather than by reading.
+//
+// WHAT THIS IS NOT. The specification proposed a `defineRuleSet({ perFile, finish })` harness that
+// would own each set's control flow and refuse a perFile that returned a parse tree. That is not
+// what this is, and the difference is deliberate. By the time this was written the traversal, the
+// reading, the parse configuration, the sort, the severity stamping and the tally had all moved
+// into lib/kernel/ on their own - so the inversion would have restructured nine working sets to
+// take ownership of what it already had. The ETREELEAK guard went with it: it only guards a
+// perFile that does not exist, and a set that accumulates parse trees in its own closure is
+// something no harness can prevent. What remains is the closing assembly, which is the part that
+// was still duplicated and the part that makes a security claim.
+import { finish } from './findings.mjs';
+
+// A set's tool name is its report key with the tool's name in front. It lives here rather than in
+// the registry because the registry imports every rule set, and every rule set imports this: taking
+// the name from there would close the loop, and the error a circular import gives - "cannot access
+// X_RULES before initialization" - names the rule table rather than the cycle that broke it.
+export const toolName = (name) => `cobolwork-${name}`;
+
+// `run` is what eachWithinMemory returned, or null for a set that did not traverse - a set refusing
+// to run for want of configuration still has to report in the same shape as one that ran.
+export function report(name, { rules, findings, stats = {}, run = null }) {
+  const skipped = run ? run.skipped.length : 0;
+
+  // The guarded loop's account of what it did not get to. Written here so every set says it the
+  // same way, which is the same reason lib/memory.mjs composes the sentence rather than each
+  // caller writing its own.
+  if (run) {
+    if (skipped) stats.filesNotRead = skipped;
+    if (run.stoppedBy) stats.stoppedBy = run.stoppedBy;
+    if (run.peakHeapBytes) stats.peakHeapBytes = run.peakHeapBytes;
+    if (run.note) stats.notRead = run.note;
+  }
+
+  const { byRule } = finish(rules, findings, name);
+
+  return {
+    tool: toolName(name),
+    summary: {
+      findings: findings.length,
+      byRule,
+      ...stats,
+      // ORed, never overwritten. A set may already know its reading was short for a reason the
+      // loop knows nothing about - the JCL parser reporting an INCLUDE it could not resolve, for
+      // one - and that claim must survive being combined with this one.
+      //
+      // A file opened and not understood is a file that did not contribute, exactly like one the
+      // loop never reached: the set has no findings from it and cannot say there were none. Only
+      // the compile set had joined those up, by hand, and the other nine reported complete coverage
+      // over source they could not parse - which is the I4 failure SECURITY.md calls a security bug
+      // rather than a defect. lib/sarif.mjs already counted all three as a shortfall, so the two
+      // halves of the tool disagreed about what a clean result means.
+      coverageIncomplete: stats.coverageIncomplete === true || skipped > 0
+        || (stats.filesUnparsed || 0) > 0 || (stats.filesUnreadable || 0) > 0,
+      nosrc: (stats.filesScanned || 0) === 0,
+    },
+    findings,
+  };
+}