@portll/cobolwork 0.5.0 → 0.7.0

package/lib/inventory.mjs

@@ -3,12 +3,13 @@ import { detectFormat } from './parser.mjs';
 import { inScope, isCopybook, isJcl, isProgram, relPath } from './sources.mjs';
 import { SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 import { treeFor } from './kernel/source-tree.mjs';
+import { drive, loopOver } from './kernel/shared-pass.mjs';
 
 const SYSTEM_COPY = /^(?:(?:DFH|DSN|CEE|IGZ|EZA|BPX|CSQ|CMQ|DLI)[A-Z0-9$#@]{0,5}|SQLCA|SQLDA|ATTRIB)$/i;
 
 // What the repository contains and, as importantly, what could not be read: a scan over a tree
 // whose copybooks are missing has not examined what it appears to have examined.
-export function inventory(root, opts = {}) {
+export function* inventorySteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const idx = tree.index;
   const files = tree.list().filter(inScope(opts));
@@ -28,27 +29,27 @@ export function inventory(root, opts = {}) {
     ebcdic: [],
     recursiveCopybooks: {},
   };
-  for (const f of files) {
+  yield loopOver(files, (f) => {
     if (isCopybook(f)) out.summary.copybookFiles++;
     if (isJcl(f)) out.summary.jclFiles++;
-    if (!isProgram(f)) continue;
+    if (!isProgram(f)) return;
     out.summary.programFiles++;
     let src;
     try {
       const s = tree.text(f);
       src = s.text;
       if (s.encoding === 'ebcdic') { out.summary.filesEbcdic++; out.ebcdic.push(tree.rel(f)); }
-    } catch (e) { out.summary.filesUnreadable++; out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`); continue; }
+    } catch (e) { out.summary.filesUnreadable++; out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`); return; }
     // A file with a program extension and no program in it is usually a copybook named .cbl; it is
     // counted so the difference between program files and programs read is accounted for.
-    if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) { out.summary.notPrograms++; continue; }
+    if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) { out.summary.notPrograms++; return; }
     let res;
     try {
       res = tree.parse(f, src);
     } catch (e) {
       out.summary.filesUnreadable++;
       out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`);
-      continue;
+      return;
     }
     out.summary.filesScanned++;
     out.summary.programs += res.programs.length;
@@ -71,8 +72,10 @@ export function inventory(root, opts = {}) {
         out.missingCopybooks[k] = (out.missingCopybooks[k] || 0) + 1;
       }
     }
-  }
+  }, { guarded: false, label: 'inventory' });
   out.summary.nosrc = out.summary.filesScanned === 0;
   out.summary.coverageIncomplete = out.summary.copiesMissing > 0 || out.summary.copiesRecursive > 0 || (out.summary.copiesOverLimit || 0) > 0 || out.summary.filesUnreadable > 0 || out.summary.copiesRefused > 0 || out.summary.dirsUnreadable > 0 || out.summary.symlinks.outside > 0;
   return out;
 }
+
+export const inventory = (root, opts = {}) => drive(inventorySteps(root, opts));

package/lib/ironwork-ids.mjs

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// What cobolwork reads from ironwork, and from which ironwork release: the formats it keys on, and
+// how ironwork says it ended a run. The run endings are ironwork's docs/run-endings.tsv; a test holds
+// them to the copy in test/fixtures/ironwork, and CI holds that copy to ironwork's.
+
+// Each format by the id its reader keys on, with the first ironwork release that writes all of what
+// cobolwork reads from it. A reader refuses an id it does not know, so a later format is not misread.
+export const IRONWORK_FORMATS = {
+  'ironwork-fuzz/v1': '0.3.0',
+  'ironwork-fuzz-interface/v1': '0.5.0',
+  'cobolwork-evidence/v1': '0.4.0',
+  'https://github.com/Portll/ironwork/blob/main/docs/evidence.md#equivalence-v1': '0.1.2',
+  'run-endings': '0.4.1',
+};
+
+// The first ironwork release whose output cobolwork reads throughout; a format listed with a later
+// release is read from that release on.
+export const IRONWORK_MINIMUM = '0.4.1';
+
+// How ironwork ends a run other than with its RETURN-CODE, by exit status.
+export const RUN_ENDINGS = { 240: 'abend', 241: 'refused', 242: 'not-generated', 243: 'stopped', 244: 'not-run', 245: 'unreadable', 246: 'usage', 255: 'internal' };
+
+// The abend codes that say a run reached what ironwork does not run.
+export const NOT_RUN_ABENDS = ['IRONWORK', 'EXEC', 'JAVA'];

package/lib/ironwork.mjs

@@ -15,18 +15,24 @@ import { EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS } from './words.mjs';
 // 12 S, 16 U. Only 0 and 4 produce a program.
 const COMPILED = new Set([0, 4]);
 const REJECTED = new Set([8, 12, 16]);
+// ironwork grades a refusal by name as severe (12), so a run that ends at 8 holds only IBM-graded
+// errors in the program, whatever their wording says; the wording decides only at 12 and 16.
+const GRADED_ERROR = 8;
 // Warnings and informational lines follow the errors, and say nothing about why a program failed.
-const NOT_AN_ERROR = /^(?:informational:|.*?: warning: )/;
+// The label follows the position, or opens a line that has none.
+const NOT_AN_ERROR = /(?:^|: )(?:warning|informational): /;
+// ironwork opens a message with its id and severity, `IWC0101-S`, as IBM opens its own.
+const MESSAGE_ID = /^((?:IW[A-Z]|IGY[A-Z]{2})\d{4})-[IWESU] (.*)$/;
 const PER_PROGRAM_MS = 60000;
 const MAX_LISTED = 10;
 
-// ironwork refuses by name what it does not model yet; that says nothing about the program. A name
-// the CICS, DL/I or SQL translator declares it reports as undefined, without saying so, and that is
-// read as its gap, not the program's.
+// ironwork refuses by name what it does not model yet, with an IWR id; that says nothing about the
+// program. A name the CICS, DL/I or SQL translator declares it reports as undefined, without saying
+// so, and that is read as its gap, not the program's.
 const NOT_SUPPORTED = /\bnot supported\b/i;
 const TRANSLATOR_NAMES = [EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS];
-function notModelled(message) {
-  if (NOT_SUPPORTED.test(message)) return true;
+function notModelled({ id, message }) {
+  if (id?.startsWith('IWR') || NOT_SUPPORTED.test(message)) return true;
   const name = /^([A-Z][A-Z0-9-]*) is not defined$/.exec(message);
   return !!name && TRANSLATOR_NAMES.some((s) => s.has(name[1]));
 }
@@ -42,9 +48,10 @@ function diagnostic(line, programPath, rel) {
   if (!m) return { path: rel, message: printable(redact(line), 200) };
   const [file, ln, col, text] = m.length === 5 ? [m[1], Number(m[2]), Number(m[3]), m[4]] : [m[1], null, null, m[2]];
   const inProgram = resolve(file) === programPath;
+  const [, id, said] = MESSAGE_ID.exec(text) || [null, null, text];
   return {
-    path: rel, ...(ln ? { line: ln, col } : {}), ...(inProgram ? {} : { member: printable(file, 80) }),
-    message: printable(redact(text), 200),
+    path: rel, ...(ln ? { line: ln, col } : {}), ...(inProgram ? {} : { member: printable(file, 80) }), ...(id ? { id } : {}),
+    message: printable(redact(said), 200),
   };
 }
 
@@ -57,9 +64,9 @@ export function ironworkVersion(path, { env = process.env } = {}) {
 // estate's copy libraries as -I, and sorts the programs into those it accepts, those it rejects,
 // and those it could not decide: a construct it does not model, a copybook the tree lacks, or a
 // run that ended some other way.
-export function checkWithIronwork(path, root, { allow = null, copylibs = [], env = process.env } = {}) {
-  const programs = [...new Set(directoryTree(root).list())].filter((p) => isProgram(p) && (!allow || allow.has(p))).sort();
-  const libraries = [...buildFileIndex(root).copyDirs, ...copylibs].flatMap((d) => ['-I', d]);
+export function checkWithIronwork(path, root, { allow = null, programs: given = null, copyDirs = null, copylibs = [], env = process.env } = {}) {
+  const programs = given || [...new Set(directoryTree(root).list())].filter((p) => isProgram(p) && (!allow || allow.has(p))).sort();
+  const libraries = [...(copyDirs || buildFileIndex(root).copyDirs), ...copylibs].flatMap((d) => ['-I', d]);
   const out = { programs: programs.length, accepted: 0, warned: 0, failed: [], notModelled: [], unresolved: [], unrun: [] };
   for (const file of programs) {
     const rel = relPath(root, file);
@@ -73,7 +80,7 @@ export function checkWithIronwork(path, root, { allow = null, copylibs = [], env
       continue;
     }
     const found = String(r.stderr || '').split(/\r?\n/).filter((l) => l && !NOT_AN_ERROR.test(l)).map((l) => diagnostic(l, resolve(file), rel));
-    const errors = found.filter((d) => !notModelled(d.message) && !UNRESOLVED.test(d.message));
+    const errors = found.filter((d) => !(r.status !== GRADED_ERROR && notModelled(d)) && !UNRESOLVED.test(d.message));
     if (errors.length) out.failed.push({ ...errors[0], errors: errors.length });
     else if (found.some((d) => UNRESOLVED.test(d.message))) out.unresolved.push(found.find((d) => UNRESOLVED.test(d.message)));
     else out.notModelled.push(found[0] || { path: rel, message: `ironwork exited ${r.status} and gave no error` });
@@ -90,7 +97,7 @@ export function ironworkVerdict(result) {
 
 export function ironworkReasons(result) {
   const reasons = [];
-  const at = (d) => `${d.path}${d.line ? `:${d.line}:${d.col}` : ''}${d.member ? ` (in ${d.member})` : ''}`;
+  const at = (d) => `${d.path}${d.line ? `:${d.line}:${d.col}` : ''}${d.member ? ` (in ${d.member})` : ''}${d.id ? ` ${d.id}` : ''}`;
   if (result.failed.length) {
     reasons.push(`ironwork check: ${result.failed.length} of ${result.programs} program(s) do not compile as Enterprise COBOL`);
     for (const d of result.failed.slice(0, MAX_LISTED)) reasons.push(`${at(d)}: ${d.message}`);

package/lib/kernel/pds-archive.mjs

@@ -0,0 +1,256 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Data sets brought off z/OS whole: a TSO TRANSMIT (XMIT) file, and an IEBCOPY unload data set, alone
+// or inside one. Each partitioned data set's members come back as their records, in EBCDIC, ended by
+// NL (X'15'), as a z/OSMF record download gives them; a sequential data set comes back as one file.
+//
+// The layouts are IBM's: z/OS TSO/E Customization, "Format of transmitted data", "Control record
+// formats" and "Numeric values" (the text units); z/OS DFSMSdfp Utilities, "Unload partitioned data
+// set format" (COPYR1, COPYR2, directory and member data records); z/OS DFSMSdfp Advanced Services,
+// "Data Extent Block (DEB) Fields" and "Device Characteristics Information" (DEVTYPE), which turn a
+// member block's MBBCCHHR into the TTR its directory entry holds.
+import { ebcdicByte } from '../sources.mjs';
+
+const EOR = 0x15;
+const UNLOAD_ID = [0xca, 0x6d, 0x0f];
+const ebcdic = (s) => Buffer.from([...s].map((c) => ebcdicByte(c)));
+const INMR = (n) => ebcdic(`INMR0${n}`);
+
+const KEY = { INMDSNAM: 0x0002, INMDSORG: 0x003c, INMLRECL: 0x0042, INMRECFM: 0x0049, INMUTILN: 0x1028, INMNUMF: 0x102f };
+const DSORG_PO = 0x0200;
+
+const textOf = (buf) => [...buf].map((b) => EBCDIC_TEXT[b]).join('');
+const EBCDIC_TEXT = (() => {
+  const t = new Array(256).fill('?');
+  for (let c = 0x20; c < 0x7f; c++) {
+    const b = ebcdicByte(String.fromCharCode(c));
+    if (b !== null) t[b] = String.fromCharCode(c);
+  }
+  return t;
+})();
+const uint = (buf) => buf.reduce((n, b) => n * 256 + b, 0);
+const isUnloadId = (buf, at) => buf.length >= at + 3 && UNLOAD_ID.every((b, i) => buf[at + i] === b);
+
+// What a file holds, from its first bytes: an XMIT file's first segment is its INMR01 header record;
+// an unload's COPYR1 carries X'CA6D0F' after its flag byte, with a BDW and SDW, an RDW, or nothing
+// before it.
+export function archiveKind(head) {
+  if (head.length >= 8 && (head[1] & 0xa0) === 0xa0 && head.subarray(2, 8).equals(INMR(1))) return 'xmit';
+  for (const at of [1, 5, 9]) if (isUnloadId(head, at)) return 'unload';
+  return null;
+}
+
+// The transmission's logical records, each from its segments: a length byte counting itself and the
+// flag byte, the flags (X'80' first, X'40' last, X'20' control record) and up to 253 bytes.
+function xmitRecords(buf) {
+  const records = [];
+  let parts = null;
+  let control = false;
+  for (let at = 0; at < buf.length;) {
+    const n = buf[at];
+    if (n < 2 || at + n > buf.length) break;
+    const flags = buf[at + 1];
+    if (flags & 0x80) { parts = []; control = (flags & 0x20) !== 0; }
+    if (parts) parts.push(buf.subarray(at + 2, at + n));
+    if (flags & 0x40 && parts) {
+      const bytes = Buffer.concat(parts);
+      records.push({ control, bytes });
+      parts = null;
+      if (control && bytes.subarray(0, 6).equals(INMR(6))) break;
+    }
+    at += n;
+  }
+  return records;
+}
+
+// Text units from `at`: a key, a count, and that many length and data pairs.
+function textUnits(bytes, at) {
+  const units = new Map();
+  while (at + 4 <= bytes.length) {
+    const key = bytes.readUInt16BE(at);
+    const count = bytes.readUInt16BE(at + 2);
+    at += 4;
+    const fields = [];
+    for (let i = 0; i < count && at + 2 <= bytes.length; i++) {
+      const len = bytes.readUInt16BE(at);
+      fields.push(bytes.subarray(at + 2, at + 2 + len));
+      at += 2 + len;
+    }
+    units.set(key, fields);
+  }
+  return units;
+}
+
+function readXmit(buf) {
+  const records = xmitRecords(buf);
+  const last = records[records.length - 1];
+  if (!last?.control || !last.bytes.subarray(0, 6).equals(INMR(6))) return { why: 'the transmission ends before its INMR06 trailer' };
+  const header = textUnits(records[0].bytes, 6);
+  const files = header.get(KEY.INMNUMF)?.[0];
+  if (files && uint(files) > 1) return { why: `the transmission holds ${uint(files)} files, and cobolwork reads one` };
+  const utilities = records.filter((r) => r.control && r.bytes.subarray(0, 6).equals(INMR(2))).map((r) => textUnits(r.bytes, 10));
+  const named = utilities.find((u) => u.has(KEY.INMDSNAM));
+  const dataSet = named ? named.get(KEY.INMDSNAM).map(textOf).join('.') : null;
+  const start = records.findIndex((r) => r.control && r.bytes.subarray(0, 6).equals(INMR(3)));
+  if (start < 0) return { why: 'the transmission has no INMR03 data control record' };
+  const data = records.slice(start + 1, -1).filter((r) => !r.control).map((r) => r.bytes);
+  const unloaded = utilities.some((u) => textOf(u.get(KEY.INMUTILN)?.[0] || Buffer.alloc(0)) === 'IEBCOPY');
+  if (unloaded) return { ...readUnloadRecords(data), kind: 'xmit', dataSet };
+  const dsorg = utilities.map((u) => u.get(KEY.INMDSORG)?.[0]).find(Boolean);
+  if (dsorg && uint(dsorg) === DSORG_PO) return { why: 'a partitioned data set transmitted without IEBCOPY' };
+  return { kind: 'xmit', dataSet, sequential: true, members: [{ name: null, bytes: joined(data) }], aliases: [] };
+}
+
+// An unload stored with its descriptors: each block a BDW, then segments, each an SDW whose third
+// byte's low two bits say whether it is a whole record (0), its first (1), last (2) or middle (3)
+// part; or each record after an RDW; or the records with nothing between them, COPYR1 and COPYR2 at
+// their fixed lengths and everything after them one stream.
+function unloadRecords(buf) {
+  if (isUnloadId(buf, 9)) {
+    const records = [];
+    let parts = [];
+    for (let at = 0; at + 4 <= buf.length;) {
+      const end = at + buf.readUInt16BE(at);
+      if (end <= at || end > buf.length) break;
+      for (let s = at + 4; s + 4 <= end;) {
+        const len = buf.readUInt16BE(s);
+        if (len < 4) break;
+        const code = buf[s + 2] & 0x03;
+        parts.push(buf.subarray(s + 4, s + len));
+        if (code === 0 || code === 2) { records.push(Buffer.concat(parts)); parts = []; }
+        s += len;
+      }
+      at = end;
+    }
+    return records;
+  }
+  if (isUnloadId(buf, 5)) {
+    const records = [];
+    for (let at = 0; at + 4 <= buf.length;) {
+      const len = buf.readUInt16BE(at);
+      if (len < 4 || at + len > buf.length) break;
+      records.push(buf.subarray(at + 4, at + len));
+      at += len;
+    }
+    return records;
+  }
+  return [buf.subarray(0, COPYR1_LENGTH), buf.subarray(COPYR1_LENGTH, COPYR1_LENGTH + COPYR2_LENGTH), buf.subarray(COPYR1_LENGTH + COPYR2_LENGTH)];
+}
+
+const COPYR1_LENGTH = 56;
+const COPYR2_LENGTH = 276;
+const DIRECTORY_BLOCK = 276;
+const BLOCK_HEADER = 12;
+
+export function readArchive(buf) {
+  const kind = archiveKind(buf.subarray(0, 16));
+  if (kind === 'xmit') return readXmit(buf);
+  if (kind === 'unload') return { ...readUnloadRecords(unloadRecords(buf)), kind: 'unload', dataSet: null };
+  return { why: 'neither an XMIT file nor an IEBCOPY unload' };
+}
+
+// COPYR1 and COPYR2, then the directory blocks and every member's blocks, which run on across
+// records: a directory block is a 12-byte count, the 8-byte key and 256 bytes of entries, and the
+// directory ends with 12 bytes of zeros; a member's blocks are each a flag byte, MBB, CCHHR, key
+// length, data length, key and data, and its last is one with no key and no data.
+function readUnloadRecords(records) {
+  const [r1, r2] = records;
+  if (!r1 || !isUnloadId(r1, 1)) return { why: 'the unload has no COPYR1 record' };
+  const format = r1[0] >> 6;
+  if (format === 2) return { why: 'IEBCOPY marked the unload incomplete or in error' };
+  if (format !== 0 || r1[0] & 0x01) return { why: 'a PDSE unload, whose attribute records cobolwork does not read yet' };
+  const headers = r1.length >= 38 ? r1.readUInt16BE(36) || 2 : 2;
+  if (headers !== 2) return { why: `the unload has ${headers} header records, and cobolwork reads two` };
+  if (!r2 || r2.length < 16 + 16) return { why: 'the unload has no COPYR2 record' };
+  const lrecl = r1.readUInt16BE(8);
+  const recfm = r1[10] >> 6;
+  const tracksPerCylinder = r1.readUInt16BE(26);
+  const extents = [];
+  for (let at = 16; at + 16 <= r2.length && extents.length < 16; at += 16) {
+    extents.push({ cyl: cylinder(r2.readUInt16BE(at + 6), r2.readUInt16BE(at + 8)), trk: r2.readUInt16BE(at + 8) & 0x0f, tracks: r2[at + 5] * 0x10000 + r2.readUInt16BE(at + 14) });
+  }
+  const stream = Buffer.concat(records.slice(2));
+  let at = 0;
+  const entries = [];
+  for (;;) {
+    if (at + BLOCK_HEADER > stream.length) return { why: 'the directory runs past the end of the unload' };
+    if (stream.subarray(at, at + BLOCK_HEADER).every((b) => b === 0)) { at += BLOCK_HEADER; break; }
+    const block = stream.subarray(at + BLOCK_HEADER + 8, at + DIRECTORY_BLOCK);
+    const used = Math.min(block.readUInt16BE(0), block.length);
+    for (let e = 2; e + 12 <= used;) {
+      const name = block.subarray(e, e + 8);
+      if (name.every((b) => b === 0xff)) break;
+      const c = block[e + 11];
+      entries.push({ name: textOf(name).trimEnd(), ttr: uint(block.subarray(e + 8, e + 11)), alias: (c & 0x80) !== 0 });
+      e += 12 + (c & 0x1f) * 2;
+    }
+    at += DIRECTORY_BLOCK;
+  }
+  const byTtr = new Map(entries.filter((e) => !e.alias).map((e) => [e.ttr, e.name]));
+  const members = [];
+  let unmatched = 0;
+  while (at + BLOCK_HEADER <= stream.length) {
+    let ttr = null;
+    const blocks = [];
+    for (;;) {
+      if (at + BLOCK_HEADER > stream.length) return { why: 'a member runs past the end of the unload' };
+      const h = stream.subarray(at, at + BLOCK_HEADER);
+      const keyLength = h[9];
+      const dataLength = h.readUInt16BE(10);
+      at += BLOCK_HEADER;
+      if (keyLength === 0 && dataLength === 0) break;
+      if (ttr === null) ttr = relativeTrack(extents, tracksPerCylinder, h) * 256 + h[8];
+      blocks.push(stream.subarray(at + keyLength, at + keyLength + dataLength));
+      at += keyLength + dataLength;
+    }
+    if (!blocks.length) continue;
+    const name = byTtr.get(ttr);
+    if (name === undefined) { unmatched++; continue; }
+    members.push({ name, bytes: joined(blocks.flatMap((b) => recordsOf(b, recfm, lrecl))) });
+  }
+  const held = new Set(members.map((m) => m.name));
+  return {
+    members,
+    aliases: entries.filter((e) => e.alias).map((e) => e.name),
+    ...(unmatched ? { unmatched } : {}),
+    ...(entries.some((e) => !e.alias && !held.has(e.name)) ? { missing: entries.filter((e) => !e.alias && !held.has(e.name)).map((e) => e.name) } : {}),
+  };
+}
+
+// A cylinder number from CC and HH, whose high 12 bits hold the cylinder's high bits on an extended
+// address volume (DEBSTRHH).
+const cylinder = (cc, hh) => (hh >> 4) * 0x10000 + cc;
+
+// The track's number from the start of the data set: the tracks of the extents before extent M, then
+// the cylinders and tracks into M.
+function relativeTrack(extents, tracksPerCylinder, h) {
+  const m = h[1];
+  const e = extents[m];
+  if (!e) return -1;
+  const before = extents.slice(0, m).reduce((n, x) => n + x.tracks, 0);
+  const hh = h.readUInt16BE(6);
+  return before + (cylinder(h.readUInt16BE(4), hh) - e.cyl) * tracksPerCylinder + ((hh & 0x0f) - e.trk);
+}
+
+// A physical block's records: fixed-length records LRECL at a time, variable ones after the BDW each
+// by its RDW, an undefined block as one record. RECFM's first two bits: 10 fixed, 01 variable, 11
+// undefined.
+function recordsOf(block, recfm, lrecl) {
+  if (recfm === 2 && lrecl > 0) {
+    const out = [];
+    for (let i = 0; i + lrecl <= block.length; i += lrecl) out.push(block.subarray(i, i + lrecl));
+    return out;
+  }
+  if (recfm === 1) {
+    const out = [];
+    for (let i = 4; i + 4 <= block.length;) {
+      const len = block.readUInt16BE(i);
+      if (len < 4) break;
+      out.push(block.subarray(i + 4, i + len));
+      i += len;
+    }
+    return out;
+  }
+  return [block];
+}
+
+const joined = (records) => Buffer.concat(records.flatMap((r) => [r, Buffer.from([EOR])]));

package/lib/kernel/registry.mjs

@@ -12,47 +12,56 @@
 // literal key `undefined` in every report for several commits. lib/scan.mjs grew an ESETNAME throw
 // to make the next rename fail loudly instead. The throw stays - see `reportKey` below - but the
 // map it guarded is gone, because a name derived from one place cannot disagree with itself.
-import { scan as scanFlow, RULES as FLOW_RULES } from '../sets/flow.mjs';
-import { scanCics, CICS_RULES } from '../sets/cics.mjs';
-import { scanHidden, HIDDEN_RULES } from '../sets/hidden.mjs';
-import { scanCopybooks, COPYBOOK_RULES } from '../sets/copybook.mjs';
+import { scan as scanFlow, scanSteps as scanFlowSteps, RULES as FLOW_RULES } from '../sets/flow.mjs';
+import { scanCics, scanCicsSteps, CICS_RULES } from '../sets/cics.mjs';
+import { scanHidden, scanHiddenSteps, HIDDEN_RULES } from '../sets/hidden.mjs';
+import { scanCopybooks, scanCopybooksSteps, COPYBOOK_RULES } from '../sets/copybook.mjs';
 import { scanJcl, JCL_RULES } from '../sets/jcl.mjs';
 import { scanBuild, BUILD_RULES } from '../sets/build.mjs';
-import { scanRecon, RECON_RULES } from '../sets/recon.mjs';
+import { scanRecon, scanReconSteps, RECON_RULES } from '../sets/recon.mjs';
 import { scanVendor, VENDOR_RULES } from '../sets/vendor.mjs';
 import { scanOpaque, OPAQUE_RULES } from '../sets/opaque.mjs';
-import { scanWeb, WEB_RULES } from '../sets/web.mjs';
-import { scanPriv, PRIV_RULES } from '../sets/priv.mjs';
-import { scanLog, LOG_RULES } from '../sets/log.mjs';
-import { scanCompile, COMPILE_RULES } from '../sets/compile.mjs';
-import { scanSemantics, SEMANTICS_RULES } from '../sets/semantics.mjs';
+import { scanWeb, scanWebSteps, WEB_RULES } from '../sets/web.mjs';
+import { scanPriv, scanPrivSteps, PRIV_RULES } from '../sets/priv.mjs';
+import { scanLog, scanLogSteps, LOG_RULES } from '../sets/log.mjs';
+import { scanCompile, scanCompileSteps, COMPILE_RULES } from '../sets/compile.mjs';
+import { scanSemantics, scanSemanticsSteps, SEMANTICS_RULES } from '../sets/semantics.mjs';
 import { scanZowe, ZOWE_RULES } from '../sets/zowe.mjs';
 import { scanAbend, ABEND_RULES } from '../sets/abend.mjs';
 import { scanHlasm, HLASM_RULES } from '../sets/hlasm.mjs';
-import { scanCrypto, CRYPTO_RULES } from '../sets/crypto.mjs';
+import { scanIms, IMS_RULES } from '../sets/ims.mjs';
+import { scanDb2, DB2_RULES } from '../sets/ddl.mjs';
+import { scanPli, PLI_RULES } from '../sets/pli.mjs';
+import { scanCrypto, scanCryptoSteps, CRYPTO_RULES } from '../sets/crypto.mjs';
+import { scanSecrets, scanSecretsSteps, SECRETS_RULES } from '../sets/secrets.mjs';
 import { toolName } from './ruleset.mjs';
 
 // The order is the order a scan runs them and the order --only lists them. It is not significant
-// to correctness; it is significant to a report being comparable with yesterday's.
+// to correctness; it is significant to a report being comparable with yesterday's. A set with
+// `steps` reads programs through the scan's one shared pass (lib/kernel/shared-pass.mjs).
 export const REGISTRY = [
-  { name: 'flow', scan: scanFlow, rules: FLOW_RULES },
-  { name: 'cics', scan: scanCics, rules: CICS_RULES },
-  { name: 'hidden', scan: scanHidden, rules: HIDDEN_RULES },
-  { name: 'copybook', scan: scanCopybooks, rules: COPYBOOK_RULES },
+  { name: 'flow', scan: scanFlow, steps: scanFlowSteps, rules: FLOW_RULES },
+  { name: 'cics', scan: scanCics, steps: scanCicsSteps, rules: CICS_RULES },
+  { name: 'hidden', scan: scanHidden, steps: scanHiddenSteps, rules: HIDDEN_RULES },
+  { name: 'copybook', scan: scanCopybooks, steps: scanCopybooksSteps, rules: COPYBOOK_RULES },
   { name: 'jcl', scan: scanJcl, rules: JCL_RULES },
   { name: 'build', scan: scanBuild, rules: BUILD_RULES },
-  { name: 'recon', scan: scanRecon, rules: RECON_RULES },
+  { name: 'recon', scan: scanRecon, steps: scanReconSteps, rules: RECON_RULES },
   { name: 'vendor', scan: scanVendor, rules: VENDOR_RULES },
   { name: 'opaque', scan: scanOpaque, rules: OPAQUE_RULES },
-  { name: 'web', scan: scanWeb, rules: WEB_RULES },
-  { name: 'compile', scan: scanCompile, rules: COMPILE_RULES },
-  { name: 'priv', scan: scanPriv, rules: PRIV_RULES },
-  { name: 'log', scan: scanLog, rules: LOG_RULES },
-  { name: 'semantics', scan: scanSemantics, rules: SEMANTICS_RULES },
+  { name: 'web', scan: scanWeb, steps: scanWebSteps, rules: WEB_RULES },
+  { name: 'compile', scan: scanCompile, steps: scanCompileSteps, rules: COMPILE_RULES },
+  { name: 'priv', scan: scanPriv, steps: scanPrivSteps, rules: PRIV_RULES },
+  { name: 'log', scan: scanLog, steps: scanLogSteps, rules: LOG_RULES },
+  { name: 'semantics', scan: scanSemantics, steps: scanSemanticsSteps, rules: SEMANTICS_RULES },
   { name: 'zowe', scan: scanZowe, rules: ZOWE_RULES },
   { name: 'abend', scan: scanAbend, rules: ABEND_RULES },
   { name: 'hlasm', scan: scanHlasm, rules: HLASM_RULES },
-  { name: 'crypto', scan: scanCrypto, rules: CRYPTO_RULES },
+  { name: 'ims', scan: scanIms, rules: IMS_RULES },
+  { name: 'ddl', scan: scanDb2, rules: DB2_RULES },
+  { name: 'pli', scan: scanPli, rules: PLI_RULES },
+  { name: 'crypto', scan: scanCrypto, steps: scanCryptoSteps, rules: CRYPTO_RULES },
+  { name: 'secrets', scan: scanSecrets, steps: scanSecretsSteps, rules: SECRETS_RULES },
 ];
 
 // Re-exported from the kernel's ruleset module, which is where it has to live: this file imports

package/lib/kernel/shared-pass.mjs

@@ -0,0 +1,163 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// One pass over the tree for every rule set that reads programs, so that a file is read and parsed
+// once per scan rather than once per set.
+//
+// A set's scan is a generator. Where it would run its guarded loop it yields `loopOver(...)` and is
+// resumed with the loop's account, exactly what eachWithinMemory would have returned. `drive` runs
+// one set on its own, its loop as it always ran. `driveTogether` runs several: it gathers the loop
+// each asks for, walks the files of all of them once in the tree's order, gives each set its own
+// files in its own order, and resumes each with its own account. While a file is being handed round,
+// the tree answers text() and parse() for it from what the first set to ask was given.
+//
+// Each set's per-file work is unchanged, so its findings are the ones it reported alone. What is
+// shared is the reading and the parse, which is why no set may change a parse result it is given.
+import { eachWithinMemory, memoryStatus, stoppedBecause, watchMemoryBuffer, MEMORY_RESERVE } from './memory.mjs';
+
+const LOOP = Symbol('loop');
+const LOOPS = Symbol('loops');
+
+// What a set yields in place of `eachWithinMemory(items, work, opts)`. `guarded: false` asks for a
+// plain loop: every item, no memory check and no byte budget.
+export const loopOver = (items, work, opts = {}) => ({ [LOOP]: true, items, work, opts });
+
+// Several loops of one set walked in the same pass, each with its own account, the set resumed with
+// their accounts in the order given. A file both loops hold goes to the first, then the second.
+export const loopsOver = (...loops) => ({ [LOOPS]: true, loops });
+const loopsOf = (value) => (value[LOOPS] ? value.loops : [value]);
+const resumeWith = (value, runs) => (value[LOOPS] ? runs : runs[0]);
+
+function plainLoop(items, work, label) {
+  for (let i = 0; i < items.length; i++) work(items[i], i);
+  return { label, processed: items.length, bytes: 0, skipped: [], complete: true, stoppedBy: null, peakHeapBytes: 0, note: null };
+}
+
+const runAlone = (req) => (req.opts.guarded === false
+  ? plainLoop(req.items, req.work, req.opts.label || 'scan')
+  : eachWithinMemory(req.items, req.work, req.opts));
+
+export function drive(steps) {
+  let step = steps.next();
+  while (!step.done) step = steps.next(resumeWith(step.value, loopsOf(step.value).map(runAlone)));
+  return step.value;
+}
+
+// A tree that, while `focus` names a file, answers bytes(), text() and parse() for that file once and
+// hands every later caller the same answer, its error included. A tree whose text is its bytes
+// through `decode` has the file read once for both.
+export function sharingTree(base) {
+  let focus = null;
+  let bytes = null;
+  let text = null;
+  let parsed = null;
+  const once = (held, read) => {
+    if (!held) { try { held = { value: read() }; } catch (e) { held = { error: e }; } }
+    return held;
+  };
+  const answer = (held) => {
+    if (held.error) throw held.error;
+    return held.value;
+  };
+  const tree = {
+    ...base,
+    focusOn(f) { focus = f; bytes = null; text = null; parsed = null; },
+    bytes(p) {
+      if (p !== focus) return base.bytes(p);
+      bytes = once(bytes, () => base.bytes(p));
+      return answer(bytes);
+    },
+    text(p) {
+      if (p !== focus) return base.text(p);
+      text = once(text, () => (base.decode ? base.decode(tree.bytes(p)) : base.text(p)));
+      return answer(text);
+    },
+    parse(p, src) {
+      if (p !== focus) return base.parse(p, src);
+      const own = src ?? tree.text(p).text;
+      if (!parsed || parsed.src !== own) {
+        try { parsed = { src: own, value: base.parse(p, own) }; } catch (e) { parsed = { src: own, error: e }; }
+      }
+      if (parsed.error) throw parsed.error;
+      return parsed.value;
+    },
+  };
+  return tree;
+}
+
+// Runs every generator to completion and returns their results in the order given. A generator
+// that throws, before its loop or inside it, yields { error } in its place, as a set that throws
+// alone does; the others carry on.
+export function driveTogether(stepsList, tree) {
+  const live = stepsList.map((steps) => ({ steps, step: null, result: undefined, error: null }));
+  const advance = (g, input, thrown) => {
+    try { g.step = thrown ? g.steps.throw(thrown) : g.steps.next(input); } catch (e) { g.error = e; g.step = { done: true }; return; }
+    if (g.step.done) g.result = g.step.value;
+  };
+  for (const g of live) advance(g);
+  for (;;) {
+    const waiting = live.filter((g) => !g.step.done);
+    if (!waiting.length) break;
+    const runs = mergedLoop(waiting.flatMap((g) => loopsOf(g.step.value)), tree);
+    let at = 0;
+    for (const g of waiting) {
+      const own = runs.slice(at, at += loopsOf(g.step.value).length);
+      advance(g, resumeWith(g.step.value, own.map((r) => r.run)), own.find((r) => r.error)?.error);
+    }
+  }
+  return live.map((g) => (g.error ? { error: g.error } : g.result));
+}
+
+// The files of every request, once each, in the order of the tree's list. A request whose items are
+// not in that order runs alone first, so no set sees its files in an order its own loop would not use.
+function mergedLoop(requests, tree) {
+  const rank = new Map(tree.list().map((p, i) => [p, i]));
+  const inOrder = (items) => items.every((p, i) => rank.has(p) && (i === 0 || rank.get(items[i - 1]) < rank.get(p)));
+  const states = requests.map((req) => ({
+    req, guarded: req.opts.guarded !== false, maxBytes: req.opts.maxBytes ?? Infinity, label: req.opts.label || 'scan',
+    next: 0, processed: 0, bytes: 0, skipped: [], stoppedBy: null, error: null, alone: null,
+  }));
+  for (const s of states) if (!inOrder(s.req.items)) {
+    try { s.alone = runAlone(s.req); } catch (e) { s.error = e; s.alone = {}; }
+  }
+  const shared = states.filter((s) => !s.alone);
+  const w = shared.map((s) => s.req.opts.watcher).find(Boolean) || watchMemoryBuffer();
+  const every = Math.min(...shared.map((s) => s.req.opts.every ?? 8), 8);
+  const files = [...new Set(shared.flatMap((s) => s.req.items))].sort((a, b) => rank.get(a) - rank.get(b));
+  let stoppedBy = null;
+  for (let i = 0; i < files.length; i++) {
+    const f = files[i];
+    const wanting = shared.filter((s) => s.req.items[s.next] === f);
+    if (!stoppedBy && wanting.some((s) => s.guarded && !s.error && s.bytes < s.maxBytes)
+      && (i % every === 0 || memoryStatus().fractionUsed > 1 - MEMORY_RESERVE)) {
+      const state = w.check();
+      if ((state === 'exhausted' || state === 'tight') && (!w.collect() || w.stillTight())) {
+        stoppedBy = state === 'exhausted' ? 'memory exhausted' : 'memory reserve';
+      }
+    }
+    tree.focusOn?.(f);
+    for (const s of wanting) {
+      const index = s.next++;
+      if (s.error) continue;
+      if (s.guarded) {
+        if (s.stoppedBy) { s.skipped.push(f); continue; }
+        if (s.bytes >= s.maxBytes) { s.stoppedBy = 'byte budget'; s.skipped.push(f); continue; }
+        if (stoppedBy) { s.stoppedBy = stoppedBy; s.skipped.push(f); continue; }
+      }
+      let took;
+      try { took = s.req.work(f, index); } catch (e) { s.error = e; continue; }
+      if (typeof took === 'number') s.bytes += took;
+      s.processed++;
+    }
+    tree.focusOn?.(null);
+  }
+  return states.map((s) => {
+    if (s.alone) return { run: s.alone, error: s.error };
+    const { items } = s.req;
+    const run = s.guarded
+      ? { label: s.label, processed: s.processed, bytes: s.bytes, skipped: s.skipped, complete: s.skipped.length === 0, stoppedBy: s.stoppedBy, peakHeapBytes: w.peak,
+        note: s.skipped.length === 0 ? null
+          : `${s.label}: ${s.skipped.length} of ${items.length} files were not read because ${stoppedBecause(s.stoppedBy)}. `
+            + 'The findings below are over what was read, and are not a result for this repository as a whole.' }
+      : { label: s.label, processed: s.processed, bytes: 0, skipped: [], complete: true, stoppedBy: null, peakHeapBytes: 0, note: null };
+    return { run, error: s.error };
+  });
+}