@portll/cobolwork 0.2.140 → 0.2.150

package/lib/kernel/git.mjs

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Reading a revision out of a repository with git plumbing only.
+import { spawnSync } from 'node:child_process';
+
+// A reviewed repository's .git/config can name commands; with fsmonitor off and plumbing only, git runs none.
+const GIT_ENV = { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0' };
+export const git = (repo, args, opts = {}) => spawnSync('git', ['-c', 'core.fsmonitor=false', '-C', repo, ...args], { env: GIT_ENV, maxBuffer: 256 * 1024 * 1024, ...opts });
+export const refused = (what, ref, why) => Object.assign(new Error(`${what} ${ref} failed: ${why}`), { code: 'EDIFFREF' });
+const BATCH_BYTES = 64 * 1024 * 1024;
+
+export function treeOf(repo, ref) {
+  if (!ref || String(ref).startsWith('-')) throw refused('git rev-parse', ref, 'a revision cannot be empty or start with "-"');
+  const r = git(repo, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{tree}`], { encoding: 'utf8' });
+  const oid = String(r.stdout || '').trim();
+  if (r.status !== 0 || !/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(oid)) throw refused('git rev-parse', ref, String(r.stderr || '').trim() || 'not a revision');
+  return oid;
+}
+
+// The parts of a tree path, or null if a part is empty, climbs, names a drive or stream, or is .git.
+export function treePathParts(path) {
+  const parts = path.split('/');
+  return parts.some((p) => !p || p === '.' || p === '..' || /[\\:\0]/.test(p) || p.toLowerCase() === '.git') ? null : parts;
+}
+
+// Every file in the tree `oid` as committed, as [{ path, bytes }] in tree order: blobs as stored,
+// with no filter or line-ending conversion. Links and submodules are left out and counted.
+export function revisionBlobs(repo, oid, ref = oid) {
+  const listed = git(repo, ['ls-tree', '-r', '-z', '-l', '--full-tree', oid]);
+  if (listed.status !== 0) throw refused('git ls-tree', ref, String(listed.stderr || '').trim() || `exit ${listed.status}`);
+  const blobs = [];
+  let links = 0;
+  for (const entry of listed.stdout.toString('utf8').split('\0')) {
+    const m = /^(\d{6}) blob ([0-9a-f]+) +(\d+)\t(.+)$/s.exec(entry);
+    if (!m) continue;
+    if (m[1] === '120000') { links++; continue; }
+    blobs.push({ oid: m[2], size: Number(m[3]), path: m[4] });
+  }
+  // Batched by bytes as well as count, and each batch's buffer sized to hold it, so one large blob
+  // cannot overflow a buffer sized for the others.
+  const batches = [];
+  for (const b of blobs) {
+    const last = batches[batches.length - 1];
+    if (!last || last.length >= 500 || last.bytes + b.size > BATCH_BYTES) batches.push(Object.assign([b], { bytes: b.size }));
+    else { last.push(b); last.bytes += b.size; }
+  }
+  for (const batch of batches) {
+    const r = git(repo, ['cat-file', '--batch'], { input: batch.map((b) => b.oid).join('\n') + '\n', maxBuffer: batch.bytes + batch.length * 128 + 4096 });
+    if (r.status !== 0) throw refused('git cat-file', ref, String(r.stderr || '').trim() || (r.error ? r.error.code || r.error.message : `exit ${r.status}`));
+    let at = 0;
+    for (const b of batch) {
+      const nl = r.stdout.indexOf(0x0a, at);
+      const head = r.stdout.toString('utf8', at, nl).split(' ');
+      if (head[1] !== 'blob') throw refused('git cat-file', ref, `${b.oid} is ${head[1] || 'missing'}`);
+      const size = Number(head[2]);
+      b.bytes = r.stdout.subarray(nl + 1, nl + 1 + size);
+      at = nl + 1 + size + 1;
+    }
+  }
+  return { blobs, links };
+}

package/lib/kernel/identity.mjs

@@ -131,14 +131,15 @@ function codeText(line, kind, format) {
   return maskSecrets(s).replace(/\s+/g, ' ').trim();
 }
 
-// Reads each file once, however many findings it holds.
+// Reads each file once, however many findings it holds, from disk or through the reader the
+// caller's source tree supplies.
 function fileFacts(root, path, cache) {
   let facts = cache.get(path);
   if (facts) return facts;
   facts = { kind: 'other', lines: [], scopes: null, format: null };
   cache.set(path, facts);
   let src;
-  try { src = readSource(join(root, path)).text; } catch { return facts; }
+  try { src = cache.read ? cache.read(path) : readSource(join(root, path)).text; } catch { return facts; }
   facts.lines = src.split(/\r?\n/);
   if (isJcl(join(root, path))) {
     facts.kind = 'jcl';
@@ -171,8 +172,8 @@ const digest = (parts) => createHash('sha256').update(parts.join('\u0000')).dige
 // Stamps `fingerprint` on every finding and returns how many shared one with an earlier finding.
 // `repo` names the repository when one report holds several, where the same program id in two
 // repositories is two programs.
-export function stampFingerprints(findings, { root, repo = '' } = {}) {
-  const cache = new Map();
+export function stampFingerprints(findings, { root, repo = '', read = null } = {}) {
+  const cache = Object.assign(new Map(), { read });
   const seen = new Set();
   let shared = 0;
   for (const f of findings) {
@@ -188,8 +189,8 @@ export function stampFingerprints(findings, { root, repo = '' } = {}) {
 // Looser keys for one finding seen in two trees whose line was edited between them: `scope` is the
 // fingerprint without the line's text, and `route` names a data-flow finding by the statement its
 // trace starts at. Null where a finding has no route.
-export function pairingKeys(findings, { root, repo = '' } = {}) {
-  const cache = new Map();
+export function pairingKeys(findings, { root, repo = '', read = null } = {}) {
+  const cache = Object.assign(new Map(), { read });
   return findings.map((f) => {
     const { scope, subject } = partsOf(f, root, cache);
     const src = f.evidence === 'path' && f.related && f.related[0];

package/lib/kernel/source-tree.mjs

@@ -26,13 +26,16 @@
 //
 // An adapter that does not have a filesystem underneath it has to make that decision itself, and
 // `contains` is where it makes it. For a tree held in memory it is key membership. For a git
-// revision it is membership of the tree object, which cannot name a path outside itself. Anything
-// cleverer than those two deserves the six containment tests pointed at it before it ships:
-// test/sources.test.mjs:44, :63, :88, :101, :144 and test/review.test.mjs:114.
-import { readFileSync, realpathSync } from 'node:fs';
-import { dirname, resolve, sep } from 'node:path';
+// revision it is membership of the tree object, which cannot name a path outside itself. For a PDS
+// export it is membership of the members the walk found, each read with no link followed. Anything
+// cleverer than those deserves the six containment tests pointed at it before it ships:
+// test/sources.test.mjs:44, :63, :88, :101, :144 and test/review.test.mjs:114, as
+// test/git-tree.test.mjs and test/pds-export.test.mjs do.
+import { closeSync, constants, fstatSync, openSync, readFileSync, readSync, realpathSync } from 'node:fs';
+import { basename, dirname, relative, resolve, sep } from 'node:path';
 import { buildFileIndex, parseSource } from '../parser.mjs';
-import { readSource, relPath } from '../sources.mjs';
+import { classifyUnder, decodeSource, diskClassifier, heldClassifier, kindOfBytes, looksEbcdic, readSource, relPath } from '../sources.mjs';
+import { revisionBlobs, treeOf, treePathParts } from './git.mjs';
 
 // The shape every adapter answers to. Checked rather than documented, because an adapter missing a
 // method fails at the first rule set that happens to call it rather than at the boundary.
@@ -64,10 +67,14 @@ export function directoryTree(root, opts = {}) {
   const idx = buildFileIndex(root);
   const systemDirs = opts.systemDirs || [];
   const top = resolve(root);
+  const kindOf = diskClassifier();
+  classifyUnder(root, kindOf);
+  if (top !== root) classifyUnder(top, kindOf);
 
   return {
     kind: 'directory',
     root,
+    kindOf,
     // The index itself, for the two callers that need more than a list: the copybook set reads
     // copyDirs, and the inventory reports on unreadable directories and symlinks.
     index: idx,
@@ -92,40 +99,203 @@ export function directoryTree(root, opts = {}) {
   };
 }
 
-// A tree that was never on disk, for tests. `files` maps a path to its contents, as a string or a
-// Buffer. Paths are used exactly as given, so a test reads the way it writes.
-//
-// It cannot parse. parseSource resolves COPY statements against real directories and reads them
-// with readSource, so a program with copybooks needs a filesystem underneath it. A rule set that
-// only reads text - hidden, recon, vendor, jcl, build - works against this; one that parses does
-// not, and says so rather than parsing something unexpected.
-export function memoryTree(files, { root = '/memory' } = {}) {
-  const store = new Map(Object.entries(files).map(([p, v]) => [p, Buffer.isBuffer(v) ? v : Buffer.from(v, 'latin1')]));
+// A tree whose files are held rather than on disk. The parser finds copybooks in the index built
+// here and reads them through `readText`, so a COPY resolves to a held file or to a system copy
+// library outside the tree, never to a file on disk that happens to sit under this tree's root.
+function heldTree({ kind, root, store, rel, systemDirs = [], copyDirs = null, classify = heldClassifier,
+  symlinks = { followed: 0, outside: 0, broken: 0 }, unreadableDirs = [] }) {
+  const index = new Map();
+  const dirs = new Set(copyDirs || []);
+  for (const p of store.keys()) {
+    index.set(resolve(root, p).toLowerCase(), p);
+    if (!copyDirs && /\.(cpy|copy|inc|cbl|cob)$/i.test(p)) dirs.add(dirname(resolve(root, p)));
+  }
+  index.root = root;
+  const top = resolve(root);
+  const absent = (p) => Object.assign(new Error(`no such file in this tree: ${p}`), { code: 'ENOENT' });
+  const bytes = (p) => {
+    const b = store.get(p);
+    if (!b) throw absent(p);
+    return b;
+  };
+  const kindOf = classify(bytes);
+  classifyUnder(top, kindOf);
+  const text = (p) => decodeSource(bytes(p));
+  const readText = (p) => {
+    if (store.has(p)) return text(p).text;
+    const r = resolve(root, p);
+    if (r === top || r.startsWith(top + sep)) throw absent(p);
+    return readSource(p).text;
+  };
   return {
-    kind: 'memory',
+    kind,
     root,
-    index: { index: store, copyDirs: [], unreadableDirs: [], symlinks: { followed: 0, outside: 0, broken: 0 } },
+    kindOf,
+    index: { index, copyDirs: [...dirs].sort(), unreadableDirs, symlinks },
     list: () => [...store.keys()].sort(),
-    bytes: (p) => {
-      const b = store.get(p);
-      if (!b) throw Object.assign(new Error(`no such file in this tree: ${p}`), { code: 'ENOENT' });
-      return b;
-    },
-    text: (p) => {
-      const b = store.get(p);
-      if (!b) throw Object.assign(new Error(`no such file in this tree: ${p}`), { code: 'ENOENT' });
-      return { text: b.toString('latin1'), encoding: 'latin1' };
-    },
-    rel: (p) => String(p).replace(/\\/g, '/').replace(new RegExp('^' + root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '/?'), ''),
+    bytes,
+    text,
+    rel,
     // Containment by construction: a path this tree does not hold is a path outside it.
     contains: (p) => store.has(p),
-    parse: () => {
-      throw Object.assign(
-        new Error('a memory tree cannot parse: COPY resolution reads directories from disk'),
-        { code: 'ETREEPARSE' },
-      );
-    },
+    parse: (file, t) => parseSource(t ?? text(file).text, file, {
+      format: 'auto',
+      includeDirs: [...dirs].sort(),
+      fileIndex: index,
+      mainDir: dirname(resolve(root, file)),
+      copyFormat: 'auto',
+      systemDirs,
+      readText,
+    }),
+  };
+}
+
+// A tree that was never on disk, for tests. `files` maps a path to its contents, as a string or a
+// Buffer. Paths are used exactly as given, so a test reads the way it writes.
+export function memoryTree(files, { root = '/memory', systemDirs = [] } = {}) {
+  const store = new Map(Object.entries(files).map(([p, v]) => [p, Buffer.isBuffer(v) ? v : Buffer.from(v, 'latin1')]));
+  const rel = (p) => String(p).replace(/\\/g, '/').replace(new RegExp('^' + root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '/?'), '');
+  return heldTree({ kind: 'memory', root, store, rel, systemDirs });
+}
+
+// A git revision, read with plumbing into memory: nothing is written to disk. Its root is a path
+// beside the repository that does not exist, `<repo>@<tree>`, so every path it answers for is
+// absolute and none of them can be found on disk. The tree object is the containment: it cannot
+// name a path outside itself, and a path that would climb, name a drive or stream, or enter .git
+// is left out, as it is when a revision is written to disk. Links and submodules are not read.
+export function gitTree(repo, ref, opts = {}) {
+  const oid = treeOf(repo, ref);
+  const { blobs, links } = revisionBlobs(repo, oid, ref);
+  const root = `${resolve(repo)}@${oid.slice(0, 12)}`;
+  const store = new Map();
+  for (const b of blobs) {
+    const parts = treePathParts(b.path);
+    if (parts) store.set(resolve(root, ...parts), b.bytes);
+  }
+  const tree = heldTree({ kind: 'git', root, store, rel: (p) => relPath(root, p), systemDirs: opts.systemDirs || [],
+    symlinks: { followed: 0, outside: 0, broken: 0, notRead: links } });
+  return Object.assign(tree, { ref, oid });
+}
+
+// z/OSMF's record mode (zowe --record) writes each record after its length, four bytes big-endian,
+// and nothing between records. Read as that only when the lengths chain exactly to the end of the
+// file, and given back as lines; anything else is the file as it is.
+const MAX_LRECL = 32760;
+export function unframeRecords(buf) {
+  let records = 0;
+  for (let at = 0; at < buf.length; records++) {
+    if (buf.length - at < 4) return buf;
+    const n = buf.readUInt32BE(at);
+    if (n > MAX_LRECL || buf.length - at - 4 < n) return buf;
+    at += 4 + n;
+  }
+  if (!records || records * 4 === buf.length) return buf;
+  const lines = (end) => {
+    const out = Buffer.alloc(buf.length - records * 3);
+    let w = 0;
+    for (let at = 0; at < buf.length;) {
+      const n = buf.readUInt32BE(at);
+      w += buf.copy(out, w, at + 4, at + 4 + n);
+      out[w++] = end;
+      at += 4 + n;
+    }
+    return out;
   };
+  const ebcdic = lines(0x15);
+  return looksEbcdic(ebcdic) ? ebcdic : lines(0x0A);
+}
+
+// A member is read from the file the walk found. A link put in its place since, or a file that is
+// not a regular one, is refused rather than followed or waited on.
+function readMember(file) {
+  const fd = openSync(file, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
+  try {
+    const st = fstatSync(fd);
+    if (!st.isFile()) throw Object.assign(new Error(`not a regular file: ${file}`), { code: 'ENOTFILE' });
+    const buf = Buffer.alloc(st.size);
+    let n = 0;
+    while (n < st.size) {
+      const r = readSync(fd, buf, n, st.size - n, n);
+      if (!r) break;
+      n += r;
+    }
+    return unframeRecords(buf.subarray(0, n));
+  } finally { closeSync(fd); }
+}
+
+const QUALIFIER = /^[A-Z@#$][A-Z0-9@#$-]{0,7}$/;
+const MEMBER = /^([A-Z@#$][A-Z0-9@#$]{0,7})(?:\.[^.]*)?$/i;
+const MAX_LISTED = 8;
+
+// Partitioned data sets exported to a directory, a member to a file: what
+// `zowe zos-files download all-members` writes (ibmuser/new/cntl/member.txt, lower-cased unless
+// --preserve-original-letter-case, with whatever extension -e gave), or a directory named for the
+// data set with its members copied in. Each member is held as `DATA.SET.NAME/MEMBER` under a root
+// beside the export that does not exist, so a finding names the member as z/OS does however it was
+// downloaded, and nothing under that root is ever read from disk.
+//
+// The extension is the download's choice, not the language's, so members are classified by their
+// contents. A file whose directory is not a data set name or whose name is not a member name is not
+// read and is counted, as is a second file for a member already held. Every data set that holds
+// anything but JCL and maps is a copy library, searched in name order: the export does not say
+// what SYSLIB concatenated.
+export function pdsExportTree(dir, opts = {}) {
+  const idx = buildFileIndex(dir);
+  const top = realpathSync(dir);
+  const root = `${resolve(dir)}@pds`;
+  const origins = new Map();
+  const found = new Map();
+  const notMembers = [];
+  const duplicates = [];
+  const dataSets = new Set();
+  for (const file of [...idx.index.values()].sort()) {
+    const where = relative(dir, dirname(file)).split(sep).filter(Boolean);
+    const dsn = where.join('.').toUpperCase();
+    const member = MEMBER.exec(basename(file));
+    if (!member || dsn.length > 44 || (dsn && !dsn.split('.').every((q) => QUALIFIER.test(q)))) {
+      notMembers.push(relPath(dir, file));
+      continue;
+    }
+    const at = resolve(root, ...(dsn ? [dsn] : []), member[1].toUpperCase());
+    if (found.has(at)) { duplicates.push(`${relPath(dir, file)}: ${relPath(root, at)} is ${found.get(at)}`); continue; }
+    let real;
+    try { real = realpathSync(file); } catch { real = file; }
+    if (real !== top && !real.startsWith(top + sep)) { notMembers.push(relPath(dir, file)); continue; }
+    origins.set(at, real);
+    found.set(at, relPath(dir, file));
+    dataSets.add(dirname(at));
+  }
+  const kinds = new Map();
+  const byKind = {};
+  const copyDirs = new Set();
+  for (const [at, file] of origins) {
+    let kind = null;
+    try { kind = kindOfBytes(readMember(file)); } catch { kind = null; }
+    kinds.set(at, kind);
+    byKind[kind || 'other'] = (byKind[kind || 'other'] || 0) + 1;
+    if (kind !== 'jcl' && kind !== 'bms') copyDirs.add(dirname(at));
+  }
+  const store = {
+    keys: () => origins.keys(),
+    has: (p) => origins.has(p),
+    get: (p) => (origins.has(p) ? readMember(origins.get(p)) : undefined),
+  };
+  const tree = heldTree({
+    kind: 'pds-export', root, store, rel: (p) => relPath(root, p), systemDirs: opts.systemDirs || [],
+    copyDirs: [...copyDirs], classify: () => (p) => kinds.get(p) ?? null, symlinks: idx.symlinks,
+    unreadableDirs: idx.unreadableDirs.map((d) => resolve(root, relative(dir, d))),
+  });
+  return Object.assign(tree, {
+    dir,
+    origin: (p) => found.get(p) ?? null,
+    export: {
+      dataSets: dataSets.size,
+      members: origins.size,
+      byKind,
+      ...(notMembers.length ? { notMembers: notMembers.length, notMembersListed: notMembers.slice(0, MAX_LISTED) } : {}),
+      ...(duplicates.length ? { duplicates: duplicates.length, duplicatesListed: duplicates.slice(0, MAX_LISTED) } : {}),
+    },
+  });
 }
 
 // What a rule set calls. Given a tree it uses it; given none it builds the directory one, so every

package/lib/parser.mjs

@@ -634,7 +634,7 @@ function isProgramFile(p, ctx) {
   const seen = (ctx.programFiles ||= new Map());
   if (!seen.has(p)) {
     let text = '';
-    try { text = readSource(p).text; } catch { /* an unreadable candidate is not known to be a program */ }
+    try { text = ctx.readText(p); } catch { /* an unreadable candidate is not known to be a program */ }
     seen.set(p, /^[^*\n]{0,6}\s*PROGRAM-ID\s*\./im.test(text));
   }
   return seen.get(p);
@@ -787,7 +787,7 @@ function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
   if (!path) return [];
   if (stack.includes(path) || stack.length > 40) { record.status = 'recursive'; return []; }
   if (++ctx.inclusions > MAX_INCLUSIONS || ctx.copyTokens > MAX_COPY_TOKENS) { record.status = 'expansion-limit'; return []; }
-  const src = readSource(path).text;
+  const src = ctx.readText(path);
   // A copybook is read in the format in force where the COPY statement sits, which a >>SOURCE
   // directive earlier in the including file may have changed from the file's starting format.
   const fmt = detectFormat(src) === 'terminal' && ctx.copyFormat === 'auto' ? 'terminal' : (at.fmt || (ctx.copyFormat !== 'auto' ? ctx.copyFormat : (inheritedFormat || ctx.mainFormat)));
@@ -1848,7 +1848,7 @@ function collectCopybookDefines(src, format, ctx, depth) {
     const path = resolveCopy(name, null, ctx);
     if (!path || seen.has(path)) continue;
     seen.add(path);
-    const text = readSource(path).text;
+    const text = ctx.readText(path);
     const fmt = detectFormat(text) === 'terminal' && ctx.copyFormat === 'auto' ? 'terminal' : current;
     normalize(text, fmt, ctx.defines, ctx.std);
     collectCopybookDefines(text, fmt, ctx, depth + 1);
@@ -1880,7 +1880,9 @@ export function parseSource(src, file, opts = {}) {
   const scheme = BINARY_SIZE[opts.std || 'default'] || BINARY_SIZE.default;
   const ctx = { mainDir: opts.mainDir || dirname(file), includeDirs: opts.includeDirs || [], systemDirs: opts.systemDirs || [],
     fileIndex: opts.fileIndex || null, allowAbsoluteCopy: !!opts.allowAbsoluteCopy, cache: new Map(), copies: [], diags: [], copyFormat: opts.copyFormat || format, defines: new Map(), std: opts.std,
-    inclusions: 0, copyTokens: 0, replacedGrowth: 0, replacedChars: 0 };
+    inclusions: 0, copyTokens: 0, replacedGrowth: 0, replacedChars: 0,
+    // Where a copybook's text comes from: the disk, or the source tree the program was read from.
+    readText: opts.readText || ((p) => readSource(p).text) };
   collectCopybookDefines(src, format, ctx, 0);
   const norm = normalize(src, format, ctx.defines, opts.std);
   ctx.mainFormat = norm.finalFormat;

package/lib/revision.json

@@ -1 +1 @@
-{"commit":"54c9654b8c13bc22c8d5683b02551362d95f0d0a","tag":"v0.2.140"}
+{"commit":"f5995391b095f62da69d501ea7f6bd11a75bdc6e","tag":"v0.2.150"}

package/lib/sbom.mjs

@@ -102,7 +102,20 @@ export function sbom(root, opts = {}) {
   const idx = tree.index;
   const top = resolve(root);
   const within = (p) => resolve(p).startsWith(top + sep);
-  const refOf = (p) => (within(p) ? relPath(root, p) : `copylib:${basename(p)}`);
+  // A member outside the tree is named by its file name, and by its directory's digest as well where
+  // another library holds a member of the same name.
+  const copylibRefs = new Map();
+  const claimed = new Map();
+  const refOf = (p) => {
+    if (within(p)) return relPath(root, p);
+    if (!copylibRefs.has(p)) {
+      let ref = `copylib:${basename(p)}`;
+      if (claimed.has(ref) && claimed.get(ref) !== p) ref = `copylib:${basename(p)}#${sha256(dirname(resolve(p))).slice(0, 8)}`;
+      claimed.set(ref, p);
+      copylibRefs.set(p, ref);
+    }
+    return copylibRefs.get(p);
+  };
 
   const entries = [];
   const programIds = new Map();
@@ -110,17 +123,24 @@ export function sbom(root, opts = {}) {
   const procIds = new Map();
   for (const f of tree.list().filter(inScope(opts)).sort()) {
     let bytes;
-    let src;
-    try { bytes = readFileSync(f); src = readSource(f).text; } catch { continue; }
-    const kind = kindOf(f, src);
+    let src = null;
+    try { bytes = readFileSync(f); } catch { continue; }
+    try { src = readSource(f).text; } catch { /* its bytes are listed; its text could not be decoded */ }
+    const kind = src === null ? (isProgram(f) ? 'program' : isCopybook(f) ? 'copybook' : isJcl(f) ? 'jcl' : isBms(f) ? 'bms' : null) : kindOf(f, src);
     if (!kind) continue;
-    const entry = { file: f, ref: relPath(root, f), bytes, src, kind };
+    const entry = { file: f, ref: relPath(root, f), bytes, src: src ?? '', kind, ...(src === null ? { unparsed: true, undecodable: true } : {}) };
+    if (src === null) { entries.push(entry); continue; }
     if (kind === 'program') {
       try {
         const res = parseSource(src, f, { format: 'auto', includeDirs: idx.copyDirs, fileIndex: idx.index, mainDir: dirname(f), copyFormat: 'auto', systemDirs: opts.systemDirs || [] });
         entry.programs = res.programs;
         entry.copies = res.copies;
-        for (const p of res.programs) if (p.id) programIds.set(String(p.id).toUpperCase(), entry.ref);
+        for (const p of res.programs) {
+          if (!p.id) continue;
+          const id = String(p.id).toUpperCase();
+          if (!programIds.has(id)) programIds.set(id, []);
+          if (!programIds.get(id).includes(entry.ref)) programIds.get(id).push(entry.ref);
+        }
       } catch { entry.unparsed = true; }
     } else if (kind === 'jcl' || kind === 'proc') {
       try { entry.jcl = parseJcl(src, f); } catch { entry.unparsed = true; }
@@ -139,9 +159,10 @@ export function sbom(root, opts = {}) {
   const dependencies = new Map();
   const depend = (from, to) => { if (!dependencies.has(from)) dependencies.set(from, new Set()); dependencies.get(from).add(to); };
   // A program the estate does not hold is a component of its own, so an edge to it is not lost.
+  // A PROGRAM-ID two files hold is an edge to each: which one a run loads is the load library's order.
   const runs = (from, name) => {
-    const target = programIds.get(name);
-    if (target) { if (target !== from) depend(from, target); return; }
+    const targets = programIds.get(name);
+    if (targets) { for (const t of targets) if (t !== from) depend(from, t); return; }
     if (platformRoutine(name)) return;
     const ref = `program:${name}`;
     depend(from, ref);
@@ -153,9 +174,12 @@ export function sbom(root, opts = {}) {
   for (const e of entries) {
     const properties = [{ name: 'cobolwork:kind', value: e.kind }];
     if (e.unparsed) properties.push({ name: 'cobolwork:unparsed', value: 'true' });
+    if (e.undecodable) properties.push({ name: 'cobolwork:undecodable', value: 'true' });
     if (e.kind === 'program' && e.programs && e.programs.length) {
       const ids = e.programs.map((p) => p.id).filter(Boolean);
       if (ids.length) properties.push({ name: 'cobolwork:program-id', value: ids.join(',') });
+      const twins = [...new Set(ids.flatMap((id) => (programIds.get(String(id).toUpperCase()) || []).filter((r) => r !== e.ref)))].sort(order);
+      if (twins.length) properties.push({ name: 'cobolwork:program-id-also-in', value: twins.join(',') });
       properties.push({ name: 'cobolwork:format', value: detectFormat(e.src) });
       const options = optionCards(e.src).flatMap((c) => c.options);
       if (options.length) properties.push({ name: 'cobolwork:options', value: options.join(' ') });
@@ -166,7 +190,7 @@ export function sbom(root, opts = {}) {
         if (!within(c.path) && !components.has(ref)) {
           let b = null;
           try { b = readFileSync(c.path); } catch { /* read at parse time, gone since */ }
-          components.set(ref, { 'bom-ref': ref, type: 'file', name: ref, ...(b ? { hashes: [{ alg: 'SHA-256', content: sha256(b) }] } : {}), properties: [{ name: 'cobolwork:kind', value: 'copybook' }, { name: 'cobolwork:source', value: 'copy library' }] });
+          components.set(ref, { 'bom-ref': ref, type: 'file', name: ref, ...(b ? { hashes: [{ alg: 'SHA-256', content: sha256(b) }] } : {}), properties: [{ name: 'cobolwork:kind', value: 'copybook' }, { name: 'cobolwork:source', value: 'copy library' }, ...(b ? [] : [{ name: 'cobolwork:unread', value: 'true' }])] });
         }
       }
       const calls = e.programs.flatMap((p) => p.calls || []);

package/lib/scan.mjs

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { sortFindings, tally, evidenceMap, EVIDENCE, EXPLOITABILITY } from './kernel/findings.mjs';
 import { directoryTree } from './kernel/source-tree.mjs';
 import { REGISTRY, RULE_SETS, ALL_RULES, reportKey } from './kernel/registry.mjs';
@@ -13,6 +13,7 @@ import { rollup } from './sets/flow.mjs';
 import { loadSite } from './site.mjs';
 import { reachFeedPaths, loadReachFeed, reachResolver, stampReach, stampEffect } from './reach.mjs';
 import { stampExploitability, byVerdict, HANDLING, witnessFeedPaths, loadWitnessFeed, applyWitness } from './exploitability.mjs';
+import { executionFeedPaths, loadExecutionFeed, applyExecution } from './execution.mjs';
 
 import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 
@@ -133,7 +134,7 @@ function scanEach(root, opts) {
 // Reach and effect annotate rather than re-rank; where no fact is declared, a note says so rather
 // than the report reading as a clean bill.
 export function applyEstateFacts(findings, checked, root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const feedRoot = opts.feedRoot || root;
   const reachFeeds = reachFeedPaths(opts).map((p) => loadReachFeed(p, { root: feedRoot }));
   const reach = reachResolver({ feeds: reachFeeds, site });
@@ -151,6 +152,8 @@ export function applyEstateFacts(findings, checked, root, opts = {}) {
   const witnessed = applyWitness(findings, checked, witnessFeeds);
   if (witnessed.overruled.length) sortFindings(findings);
   const byExploitability = byVerdict(findings);
+  const executionFeeds = executionFeedPaths(opts).map((p) => loadExecutionFeed(p));
+  const { byExecution } = applyExecution(findings, root, executionFeeds);
   const driven = byExploitability ? byExploitability['attacker-driven'] : 0;
   const loaded = witnessFeeds.filter((f) => !f.problem);
   return {
@@ -164,6 +167,8 @@ export function applyEstateFacts(findings, checked, root, opts = {}) {
     ...(byExploitability?.exploitable || byExploitability?.confirmed ? { handling: HANDLING } : {}),
     ...(loaded.length ? { witnessFeeds: loaded.map((f) => ({ file: f.file, witness: f.witness, recorded: f.recorded })), byWitness: witnessed.byWitness } : {}),
     ...(witnessFeeds.some((f) => f.problem || f.refused.length) ? { witnessFeedProblems: witnessFeeds.flatMap((f) => (f.problem ? [`${f.file}: ${f.problem}`] : f.refused.map((r) => `${f.file}: ${r.why}`))) } : {}),
+    ...(byExecution ? { executionFeeds: executionFeeds.filter((f) => !f.problem).map((f) => ({ file: f.file, programs: f.programs.size })), byExecution } : {}),
+    ...(executionFeeds.some((f) => f.problem) ? { executionFeedProblems: executionFeeds.filter((f) => f.problem).map((f) => `${f.file}: ${f.problem}`) } : {}),
     // A result no path finding carries: the code changed since the test, or the route is gone.
     ...(witnessed.unmatched.length ? { witnessUnmatched: witnessed.unmatched } : {}),
     // Routes the reading refuted that the estate reproduced: the check model was wrong about them.
@@ -234,10 +239,12 @@ export function scanAll(root, opts = {}) {
   const startedBy = parts.find((p) => p.tool === 'cobolwork-flow')?.startedBy || {};
   for (const p of parts) for (const f of p.findings) findings.push(!f.startedBy && f.program && startedBy[f.program] ? { ...f, startedBy: startedBy[f.program] } : { ...f });
   sortFindings(findings);
-  const identity = stampFingerprints(findings, { root, repo: opts.repoName || '' });
+  // A tree held in memory is read for the fingerprint's line text as it was for the finding.
+  const read = tree.kind === 'directory' ? null : (path) => tree.text(resolve(tree.root, path)).text;
+  const identity = stampFingerprints(findings, { root, repo: opts.repoName || '', read });
   // Routes a check stops keep their identity too, so a finding a fix clears can be found again here.
   const checked = parts.find((p) => p.tool === 'cobolwork-flow')?.checked || [];
-  stampFingerprints(checked, { root, repo: opts.repoName || '' });
+  stampFingerprints(checked, { root, repo: opts.repoName || '', read });
 
   const estate = applyEstateFacts(findings, checked, root, opts);
 

package/lib/sets/build.mjs

@@ -132,7 +132,7 @@ function withFeeds(coverage, feeds) {
 }
 
 export function scanBuild(root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts)).filter(isBuildFile);
   const findings = [];

package/lib/sets/cics.mjs

@@ -136,6 +136,31 @@ function execOptions(exec) {
 
 const subtreeRead = (x) => x.directRefs > 0 || x.children.some(subtreeRead);
 
+// Whether EIBCALEN bounds what the program reads: a condition tests it, or a field it was moved or
+// computed into; it is the start or length of a reference modification, or what an OCCURS DEPENDING
+// ON counts by. A CALL handed it, or the whole EIB, may test it where this program cannot see.
+// Named anywhere else - logged, passed on to a LINK - it checks nothing.
+function testsLength(p) {
+  // A statement's own tokens, since its sources leave out what a condition or an expression wraps
+  // in parentheses: IF (EIBCALEN > 0).
+  const tokens = (p.proc && p.proc.tokens) || [];
+  const own = (st) => (st.at != null ? tokens.slice(st.at + 1, st.end) : st.sources || []);
+  const carriers = new Set(['EIBCALEN']);
+  for (let grew = true; grew;) {
+    grew = false;
+    for (const st of p.statements || []) {
+      if (!own(st).some((t) => carriers.has(t.u))) continue;
+      for (const t of st.targets || []) if (!carriers.has(t.u)) { carriers.add(t.u); grew = true; }
+    }
+  }
+  const carries = (toks) => (toks || []).some((t) => t && carriers.has(t.u));
+  return p.items.some((i) => i.dependingOn === 'EIBCALEN') || (p.statements || []).some((st) =>
+    ((st.verb === 'IF' || st.verb === 'EVALUATE' || st.verb === 'WHEN') && carries(own(st)))
+    || (st.loops || []).some((l) => carries(l.until))
+    || (st.indexes || []).some((x) => x.kind.startsWith('refmod-') && carries([x.tok]))
+    || (st.verb === 'CALL' && (st.sources || []).some((t) => t.u === 'EIBCALEN' || t.u === 'DFHEIBLK')));
+}
+
 // Commands whose failure a program ordinarily lets pass: removing a temporary-storage queue that may
 // not exist, a line written to a transient-data log, the clock, a screen sent to a terminal that may
 // have gone, an ASSIGN, and the RETURN that ends the task anyway.
@@ -191,7 +216,6 @@ export function scanCics(root, opts = {}) {
   // The whole of what the second pass reads, computed once, while the tree is still here.
   function summarise(p, path) {
     const isSignOn = [...new Set([...p.refs.map(x => x.tok.u), ...p.execs.flatMap(e => e.toks.filter(t => t.t === 'word').map(t => t.u))])].some((w) => PASSWORD.test(w));
-    const words = new Set([...p.refs.map(x => x.tok.u), ...p.execs.flatMap(e => e.toks.filter(t => t.t === 'word').map(t => t.u))]);
     const commarea = p.items.find(i => i.name === 'DFHCOMMAREA' && i.section === 'LINKAGE');
     // A variable naming the program usually holds one literal from its VALUE clause.
     const valueOf = (name) => {
@@ -236,7 +260,7 @@ export function scanCics(root, opts = {}) {
       ownCics: p.execs.some((e) => e.kind === 'CICS'),
       pointsItself,
       commarea: commarea ? { ...where(commarea, path), size: commarea.size, read: subtreeRead(commarea) } : null,
-      checksLength: words.has('EIBCALEN'),
+      checksLength: testsLength(p),
       transfers,
       signOnBypasses: isSignOn ? signOnBypasses(p, path) : [],
       // Gated the same way: a program with no password-named field is not answering a sign-on,
@@ -393,7 +417,7 @@ export function scanCics(root, opts = {}) {
         findings.push({
           rule: 'cics-commarea-without-length-check', path: s.commarea.path, line: s.commarea.line, program: s.id,
           ...(covered ? { sev: 'low' } : {}),
-          detail: `${s.id} reads DFHCOMMAREA (${s.commarea.size} bytes declared) and never references EIBCALEN, so a shorter or absent communication area is read as if it were whole${covered ? `; every caller in the tree passes at least ${s.commarea.size} bytes, and no transaction starts it` : ''}`,
+          detail: `${s.id} reads DFHCOMMAREA (${s.commarea.size} bytes declared) and never tests EIBCALEN, so a shorter or absent communication area is read as if it were whole${covered ? `; every caller in the tree passes at least ${s.commarea.size} bytes, and no transaction starts it` : ''}`,
         });
       }
     }

package/lib/sets/copybook.mjs

@@ -7,7 +7,7 @@ import { finish } from '../kernel/findings.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
-import { cobolCard } from '../card.mjs';
+import { cobolCard } from '../cards.mjs';
 
 // A copybook is resolved by name along a search path: the program's own directory first, then the
 // include directories, then the system library. Two files answering to one name mean the record