@portll/cobolwork 0.2.140 → 0.2.150

package/lib/sets/hidden.mjs

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { detectFormat, expandTabs } from '../parser.mjs';
-import { cobolCard } from '../card.mjs';
+import { cobolCard } from '../cards.mjs';
 import { decodeEbcdic, inScope, isJcl, isSource, looksEbcdic, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread } from '../kernel/source-tree.mjs';

package/lib/sets/jcl.mjs

@@ -237,7 +237,7 @@ export function scanJcl(root, opts = {}) {
   // A vendor product supplies its own utilities. An estate that has loaded the pack for a product
   // is telling us those programs exist, so reporting them as undefined would be a false positive
   // created by the customer having told us the truth.
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const { loaded } = loadPacks(opts.packs || site.vendorPacks || [], { allowUnvalidated: true });
   const vendorPrograms = programsOf(loaded);
 

package/lib/sets/priv.mjs

@@ -102,7 +102,7 @@ function under(dsn, prefixes) {
 }
 
 export function scanPriv(root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts));
   const findings = [];

package/lib/sets/recon.mjs

@@ -17,7 +17,7 @@ import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
 import { loadSite, classifyPath, productionQualifierOf, SITE_FILE } from '../site.mjs';
 import { parseJcl, dispositionOf } from '../jcl.mjs';
-import { statementCard } from '../card.mjs';
+import { statementCard } from '../cards.mjs';
 
 
 // The production-dataset rules claim more than the name rule: that the job acts on production, not
@@ -161,7 +161,7 @@ function productionTouches(job, qualifiers) {
 }
 
 export function scanRecon(root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts))
     .filter((f) => isJcl(f) || isProgram(f) || isCopybook(f));

package/lib/sets/semantics.mjs

@@ -216,7 +216,7 @@ export function scanSemantics(root, opts = {}) {
     truncOptPrograms: 0, truncUnknownPrograms: 0, arithExtendPrograms: 0, beyondEnterprisePrograms: 0, computesRead: 0, computesUnread: 0,
     rangesRead: 0, collatingDeclaredPrograms: 0,
   };
-  const site = loadSite(root, opts.site || null).compilerOptions || [];
+  const site = loadSite(root, opts.site || null, opts.tree).compilerOptions || [];
   const parsedJcl = [];
   for (const f of all.filter(isJcl)) {
     try { parsedJcl.push({ ...parseJcl(tree.text(f).text, f), file: relPath(root, f) }); } catch { /* the jcl set reports it */ }

package/lib/sets/vendor.mjs

@@ -26,7 +26,7 @@ export const VENDOR_RULES = {
 const evidenceOf = (r) => (r.severity === 'info' ? { evidence: 'context' } : {});
 
 export function scanVendor(root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const wanted = opts.packs || site.vendorPacks || [];
   const { loaded, refused, problems, caveats } = loadPacks(wanted, {
     allowUnvalidated: !!(opts.allowUnvalidatedPacks ?? site.allowUnvalidatedPacks),

package/lib/site.mjs

@@ -9,7 +9,7 @@
 // It is also the only part of cobolwork that is inherently per-customer and inherently recurring:
 // conventions drift, environments are renamed, estates are acquired.
 import { readFileSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { resolvesInside } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
 
@@ -43,14 +43,24 @@ const EMPTY = {
 // A configuration that is absent is not a configuration that says "nothing is production". The
 // difference is the whole point: without it the recon rules have not looked, and a scan must say
 // it has not looked rather than report a clean result.
-export function loadSite(root, explicit = null) {
-  const path = explicit || join(root, SITE_FILE);
-  if (!existsSync(path)) return { ...EMPTY, path };
-  // A site file in the tree is not read through a link out of it; one the caller names is.
-  if (!explicit && !resolvesInside(root, path)) return { ...EMPTY, path, problems: [`${SITE_FILE} is a link that leads outside the tree, so it was not read`] };
+// `tree` is the scan's source tree: one held in memory (a git revision) supplies the site file
+// itself, and on disk the file is read from `root`.
+export function loadSite(root, explicit = null, tree = null) {
+  const held = !explicit && tree && tree.kind !== 'directory';
+  const path = explicit || (held ? resolve(tree.root, SITE_FILE) : join(root, SITE_FILE));
+  let read;
+  if (held) {
+    if (!tree.contains(path)) return { ...EMPTY, path };
+    read = () => tree.bytes(path).toString('utf8');
+  } else {
+    if (!existsSync(path)) return { ...EMPTY, path };
+    // A site file in the tree is not read through a link out of it; one the caller names is.
+    if (!explicit && !resolvesInside(root, path)) return { ...EMPTY, path, problems: [`${SITE_FILE} is a link that leads outside the tree, so it was not read`] };
+    read = () => readFileSync(path, 'utf8');
+  }
 
   let raw;
-  try { raw = JSON.parse(readFileSync(path, 'utf8')); } catch (e) {
+  try { raw = JSON.parse(read()); } catch (e) {
     return { ...EMPTY, path, problems: [`${SITE_FILE} is not readable as JSON: ${printable(e.message, 120)}`] };
   }
   if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return { ...EMPTY, path, problems: [`${SITE_FILE} is not a JSON object`] };

package/lib/sources.mjs

@@ -23,41 +23,90 @@ const has = (list) => (p) => list.includes(extname(p).toLowerCase());
 // of it and reporting a clean zero, which is the failure this project exists to refuse.
 //
 // So a file with no extension is classified by what is in it. Only the first few kilobytes are
-// read, only files without an extension are sniffed, and the answer is cached: this runs over
-// every file in the tree and must not cost a parse.
+// read and only files without an extension are sniffed. The tree a path belongs to classifies it
+// and keeps the answer for as long as the tree lives: this runs over every file in the tree and
+// must not cost a parse, and a cache that outlives the scan grows by one entry per file for the
+// life of the process.
 const SNIFF_BYTES = 4096;
-const sniffed = new Map();
 
 // Names that are certainly not source, so the commonest extensionless files cost nothing.
 const NOT_SOURCE = /^(LICEN[CS]E|README|MAKEFILE|DOCKERFILE|CHANGELOG|NOTICE|AUTHORS|CONTRIBUTING|CODEOWNERS|VERSION|COPYING|Jenkinsfile|Procfile|\..*)$/i;
+const notSource = (path) => NOT_SOURCE.test(path.split(/[\\/]/).pop());
+
+function kindOfHead(head) {
+  // An EBCDIC member is decoded before it is read, or every test below sees mojibake.
+  const text = looksEbcdic(head) ? decodeEbcdic(head) : head.toString('latin1');
+  // A NUL byte means binary. Tested on the bytes rather than with a literal, because a literal
+  // NUL in source is exactly the character an editor or a patch is most likely to eat.
+  if (head.includes(0)) return null;
+  if (/^\s*\/\/\S*\s+(?:JOB|EXEC|PROC|DD)\b/m.test(text)) return 'jcl';
+  if (/\bIDENTIFICATION\s+DIVISION|\bPROGRAM-ID\s*\./i.test(text)) return 'program';
+  // A copybook is a data description with no divisions: level numbers and picture clauses.
+  if (/^[^*\n]{0,10}\s*\d\d\s+[A-Z][A-Z0-9-]*\s+(?:PIC|PICTURE|OCCURS|REDEFINES|USAGE)\b/im.test(text)) return 'copybook';
+  // A BMS map opens its mapset with DFHMSD in the operation field. Tested last, so nothing that
+  // classified before classifies differently.
+  if (/^[^*\s]*[ \t]+DFHMSD(?=\s|$)/im.test(text)) return 'bms';
+  return null;
+}
 
-export function sniffKind(path) {
-  if (sniffed.has(path)) return sniffed.get(path);
-  let kind = null;
-  const name = path.split(/[\\/]/).pop();
-  if (!NOT_SOURCE.test(name)) {
-    try {
-      const fd = openSync(path, 'r');
-      const buf = Buffer.alloc(SNIFF_BYTES);
-      let n;
-      try { n = readSync(fd, buf, 0, SNIFF_BYTES, 0); } finally { closeSync(fd); }
-      const head = buf.subarray(0, n);
-      // An EBCDIC member is decoded before it is read, or every test below sees mojibake.
-      const text = looksEbcdic(head) ? decodeEbcdic(head) : head.toString('latin1');
-      // A NUL byte means binary. Tested on the bytes rather than with a literal, because a literal
-      // NUL in source is exactly the character an editor or a patch is most likely to eat.
-      if (head.includes(0)) kind = null;
-      else if (/^\s*\/\/\S*\s+(?:JOB|EXEC|PROC|DD)\b/m.test(text)) kind = 'jcl';
-      else if (/\bIDENTIFICATION\s+DIVISION|\bPROGRAM-ID\s*\./i.test(text)) kind = 'program';
-      // A copybook is a data description with no divisions: level numbers and picture clauses.
-      else if (/^[^*\n]{0,10}\s*\d\d\s+[A-Z][A-Z0-9-]*\s+(?:PIC|PICTURE|OCCURS|REDEFINES|USAGE)\b/im.test(text)) kind = 'copybook';
-      // A BMS map opens its mapset with DFHMSD in the operation field. Tested last, so nothing that
-      // classified before classifies differently.
-      else if (/^[^*\s]*[ \t]+DFHMSD(?=\s|$)/im.test(text)) kind = 'bms';
-    } catch { kind = null; }
+// What a member's bytes say it is, for a tree that holds them.
+export const kindOfBytes = (buf) => kindOfHead(buf.subarray(0, SNIFF_BYTES));
+
+function sniffDisk(path) {
+  if (notSource(path)) return null;
+  try {
+    const fd = openSync(path, 'r');
+    const buf = Buffer.alloc(SNIFF_BYTES);
+    let n;
+    try { n = readSync(fd, buf, 0, SNIFF_BYTES, 0); } finally { closeSync(fd); }
+    return kindOfHead(buf.subarray(0, n));
+  } catch { return null; }
+}
+
+// A tree's classifier for the paths under its root, remembered for as long as the tree lives.
+const classifiers = new Map();
+
+export const classifyUnder = (root, kindOf) => classifiers.set(root.replace(/[\\/]+$/, ''), new WeakRef(kindOf));
+
+// The classifier of the deepest live tree whose root the path sits under.
+function classifierFor(path) {
+  let best = null;
+  let depth = -1;
+  for (const [root, ref] of classifiers) {
+    if (root.length <= depth || !path.startsWith(root + sep)) continue;
+    const kindOf = ref.deref();
+    if (!kindOf) { classifiers.delete(root); continue; }
+    best = kindOf;
+    depth = root.length;
   }
-  sniffed.set(path, kind);
-  return kind;
+  return best;
+}
+
+// A directory tree's classifier: sniffed from disk once per path.
+export function diskClassifier() {
+  const seen = new Map();
+  return (path) => {
+    if (!seen.has(path)) seen.set(path, sniffDisk(path));
+    return seen.get(path);
+  };
+}
+
+// A held tree's classifier: sniffed from what the tree holds, never from disk.
+export function heldClassifier(bytesOf) {
+  const seen = new Map();
+  return (path) => {
+    if (!seen.has(path)) {
+      let kind = null;
+      if (!notSource(path)) try { kind = kindOfBytes(bytesOf(path)); } catch { kind = null; }
+      seen.set(path, kind);
+    }
+    return seen.get(path);
+  };
+}
+
+export function sniffKind(path) {
+  const kindOf = classifierFor(path);
+  return kindOf ? kindOf(path) : sniffDisk(path);
 }
 
 // Extension first, because it is free and it is right whenever it is present.
@@ -72,13 +121,6 @@ export const isSource = (p) => (extname(p)
   ? [...PROGRAM_EXT, ...COPY_EXT, ...JCL_EXT].includes(extname(p).toLowerCase())
   : ![null, 'bms'].includes(sniffKind(p)));
 
-// For a report that wants to say how much of what it read had no extension to go on.
-export const sniffedCount = () => {
-  let n = 0;
-  for (const v of sniffed.values()) if (v) n++;
-  return n;
-};
-
 // `opts.only` is about rule sets; this is about files. A rule set given an allow list reads the
 // files on it and no others.
 //
@@ -159,8 +201,10 @@ export function decodeEbcdic(buf) {
 
 // Every rule set reads source through here, so an EBCDIC member is decoded once, the same way,
 // and reported as EBCDIC instead of silently parsing as noise.
-export function readSource(path) {
-  const buf = readFileSync(path);
+export const readSource = (path) => decodeSource(readFileSync(path));
+
+// The same decoding for bytes that did not come from a file: a blob read out of a git revision.
+export function decodeSource(buf) {
   if (looksEbcdic(buf)) return { text: decodeEbcdic(buf), encoding: 'ebcdic' };
   return { text: buf.toString('latin1'), encoding: 'latin1' };
 }

package/lib/utilities.mjs

@@ -9,7 +9,7 @@
 // then only the ones its DD names decide on their own.
 
 import { ftpSession } from './ftp.mjs';
-import { statementCard } from './card.mjs';
+import { statementCard } from './cards.mjs';
 
 const upper = (s) => String(s).toUpperCase();
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@portll/cobolwork",
-  "version": "0.2.140",
+  "version": "0.2.150",
   "description": "COBOL, JCL and CICS security analysis with zero runtime dependencies: reference-format parser, cross-program data flow, mainframe credential rules. Plugs into commitwork.",
   "license": "AGPL-3.0-or-later",
   "author": "Portll <john@portll.net>",