@portll/cobolwork 0.2.117 → 0.2.150

package/lib/sbom.mjs

@@ -10,6 +10,7 @@ import { parseJcl } from './jcl.mjs';
 import { optionCards } from './options.mjs';
 import { inScope, isBms, isCopybook, isJcl, isProgram, readSource, relPath } from './sources.mjs';
 import { TOOL_VERSION } from './version.mjs';
+import { revisionOf } from './revision.mjs';
 import { treeFor } from './kernel/source-tree.mjs';
 
 // The namespace of cobolwork's version 5 UUIDs (RFC 4122 §4.3), fixed so a serial number repeats.
@@ -26,6 +27,50 @@ function uuidv5(name) {
 const sha256 = (b) => createHash('sha256').update(b).digest('hex');
 const order = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
 
+// A routine a platform provides, which its platform component already stands for.
+const platformRoutine = (n) => n === 'CBLTDLI' || n === 'AIBTDLI' || /^MQ[A-Z0-9]{2,}$/.test(n) || /^CEE[A-Z0-9]+$/.test(n);
+
+// IBM programs a job step runs rather than one the estate builds.
+const SYSTEM_PROGRAMS = new Set(['ADRDSSU', 'AMASPZAP', 'ASMA90', 'BPXBATCH', 'DFHCSDUP', 'DFHEAP1$', 'DFHECP1$', 'DFSRRC00', 'DFSORT', 'DSNTEP2', 'DSNTEP4',
+  'DSNHPC', 'DSNTIAD', 'DSNTIAUL', 'DSNUTILB', 'FTP', 'HEWL', 'IEWBLINK', 'IEWL', 'ICEGENER', 'ICEMAN', 'ICETOOL', 'IDCAMS', 'IEBCOMPR',
+  'IEBCOPY', 'IEBDG', 'IEBGENER', 'IEBUPDTE', 'IEFBR14', 'IEHLIST', 'IEHPROGM', 'IGYCRCTL', 'IKJEFT01', 'IKJEFT1A', 'IKJEFT1B',
+  'SDSF', 'SORT']);
+const TSO_BATCH = new Set(['IKJEFT01', 'IKJEFT1A', 'IKJEFT1B']);
+// The IMS region types whose PARM names the application program second: PARM='BMP,PGM,PSB'.
+const IMS_REGIONS = new Set(['BMP', 'DLI', 'DBB', 'IFP', 'JBP', 'JMP']);
+
+// What a step runs beyond its PGM: the program a TSO batch step's DSN RUN names, or an IMS region's
+// application program.
+function programsRunBy(step) {
+  const pgm = String(step.pgm || '').toUpperCase();
+  const out = [];
+  if (TSO_BATCH.has(pgm)) {
+    for (const dd of step.dds || []) {
+      if (dd.name !== 'SYSTSIN') continue;
+      const text = (dd.inStream || []).map((l) => l.text).join(' ');
+      for (const m of text.matchAll(/\bRUN\s+PROG(?:RAM)?\s*\(\s*([A-Z0-9#@$]{1,8})\s*\)/gi)) out.push(m[1].toUpperCase());
+    }
+  } else if (pgm === 'DFSRRC00' && step.parm) {
+    const [region, app] = String(step.parm).replace(/^['(]|[')]$/g, '').split(',').map((x) => x.trim().toUpperCase());
+    if (IMS_REGIONS.has(region) && /^[A-Z0-9#@$]{1,8}$/.test(app || '')) out.push(app);
+  }
+  return out;
+}
+
+// The literal operand of a CICS option, `PROGRAM('X')`; the field's name where a field is given.
+function cicsOperand(toks, option) {
+  const at = toks.findIndex((t, i) => t.t === 'word' && t.u === option && toks[i + 1] && toks[i + 1].v === '(');
+  if (at < 0) return null;
+  const inner = [];
+  for (let i = at + 2, depth = 1; i < toks.length; i++) {
+    if (toks[i].v === '(') depth++;
+    else if (toks[i].v === ')' && --depth === 0) break;
+    inner.push(toks[i]);
+  }
+  if (inner.length === 1 && inner[0].t === 'lit') return { literal: String(inner[0].v).trim().toUpperCase() };
+  return inner.length ? { field: inner.map((t) => t.v).join('') } : null;
+}
+
 function platformsOf(src, calls) {
   const s = new Set();
   if (/EXEC\s+CICS\b/i.test(src)) s.add('CICS');
@@ -35,7 +80,7 @@ function platformsOf(src, calls) {
     if (c.kind !== 'L') continue;
     const n = String(c.name).toUpperCase();
     if (n === 'CBLTDLI' || n === 'AIBTDLI') s.add('IMS');
-    if (/^MQ[A-Z]{2,}$/.test(n)) s.add('MQ');
+    if (/^MQ[A-Z0-9]{2,}$/.test(n)) s.add('MQ');
     if (/^CEE[A-Z0-9]+$/.test(n)) s.add('Language Environment');
   }
   return [...s];
@@ -57,26 +102,55 @@ export function sbom(root, opts = {}) {
   const idx = tree.index;
   const top = resolve(root);
   const within = (p) => resolve(p).startsWith(top + sep);
-  const refOf = (p) => (within(p) ? relPath(root, p) : `copylib:${basename(p)}`);
+  // A member outside the tree is named by its file name, and by its directory's digest as well where
+  // another library holds a member of the same name.
+  const copylibRefs = new Map();
+  const claimed = new Map();
+  const refOf = (p) => {
+    if (within(p)) return relPath(root, p);
+    if (!copylibRefs.has(p)) {
+      let ref = `copylib:${basename(p)}`;
+      if (claimed.has(ref) && claimed.get(ref) !== p) ref = `copylib:${basename(p)}#${sha256(dirname(resolve(p))).slice(0, 8)}`;
+      claimed.set(ref, p);
+      copylibRefs.set(p, ref);
+    }
+    return copylibRefs.get(p);
+  };
 
   const entries = [];
   const programIds = new Map();
+  const mapsets = new Map();
+  const procIds = new Map();
   for (const f of tree.list().filter(inScope(opts)).sort()) {
     let bytes;
-    let src;
-    try { bytes = readFileSync(f); src = readSource(f).text; } catch { continue; }
-    const kind = kindOf(f, src);
+    let src = null;
+    try { bytes = readFileSync(f); } catch { continue; }
+    try { src = readSource(f).text; } catch { /* its bytes are listed; its text could not be decoded */ }
+    const kind = src === null ? (isProgram(f) ? 'program' : isCopybook(f) ? 'copybook' : isJcl(f) ? 'jcl' : isBms(f) ? 'bms' : null) : kindOf(f, src);
     if (!kind) continue;
-    const entry = { file: f, ref: relPath(root, f), bytes, src, kind };
+    const entry = { file: f, ref: relPath(root, f), bytes, src: src ?? '', kind, ...(src === null ? { unparsed: true, undecodable: true } : {}) };
+    if (src === null) { entries.push(entry); continue; }
     if (kind === 'program') {
       try {
         const res = parseSource(src, f, { format: 'auto', includeDirs: idx.copyDirs, fileIndex: idx.index, mainDir: dirname(f), copyFormat: 'auto', systemDirs: opts.systemDirs || [] });
         entry.programs = res.programs;
         entry.copies = res.copies;
-        for (const p of res.programs) if (p.id) programIds.set(String(p.id).toUpperCase(), entry.ref);
+        for (const p of res.programs) {
+          if (!p.id) continue;
+          const id = String(p.id).toUpperCase();
+          if (!programIds.has(id)) programIds.set(id, []);
+          if (!programIds.get(id).includes(entry.ref)) programIds.get(id).push(entry.ref);
+        }
       } catch { entry.unparsed = true; }
     } else if (kind === 'jcl' || kind === 'proc') {
       try { entry.jcl = parseJcl(src, f); } catch { entry.unparsed = true; }
+      if (kind === 'proc') {
+        const m = /^\/\/([A-Z0-9#@$]{1,8})\s+PROC\b/im.exec(src);
+        procIds.set((m ? m[1] : basename(f, extname(f))).toUpperCase(), entry.ref);
+      }
+    } else if (kind === 'bms') {
+      const m = /^([A-Z0-9#@$]{1,8})\s+DFHMSD\b/im.exec(src);
+      mapsets.set((m ? m[1] : basename(f, extname(f))).toUpperCase(), entry.ref);
     }
     entries.push(entry);
   }
@@ -84,13 +158,28 @@ export function sbom(root, opts = {}) {
   const components = new Map();
   const dependencies = new Map();
   const depend = (from, to) => { if (!dependencies.has(from)) dependencies.set(from, new Set()); dependencies.get(from).add(to); };
+  // A program the estate does not hold is a component of its own, so an edge to it is not lost.
+  // A PROGRAM-ID two files hold is an edge to each: which one a run loads is the load library's order.
+  const runs = (from, name) => {
+    const targets = programIds.get(name);
+    if (targets) { for (const t of targets) if (t !== from) depend(from, t); return; }
+    if (platformRoutine(name)) return;
+    const ref = `program:${name}`;
+    depend(from, ref);
+    if (!components.has(ref)) {
+      components.set(ref, { 'bom-ref': ref, type: 'application', name, properties: [{ name: 'cobolwork:kind', value: SYSTEM_PROGRAMS.has(name) ? 'system-program' : 'external-program' }] });
+    }
+  };
 
   for (const e of entries) {
     const properties = [{ name: 'cobolwork:kind', value: e.kind }];
     if (e.unparsed) properties.push({ name: 'cobolwork:unparsed', value: 'true' });
+    if (e.undecodable) properties.push({ name: 'cobolwork:undecodable', value: 'true' });
     if (e.kind === 'program' && e.programs && e.programs.length) {
       const ids = e.programs.map((p) => p.id).filter(Boolean);
       if (ids.length) properties.push({ name: 'cobolwork:program-id', value: ids.join(',') });
+      const twins = [...new Set(ids.flatMap((id) => (programIds.get(String(id).toUpperCase()) || []).filter((r) => r !== e.ref)))].sort(order);
+      if (twins.length) properties.push({ name: 'cobolwork:program-id-also-in', value: twins.join(',') });
       properties.push({ name: 'cobolwork:format', value: detectFormat(e.src) });
       const options = optionCards(e.src).flatMap((c) => c.options);
       if (options.length) properties.push({ name: 'cobolwork:options', value: options.join(' ') });
@@ -101,15 +190,26 @@ export function sbom(root, opts = {}) {
         if (!within(c.path) && !components.has(ref)) {
           let b = null;
           try { b = readFileSync(c.path); } catch { /* read at parse time, gone since */ }
-          components.set(ref, { 'bom-ref': ref, type: 'file', name: ref, ...(b ? { hashes: [{ alg: 'SHA-256', content: sha256(b) }] } : {}), properties: [{ name: 'cobolwork:kind', value: 'copybook' }, { name: 'cobolwork:source', value: 'copy library' }] });
+          components.set(ref, { 'bom-ref': ref, type: 'file', name: ref, ...(b ? { hashes: [{ alg: 'SHA-256', content: sha256(b) }] } : {}), properties: [{ name: 'cobolwork:kind', value: 'copybook' }, { name: 'cobolwork:source', value: 'copy library' }, ...(b ? [] : [{ name: 'cobolwork:unread', value: 'true' }])] });
         }
       }
       const calls = e.programs.flatMap((p) => p.calls || []);
       const unresolved = new Set();
       for (const c of calls) {
         if (c.kind !== 'L') { unresolved.add(String(c.name)); continue; }
-        const target = programIds.get(String(c.name).toUpperCase());
-        if (target && target !== e.ref) depend(e.ref, target);
+        runs(e.ref, String(c.name).toUpperCase());
+      }
+      for (const x of e.programs.flatMap((p) => p.execs || [])) {
+        if (x.kind !== 'CICS') continue;
+        const words = x.toks.filter((t) => t.t === 'word').map((t) => t.u);
+        if (words[0] === 'LINK' || words[0] === 'XCTL') {
+          const op = cicsOperand(x.toks, 'PROGRAM');
+          if (op && op.literal) runs(e.ref, op.literal); else if (op) unresolved.add(op.field);
+        } else if ((words[0] === 'SEND' || words[0] === 'RECEIVE') && words[1] === 'MAP') {
+          const op = cicsOperand(x.toks, 'MAPSET') || cicsOperand(x.toks, 'MAP');
+          const target = op && op.literal ? mapsets.get(op.literal) : null;
+          if (target) depend(e.ref, target);
+        }
       }
       if (unresolved.size) properties.push({ name: 'cobolwork:unresolved-call', value: [...unresolved].sort(order).join(',') });
       for (const p of platformsOf(e.src, calls)) {
@@ -119,10 +219,17 @@ export function sbom(root, opts = {}) {
       }
     }
     if (e.jcl) {
+      const inStream = new Set((e.jcl.procs || []).map((p) => String(p.name || '').toUpperCase()));
+      const missing = new Set();
       for (const step of e.jcl.steps || []) {
-        const target = step.pgm ? programIds.get(String(step.pgm).toUpperCase()) : null;
-        if (target) depend(e.ref, target);
+        if (step.pgm && /^[A-Z0-9#@$]{1,8}$/i.test(step.pgm)) runs(e.ref, String(step.pgm).toUpperCase());
+        for (const name of programsRunBy(step)) runs(e.ref, name);
+        if (!step.proc) continue;
+        const name = String(step.proc).toUpperCase();
+        const proc = procIds.get(name);
+        if (proc) { if (proc !== e.ref) depend(e.ref, proc); } else if (!inStream.has(name)) missing.add(name);
       }
+      if (missing.size) properties.push({ name: 'cobolwork:unresolved-proc', value: [...missing].sort(order).join(',') });
     }
     components.set(e.ref, { 'bom-ref': e.ref, type: 'file', name: e.ref, hashes: [{ alg: 'SHA-256', content: sha256(e.bytes) }], properties });
   }
@@ -130,6 +237,8 @@ export function sbom(root, opts = {}) {
   const list = [...components.values()].sort((a, b) => order(a['bom-ref'], b['bom-ref']));
   const digests = list.filter((c) => c.hashes).map((c) => c.hashes[0].content).sort(order);
   const name = opts.name || basename(top);
+  let revision = null;
+  try { revision = revisionOf(top); } catch { /* not a repository, or git is missing */ }
   const epoch = process.env.SOURCE_DATE_EPOCH;
   return {
     bomFormat: 'CycloneDX',
@@ -139,7 +248,10 @@ export function sbom(root, opts = {}) {
     metadata: {
       ...(epoch && /^\d+$/.test(epoch) ? { timestamp: new Date(Number(epoch) * 1000).toISOString() } : {}),
       tools: { components: [{ type: 'application', name: 'cobolwork', version: TOOL_VERSION }] },
-      component: { 'bom-ref': `estate:${name}`, type: 'application', name },
+      component: {
+        'bom-ref': `estate:${name}`, type: 'application', name,
+        ...(revision ? { version: revision.commit, properties: [{ name: 'cobolwork:dirty', value: String(revision.dirty) }] } : {}),
+      },
     },
     components: list,
     dependencies: [...dependencies.entries()].sort(([a], [b]) => order(a, b)).map(([ref, set]) => ({ ref, dependsOn: [...set].sort(order) })),

package/lib/scan.mjs

@@ -1,5 +1,5 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { sortFindings, tally, evidenceMap, EVIDENCE, EXPLOITABILITY } from './kernel/findings.mjs';
 import { directoryTree } from './kernel/source-tree.mjs';
 import { REGISTRY, RULE_SETS, ALL_RULES, reportKey } from './kernel/registry.mjs';
@@ -13,6 +13,7 @@ import { rollup } from './sets/flow.mjs';
 import { loadSite } from './site.mjs';
 import { reachFeedPaths, loadReachFeed, reachResolver, stampReach, stampEffect } from './reach.mjs';
 import { stampExploitability, byVerdict, HANDLING, witnessFeedPaths, loadWitnessFeed, applyWitness } from './exploitability.mjs';
+import { executionFeedPaths, loadExecutionFeed, applyExecution } from './execution.mjs';
 
 import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 
@@ -133,7 +134,7 @@ function scanEach(root, opts) {
 // Reach and effect annotate rather than re-rank; where no fact is declared, a note says so rather
 // than the report reading as a clean bill.
 export function applyEstateFacts(findings, checked, root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const feedRoot = opts.feedRoot || root;
   const reachFeeds = reachFeedPaths(opts).map((p) => loadReachFeed(p, { root: feedRoot }));
   const reach = reachResolver({ feeds: reachFeeds, site });
@@ -151,6 +152,8 @@ export function applyEstateFacts(findings, checked, root, opts = {}) {
   const witnessed = applyWitness(findings, checked, witnessFeeds);
   if (witnessed.overruled.length) sortFindings(findings);
   const byExploitability = byVerdict(findings);
+  const executionFeeds = executionFeedPaths(opts).map((p) => loadExecutionFeed(p));
+  const { byExecution } = applyExecution(findings, root, executionFeeds);
   const driven = byExploitability ? byExploitability['attacker-driven'] : 0;
   const loaded = witnessFeeds.filter((f) => !f.problem);
   return {
@@ -164,6 +167,8 @@ export function applyEstateFacts(findings, checked, root, opts = {}) {
     ...(byExploitability?.exploitable || byExploitability?.confirmed ? { handling: HANDLING } : {}),
     ...(loaded.length ? { witnessFeeds: loaded.map((f) => ({ file: f.file, witness: f.witness, recorded: f.recorded })), byWitness: witnessed.byWitness } : {}),
     ...(witnessFeeds.some((f) => f.problem || f.refused.length) ? { witnessFeedProblems: witnessFeeds.flatMap((f) => (f.problem ? [`${f.file}: ${f.problem}`] : f.refused.map((r) => `${f.file}: ${r.why}`))) } : {}),
+    ...(byExecution ? { executionFeeds: executionFeeds.filter((f) => !f.problem).map((f) => ({ file: f.file, programs: f.programs.size })), byExecution } : {}),
+    ...(executionFeeds.some((f) => f.problem) ? { executionFeedProblems: executionFeeds.filter((f) => f.problem).map((f) => `${f.file}: ${f.problem}`) } : {}),
     // A result no path finding carries: the code changed since the test, or the route is gone.
     ...(witnessed.unmatched.length ? { witnessUnmatched: witnessed.unmatched } : {}),
     // Routes the reading refuted that the estate reproduced: the check model was wrong about them.
@@ -234,10 +239,12 @@ export function scanAll(root, opts = {}) {
   const startedBy = parts.find((p) => p.tool === 'cobolwork-flow')?.startedBy || {};
   for (const p of parts) for (const f of p.findings) findings.push(!f.startedBy && f.program && startedBy[f.program] ? { ...f, startedBy: startedBy[f.program] } : { ...f });
   sortFindings(findings);
-  const identity = stampFingerprints(findings, { root, repo: opts.repoName || '' });
+  // A tree held in memory is read for the fingerprint's line text as it was for the finding.
+  const read = tree.kind === 'directory' ? null : (path) => tree.text(resolve(tree.root, path)).text;
+  const identity = stampFingerprints(findings, { root, repo: opts.repoName || '', read });
   // Routes a check stops keep their identity too, so a finding a fix clears can be found again here.
   const checked = parts.find((p) => p.tool === 'cobolwork-flow')?.checked || [];
-  stampFingerprints(checked, { root, repo: opts.repoName || '' });
+  stampFingerprints(checked, { root, repo: opts.repoName || '', read });
 
   const estate = applyEstateFacts(findings, checked, root, opts);
 

package/lib/sets/build.mjs

@@ -132,7 +132,7 @@ function withFeeds(coverage, feeds) {
 }
 
 export function scanBuild(root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts)).filter(isBuildFile);
   const findings = [];

package/lib/sets/cics.mjs

@@ -136,6 +136,31 @@ function execOptions(exec) {
 
 const subtreeRead = (x) => x.directRefs > 0 || x.children.some(subtreeRead);
 
+// Whether EIBCALEN bounds what the program reads: a condition tests it, or a field it was moved or
+// computed into; it is the start or length of a reference modification, or what an OCCURS DEPENDING
+// ON counts by. A CALL handed it, or the whole EIB, may test it where this program cannot see.
+// Named anywhere else - logged, passed on to a LINK - it checks nothing.
+function testsLength(p) {
+  // A statement's own tokens, since its sources leave out what a condition or an expression wraps
+  // in parentheses: IF (EIBCALEN > 0).
+  const tokens = (p.proc && p.proc.tokens) || [];
+  const own = (st) => (st.at != null ? tokens.slice(st.at + 1, st.end) : st.sources || []);
+  const carriers = new Set(['EIBCALEN']);
+  for (let grew = true; grew;) {
+    grew = false;
+    for (const st of p.statements || []) {
+      if (!own(st).some((t) => carriers.has(t.u))) continue;
+      for (const t of st.targets || []) if (!carriers.has(t.u)) { carriers.add(t.u); grew = true; }
+    }
+  }
+  const carries = (toks) => (toks || []).some((t) => t && carriers.has(t.u));
+  return p.items.some((i) => i.dependingOn === 'EIBCALEN') || (p.statements || []).some((st) =>
+    ((st.verb === 'IF' || st.verb === 'EVALUATE' || st.verb === 'WHEN') && carries(own(st)))
+    || (st.loops || []).some((l) => carries(l.until))
+    || (st.indexes || []).some((x) => x.kind.startsWith('refmod-') && carries([x.tok]))
+    || (st.verb === 'CALL' && (st.sources || []).some((t) => t.u === 'EIBCALEN' || t.u === 'DFHEIBLK')));
+}
+
 // Commands whose failure a program ordinarily lets pass: removing a temporary-storage queue that may
 // not exist, a line written to a transient-data log, the clock, a screen sent to a terminal that may
 // have gone, an ASSIGN, and the RETURN that ends the task anyway.
@@ -191,7 +216,6 @@ export function scanCics(root, opts = {}) {
   // The whole of what the second pass reads, computed once, while the tree is still here.
   function summarise(p, path) {
     const isSignOn = [...new Set([...p.refs.map(x => x.tok.u), ...p.execs.flatMap(e => e.toks.filter(t => t.t === 'word').map(t => t.u))])].some((w) => PASSWORD.test(w));
-    const words = new Set([...p.refs.map(x => x.tok.u), ...p.execs.flatMap(e => e.toks.filter(t => t.t === 'word').map(t => t.u))]);
     const commarea = p.items.find(i => i.name === 'DFHCOMMAREA' && i.section === 'LINKAGE');
     // A variable naming the program usually holds one literal from its VALUE clause.
     const valueOf = (name) => {
@@ -221,19 +245,22 @@ export function scanCics(root, opts = {}) {
         declared,
       });
     }
-    // A program that points DFHCOMMAREA at storage of its own is not reading a caller's area.
+    // A program that points DFHCOMMAREA at storage of its own is not reading a caller's area. Only
+    // the receiving side of SET, before TO, repoints it: SET P TO ADDRESS OF DFHCOMMAREA reads the
+    // caller's area's address.
     const tokens = (p.proc && p.proc.tokens) || [];
     const pointsItself = (p.statements || []).some((st) => {
       if (st.verb !== 'SET') return false;
       const w = tokens.slice(st.at + 1, st.end).filter((t) => t.t === 'word').map((t) => t.u);
-      return w.some((x, k) => x === 'ADDRESS' && w[k + 1] === 'OF' && w[k + 2] === 'DFHCOMMAREA');
+      const receiving = w.includes('TO') ? w.slice(0, w.indexOf('TO')) : w;
+      return receiving.some((x, k) => x === 'ADDRESS' && receiving[k + 1] === 'OF' && receiving[k + 2] === 'DFHCOMMAREA');
     });
     return {
       id: p.id, path,
       ownCics: p.execs.some((e) => e.kind === 'CICS'),
       pointsItself,
       commarea: commarea ? { ...where(commarea, path), size: commarea.size, read: subtreeRead(commarea) } : null,
-      checksLength: words.has('EIBCALEN'),
+      checksLength: testsLength(p),
       transfers,
       signOnBypasses: isSignOn ? signOnBypasses(p, path) : [],
       // Gated the same way: a program with no password-named field is not answering a sign-on,
@@ -390,7 +417,7 @@ export function scanCics(root, opts = {}) {
         findings.push({
           rule: 'cics-commarea-without-length-check', path: s.commarea.path, line: s.commarea.line, program: s.id,
           ...(covered ? { sev: 'low' } : {}),
-          detail: `${s.id} reads DFHCOMMAREA (${s.commarea.size} bytes declared) and never references EIBCALEN, so a shorter or absent communication area is read as if it were whole${covered ? `; every caller in the tree passes at least ${s.commarea.size} bytes, and no transaction starts it` : ''}`,
+          detail: `${s.id} reads DFHCOMMAREA (${s.commarea.size} bytes declared) and never tests EIBCALEN, so a shorter or absent communication area is read as if it were whole${covered ? `; every caller in the tree passes at least ${s.commarea.size} bytes, and no transaction starts it` : ''}`,
         });
       }
     }

package/lib/sets/copybook.mjs

@@ -7,6 +7,7 @@ import { finish } from '../kernel/findings.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
+import { cobolCard } from '../cards.mjs';
 
 // A copybook is resolved by name along a search path: the program's own directory first, then the
 // include directories, then the system library. Two files answering to one name mean the record
@@ -37,7 +38,7 @@ const SYSTEM_NAME = /^(DFHAID|DFHBMSCA|DFHEIBLK|DFHCOMMAREA|DFHRESP|SQLCA|SQLDA|
 function layout(src) {
   const fixed = detectFormat(src) === 'fixed';
   return src.split(/\r?\n/)
-    .map(l => (fixed ? l.slice(6, 72) : l).trimEnd())
+    .map(l => { const c = cobolCard(l, fixed ? 'fixed' : 'free'); return (c.indicator + c.text).trimEnd(); })
     .filter(l => l.trim() && !/^[*/]/.test(l.trimStart()))
     .join('\n')
     .toUpperCase();

package/lib/sets/hidden.mjs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { detectFormat, expandTabs } from '../parser.mjs';
+import { cobolCard } from '../cards.mjs';
 import { decodeEbcdic, inScope, isJcl, isSource, looksEbcdic, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
@@ -69,14 +70,9 @@ const BANNER = /^[\s*\-=_+#/\\.]*$/;
 
 function areas(line, format) {
   const l = expandTabs(line);
-  if (format === 'free' || format === 'terminal') return { sequence: '', indicator: '', text: l, identification: '', ignored: '' };
-  return {
-    sequence: l.slice(0, 6),
-    indicator: l[6] || ' ',
-    text: l.slice(7, format === 'variable' ? 250 : 72),
-    identification: format === 'variable' ? '' : l.slice(72, 80),
-    ignored: format === 'variable' ? '' : l.slice(72),
-  };
+  const card = cobolCard(l, format);
+  const fixed = format === 'fixed';
+  return { ...card, identification: fixed ? card.ignored.slice(0, 8) : '', ignored: fixed ? card.ignored : '' };
 }
 
 // A floating comment, from the *> that starts it outside a literal to the end of the line.

package/lib/sets/jcl.mjs

@@ -7,7 +7,8 @@ import { inScope, isJcl, isProgram, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { parseJcl, dispositionOf } from '../jcl.mjs';
-import { ddGroups } from '../utilities.mjs';
+import { ddGroups, tsoCommands } from '../utilities.mjs';
+import { ftpSession } from '../ftp.mjs';
 import { loadSite, productionQualifierOf, SITE_FILE } from '../site.mjs';
 import { loadPacks, programsOf } from '../packs.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
@@ -84,11 +85,16 @@ const SORTS = new Set(['SORT', 'ICEMAN', 'SYNCSORT']);
 const CONTROL_DDS = [
   [TSO, ['SYSTSIN']], [new Set(['BPXBATCH']), ['STDPARM']], [new Set(['IDCAMS']), ['SYSIN']],
   [SORTS, ['SYSIN', 'DFSPARM']], [new Set(['ICETOOL']), ['TOOLIN', 'DFSPARM']],
-  [new Set(['DSNTEP2', 'DSNTEP4', 'DSNTIAD']), ['SYSIN']],
 ];
 const TSO_RUNS = /^\s*(CALL|EXEC|EX|SUBMIT|ISPSTART|RUN)\b/i;
-// RUN PROGRAM(DSNTEP2) under the DSN command processor makes that step's SYSIN a program's SQL.
-const RUNS_SQL_PROGRAM = /\bRUN\s+PROGRAM\s*\(\s*(DSNTEP2|DSNTEP4|DSNTIAD)\s*\)/i;
+// Db2's sample programs read SQL statements from SYSIN, run as the step's program or under TSO by
+// DSN RUN PROGRAM. Those statements are the program's input, not commands to TSO or a security product.
+const SQL_PROGRAMS = new Set(['DSNTEP2', 'DSNTEP4', 'DSNTIAD']);
+const sqlInput = (step, groups) => {
+  const pgm = step.pgm ? step.pgm.toUpperCase() : null;
+  const runsSql = SQL_PROGRAMS.has(pgm) || tsoCommands(step).runs.some((x) => SQL_PROGRAMS.has(x.program));
+  return runsSql ? groups.get('SYSIN') || [] : [];
+};
 
 // Utilities whose commands come from a DD: a cataloged data set there is commands nobody reviewed
 // with the job; a MODS exit is a load module the sort runs; TSO commands say what else runs.
@@ -99,7 +105,7 @@ function controlCardFindings(r, path, findings) {
     const groups = ddGroups(step);
     const cards = (name) => (groups.get(name) || []).flatMap((dd) => dd.inStream || []);
     const ddNames = new Set(CONTROL_DDS.filter(([pgms]) => pgms.has(pgm)).flatMap(([, dds]) => dds));
-    if (TSO.has(pgm) && cards('SYSTSIN').some((l) => RUNS_SQL_PROGRAM.test(l.text))) ddNames.add('SYSIN');
+    if (sqlInput(step, groups).length) ddNames.add('SYSIN');
     for (const name of ddNames) {
       for (const dd of groups.get(name) || []) {
         if (dd.dsn && !dd.inStream && !dd.temporary) {
@@ -127,116 +133,6 @@ function controlCardFindings(r, path, findings) {
   }
 }
 
-// IBM's batch FTP client takes the host and its options on PARM, or the host as the first line of
-// its input, reads subcommands from INPUT, and logs on from NETRC or from its input. CardDemo's job
-// gives the same lines on SYSIN, which is read here when there is no INPUT, though the page below
-// does not say the client falls back to it.
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halu001/ftpreq.htm
-// A session is in cleartext unless something asks for TLS: -a TLS or -r TLS on PARM, or
-// SECURE_MECHANISM TLS in an in-stream SYSFTPD. A SYSFTPD naming a data set is configuration this
-// reader cannot see, so that step is not called cleartext either way.
-//   https://www.ibm.com/docs/en/zos/2.1.0?topic=commands-ftp-command
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halz002/ftp_cust_client_for_tls.htm
-// PUT, MPUT and APPEND send local files, and IBM's capitals - PUt, MPut, APpend, LCd - are the
-// shortest spellings. A local name in quotes is a whole data set name, //DD: names a DD of the
-// step, and anything else hangs off the local working directory, which only a quoted LCD fixes.
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/ftpsubcmd.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/put.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/mput.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/append.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/lcd.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/dd_name_support.htm
-const FTP_SENDS = [[/^PUT?$/i, 'PUT'], [/^MP(?:UT?)?$/i, 'MPUT'], [/^AP(?:P(?:E(?:ND?)?)?)?$/i, 'APPEND']];
-
-// Every subcommand on the page above, in IBM's spelling.
-const FTP_SUBCOMMANDS = ['?', '!', 'ACCount', 'APpend', 'AUth', 'AScii', 'BIG5', 'BINary', 'BLock', 'CCc', 'CD',
-  'CDUp', 'CLEar', 'CLose', 'COMpress', 'CProtect', 'CWd', 'DEBug', 'DELEte', 'DELImit', 'DIr', 'DUMP', 'EBcdic',
-  'EUckanji', 'FEature', 'FIle', 'Get', 'GLob', 'HAngeul', 'HElp', 'Ibmkanji', 'JIS78kj', 'JIS83kj', 'Ksc5601',
-  'LANGuage', 'LCd', 'LOCSIte', 'LOCSTat', 'LMkdir', 'LPwd', 'LS', 'MDelete', 'MGet', 'MKdir', 'MKFifo', 'MOde',
-  'MPut', 'MVSGet', 'MVSPut', 'NOop', 'Open', 'PAss', 'PRIvate', 'PROMpt', 'PROTect', 'PROXy', 'PUt', 'PWd', 'QUIt',
-  'QUOte', 'RECord', 'REName', 'REStart', 'RMdir', 'SAfe', 'SChinese', 'SENDPort', 'SENDSite', 'SIte', 'SJiskanji',
-  'SRestart', 'STAtus', 'STREam', 'STRucture', 'SUnique', 'SYstem', 'TChinese', 'TSO', 'TYpe', 'UCs2', 'User', 'Verbose']
-  .map((name) => ({ full: name.toUpperCase(), min: name.match(/^[^a-z]*/)[0].length }));
-const subcommandOf = (word) => {
-  const w = word.toUpperCase();
-  const hit = FTP_SUBCOMMANDS.find((c) => w.length >= c.min && c.full.startsWith(w));
-  return hit ? hit.full : null;
-};
-// A value that stands for a password rather than being one: a symbolic, a mask, or the word itself.
-const PLACEHOLDER = /^(&.*|\*+|X+|\?+|<.*>|\{.*\}|PASSWORD|PASSWD|PWD|PASS|SECRET)$/i;
-
-function ftpSession(step) {
-  const dds = ddGroups(step);
-  let host = null, tls = null;
-  const words = (step.parm || '').split('(')[0].trim().split(/\s+/).filter(Boolean);
-  for (let i = 0; i < words.length; i++) {
-    if (/^-[ar]$/i.test(words[i])) {
-      const mechanism = (words[++i] || '').toUpperCase();
-      if (mechanism === 'TLS') tls = 'asked';
-      else if (mechanism === 'NEVER' && !tls) tls = 'never';
-    } else if (!words[i].startsWith('-') && !host) host = words[i];
-  }
-  let configUnread = null;
-  for (const dd of dds.get('SYSFTPD') || []) {
-    if (dd.inStream) { if (tls !== 'never' && dd.inStream.some((l) => /^\s*SECURE_MECHANISM\s+TLS\b/i.test(l.text))) tls = 'asked'; }
-    else if (dd.dsn && !configUnread) configUnread = dd.dsn;
-  }
-
-  const input = dds.get('INPUT') || dds.get('SYSIN') || [];
-  const hostLine = !host && input[0]?.inStream;
-  let inputUnread = null, lcd = null, unixDir = false;
-  const sends = [];
-  // With no NETRC to answer for it, the client asks for a user ID and a password and in batch reads
-  // them from its input after the host: "user password" on one line, or each on its own. A first
-  // line that is a subcommand means something else answered, a NETRC the client found for itself.
-  // USER and PASS give the same answers later in a session.
-  let expect = dds.has('NETRC') ? null : 'logon', user = null;
-  const passwords = [];
-  const password = (dd, line, value) => { if (!PLACEHOLDER.test(value)) passwords.push({ dd, line, user }); };
-  const local = (name) => {
-    const dd = /^\/\/DD:([A-Z$#@][A-Z0-9$#@]{0,7})$/i.exec(name);
-    if (dd) return { dd: dd[1].toUpperCase(), dsns: (dds.get(dd[1].toUpperCase()) || []).map((d) => d.dsn).filter(Boolean) };
-    const quoted = /^'(.+)'$/.exec(name);
-    if (quoted) return { dsns: [quoted[1].toUpperCase()] };
-    if (name.startsWith('/') || unixDir) return { dsns: [] };
-    return lcd ? { dsns: [`${lcd}.${name.toUpperCase()}`] } : { dsns: [], relative: name };
-  };
-  for (const dd of input) {
-    if (!dd.inStream) { if (dd.dsn && !inputUnread) inputUnread = dd.dsn; continue; }
-    for (const { line, text } of dd.inStream) {
-      const [verb, ...args] = text.slice(0, 72).trim().split(/\s+/);
-      if (!verb) continue;
-      if (hostLine && !host) { host = verb; continue; }
-      const sub = subcommandOf(verb);
-      if (expect && !sub) {
-        if (expect === 'logon') user = verb;
-        if (expect === 'password') password(dd, line, verb);
-        else if (args[0]) password(dd, line, args[0]);
-        expect = expect === 'logon' && !args[0] ? 'password' : null;
-        continue;
-      }
-      expect = null;
-      if (sub === 'USER' && args[0]) {
-        user = args[0];
-        if (args[1]) password(dd, line, args[1]); else expect = 'password';
-        continue;
-      }
-      if (sub === 'PASS' && args[0]) { password(dd, line, args[0]); continue; }
-      if (/^LCD?$/i.test(verb) && args[0]) {
-        const quoted = /^'(.+)'$/.exec(args[0]);
-        unixDir = args[0].startsWith('/');
-        lcd = quoted ? quoted[1].toUpperCase() : lcd && !unixDir ? `${lcd}.${args[0].toUpperCase()}` : null;
-        continue;
-      }
-      const sent = FTP_SENDS.find(([re]) => re.test(verb));
-      if (!sent) continue;
-      for (const name of sent[1] === 'MPUT' ? args : args.slice(0, 1)) sends.push({ line, verb: sent[1], name, ...local(name) });
-    }
-  }
-  const cleartext = tls === 'never' || (!tls && !configUnread);
-  return { host, cleartext, configUnread, inputUnread, logon: dds.has('NETRC') ? 'NETRC' : null, sends, passwords, input };
-}
-
 // Credentials, which are the reason this rule set is critical rather than merely interesting.
 // Each run of blanks follows the ( or = that separates it from the next, which keeps matching linear.
 const CREDENTIAL = /\b(?:PASSWORD|PASSWRD|PHRASE|PASSPHRASE)\s*(?:\(\s*)?(?:=\s*)?['"]?([A-Z0-9@#$_.-]{3,})/i;
@@ -247,7 +143,7 @@ const CREDENTIAL = /\b(?:PASSWORD|PASSWRD|PHRASE|PASSPHRASE)\s*(?:\(\s*)?(?:=\s*
 const SECURITY_COMMAND = /^\s*(?:TSS\s+)?(ADDUSER|ALTUSER|DELUSER|ADDGROUP|ALTGROUP|CONNECT|REMOVE|PERMIT|SETROPTS|RDEFINE|RALTER|RDELETE|ADDSD|ALTDSD|DELDSD|PASSWORD|ACFNRULE)\b/i;
 
 // ACF2 speaks differently: a mode change followed by INSERT or CHANGE against a logonid.
-const ACF2_COMMAND = /^\s*(?:SET\s+(?:LID|RULE|PROFILE)|INSERT\s+[A-Z0-9$#@]+|CHANGE\s+[A-Z0-9$#@]+\s)/i;
+const ACF2_COMMAND = /^\s*(?:SET\s+(?:LID|RULE|PROFILE)|INSERT\s+(?!INTO\b)[A-Z0-9$#@]+|CHANGE\s+[A-Z0-9$#@]+\s)/i;
 
 // Destruction in-stream. DELETE with a dataset name, or a REPRO that replaces its target. SQL's
 // DELETE FROM removes rows through a database utility, not a catalogued dataset.
@@ -341,7 +237,7 @@ export function scanJcl(root, opts = {}) {
   // A vendor product supplies its own utilities. An estate that has loaded the pack for a product
   // is telling us those programs exist, so reporting them as undefined would be a false positive
   // created by the customer having told us the truth.
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const { loaded } = loadPacks(opts.packs || site.vendorPacks || [], { allowUnvalidated: true });
   const vendorPrograms = programsOf(loaded);
 
@@ -370,9 +266,10 @@ export function scanJcl(root, opts = {}) {
       if (d && !stats.firstReadInPart) stats.firstReadInPart = `${path}${d.line ? `:${d.line}` : ''}: ${d.text}`;
     }
 
-    const ftpSteps = new Map(r.steps.filter((s) => s.pgm && s.pgm.toUpperCase() === 'FTP' && !defined.has('FTP')).map((s) => [s, ftpSession(s)]));
+    const ftpSteps = new Map(r.steps.filter((s) => s.pgm && s.pgm.toUpperCase() === 'FTP' && !defined.has('FTP')).map((s) => [s, ftpSession(s, ddGroups(s))]));
     const ftpInput = new Map();
     for (const ftp of ftpSteps.values()) for (const dd of ftp.input) ftpInput.set(dd, ftp);
+    const sqlDds = new Set(r.steps.flatMap((s) => sqlInput(s, ddGroups(s))));
 
     for (const dd of r.dds) {
       if (!dd.inStream) continue;
@@ -398,13 +295,13 @@ export function scanJcl(root, opts = {}) {
             detail: `in-stream FTP input gives the password ${logon.user ? `for ${logon.user} ` : ''}that step ${dd.step || '(none)'} logs on ${ftp.host ? `to ${ftp.host} ` : ''}with, in a file anyone who can read the repository can read` });
           continue;
         }
-        const sec = text.match(SECURITY_COMMAND);
+        const sec = !sqlDds.has(dd) && text.match(SECURITY_COMMAND);
         if (sec) {
           findings.push({ rule: 'jcl-instream-security-command', path, line: l.line, step: dd.step,
             detail: `in-stream data issues ${sec[1].toUpperCase()}, which grants or alters authority, from step ${dd.step || '(none)'}` });
           continue;
         }
-        if (ACF2_COMMAND.test(text)) {
+        if (!sqlDds.has(dd) && ACF2_COMMAND.test(text)) {
           findings.push({ rule: 'jcl-instream-security-command', path, line: l.line, step: dd.step,
             detail: `in-stream data issues an ACF2 administration command from step ${dd.step || '(none)'}` });
           continue;

package/lib/sets/priv.mjs

@@ -102,7 +102,7 @@ function under(dsn, prefixes) {
 }
 
 export function scanPriv(root, opts = {}) {
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts));
   const findings = [];