@portll/cobolwork 0.6.0 → 0.7.0

package/lib/precompile-check.mjs

@@ -0,0 +1,124 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// The compile check `cobolwork build --precompile` runs before the caller's compiler: each program the
+// compiler command names that holds EXEC SQL or EXEC CICS is translated by lib/precompile.mjs into a
+// directory of its own, beside the stand-in copybooks and translated members it needs, and given to
+// the same compiler with -fsyntax-only, so a program GnuCOBOL refuses only for its EXEC blocks is
+// checked. Nothing it writes outlives the check.
+import { spawnSync } from 'node:child_process';
+import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { basename, delimiter, dirname, extname, join, relative, resolve } from 'node:path';
+import { parseBms } from './bms.mjs';
+import { printable } from './kernel/printable.mjs';
+import { detectFormat, parseSource } from './parser.mjs';
+import { precompile } from './precompile.mjs';
+import { isBms, isProgram, readSource } from './sources.mjs';
+
+const COPY_EXTENSIONS = ['', '.cpy', '.CPY', '.cbl', '.CBL', '.cob', '.COB'];
+const MAX_ERRORS = 3;
+
+// A COPY member's text from the first directory holding it, as cobc searches them.
+export const copybookIn = (dirs) => (name) => {
+  for (const d of dirs) {
+    for (const ext of COPY_EXTENSIONS) {
+      const p = join(d, name + ext);
+      try { if (statSync(p).isFile()) return readSource(p).text; } catch { /* not here */ }
+    }
+  }
+  return null;
+};
+
+// The BMS files under a repository, the member names its other files supply, and the mapsets it holds
+// no copybook for, whose symbolic maps the translator writes.
+export function repositoryBms(repoRoot) {
+  const files = [];
+  const walk = (d) => {
+    let es;
+    try { es = readdirSync(d, { withFileTypes: true }); } catch { return; }
+    for (const e of es) {
+      if (e.name === '.git' || e.name.startsWith('._')) continue;
+      const p = join(d, e.name);
+      if (e.isDirectory()) walk(p); else files.push(p);
+    }
+  };
+  walk(repoRoot);
+  const bms = files.filter(isBms);
+  const members = new Set(files.filter((f) => !bms.includes(f)).map((f) => basename(f, extname(f)).toUpperCase()));
+  const mapsets = bms.flatMap((f) => { try { return parseBms(readFileSync(f, 'latin1')).mapsets; } catch { return []; } })
+    .filter((ms) => ms.name && !members.has(ms.name.toUpperCase()));
+  return { bms, has: (n) => members.has(n.toUpperCase()), mapsets };
+}
+
+const EXEC = /EXEC\s+(?:SQL|CICS)\b/i;
+const DLI = /EXEC\s+DLI\b/i;
+const redact = (s) => String(s).replace(/'[^']*'|"[^"]*"/g, "'…'");
+
+// The copy directories a cobc or gcobol argument vector names, in order: -I <dir> and -I<dir>.
+function includeDirsOf(argv, cwd) {
+  const dirs = [];
+  argv.forEach((a, i) => {
+    if (a === '-I' && argv[i + 1]) dirs.push(resolve(cwd, argv[i + 1]));
+    else if (a.startsWith('-I') && a.length > 2) dirs.push(resolve(cwd, a.slice(2)));
+  });
+  return dirs;
+}
+
+// The argument vector with -o and its value taken out: a syntax check writes nothing.
+const withoutOutput = (argv) => argv.filter((a, i) => a !== '-o' && argv[i - 1] !== '-o' && !(a.startsWith('-o') && a.length > 2));
+
+// A compiler's first error lines, each named by the source line it came from and with every quoted
+// literal masked, since a diagnostic may quote the program and no output of the gate carries source.
+function errorsOf(stderr, translated, map, rel) {
+  const out = [];
+  for (const line of String(stderr || '').split('\n')) {
+    const m = /^(.*?):(\d+):\s*error:\s*(.*)$/.exec(line);
+    if (!m) continue;
+    const at = resolve(m[1]) === translated ? `${rel}:${map[Number(m[2]) - 1] ?? m[2]}` : `${basename(m[1])}:${m[2]}`;
+    out.push(printable(`${at}: ${redact(m[3])}`, 300));
+    if (out.length === MAX_ERRORS) break;
+  }
+  return out;
+}
+
+// Runs the check. `compilerPath` is the resolved compiler and `argv` the caller's arguments after it;
+// `cwd` is where the caller's compiler runs, so its relative paths resolve there. Returns, for each
+// program the argument vector names that holds EXEC SQL or CICS, whether the compiler accepted its
+// translation and its first errors; programs holding EXEC DLI, which nothing translates, are skipped.
+export function precompileCheck({ compilerPath, argv, cwd = process.cwd(), repo, copylibs = [], env = process.env }) {
+  const sources = argv.map((a, i) => ({ a, i, path: resolve(cwd, a) }))
+    .filter(({ a, i, path }) => !a.startsWith('-') && argv[i - 1] !== '-o' && argv[i - 1] !== '-I' && isProgram(path) && isFile(path));
+  const includeDirs = [...includeDirsOf(argv, cwd), ...String(env.COBCPY || '').split(delimiter).filter(Boolean), ...copylibs];
+  const others = withoutOutput(argv.filter((_, i) => !sources.some((s) => s.i === i)));
+  const programs = [];
+  const skipped = [];
+  let mapsets = null;
+  const root = mkdtempSync(join(tmpdir(), 'cw-precompile-'));
+  try {
+    for (const { path } of sources) {
+      const rel = printable(relative(repo, path).startsWith('..') ? relative(cwd, path) : relative(repo, path), 200);
+      const text = readSource(path).text;
+      if (DLI.test(text)) { skipped.push({ path: rel, why: 'EXEC DLI is not translated' }); continue; }
+      if (!EXEC.test(text)) continue;
+      const format = detectFormat(text);
+      const dirs = [dirname(path), ...includeDirs];
+      let items = [];
+      try { items = parseSource(text, path, { format, includeDirs: dirs }).programs.flatMap((p) => p.items); } catch { /* unparsed: no host-variable arrays known */ }
+      if (/EXEC\s+CICS\b/i.test(text) && !mapsets) mapsets = repositoryBms(repo).mapsets;
+      const p = precompile(text, { format, items, mapsets: /EXEC\s+CICS\b/i.test(text) ? mapsets : [], copybook: copybookIn(dirs) });
+      const dir = mkdtempSync(join(root, 'p-'));
+      const translated = join(dir, basename(path));
+      writeFileSync(translated, p.text);
+      for (const [name, body] of Object.entries(p.copybooks)) writeFileSync(join(dir, `${name}.cpy`), body);
+      const r = spawnSync(compilerPath, ['-fsyntax-only', '-I', dir, ...others, translated], { cwd, env, encoding: 'utf8', windowsHide: true });
+      const ok = !r.error && r.status === 0;
+      programs.push({ path: rel, ok, ...(ok ? {} : { errors: r.error ? [printable(r.error.message, 200)] : errorsOf(r.stderr, translated, p.map, rel) }) });
+    }
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+  return { argv: ['-fsyntax-only', '-I', '<translation>', ...others, '<program>'], programs, skipped };
+}
+
+function isFile(path) {
+  try { return statSync(path).isFile(); } catch { return false; }
+}

package/lib/precompile-cics.mjs

@@ -39,13 +39,17 @@ export function cicsOptions(text) {
   return out;
 }
 
-// Which table entry a command is: the one an option word identifies (SEND MAP by MAP), else the verb
-// alone, else the one IBM makes the default (READQ is READQ TS).
+// Which table entry a command is: the longest one whose name the words after the verb spell in order
+// (WEB READ HTTPHEADER; INQUIRE TRANSACTION though a PROGRAM option follows), else the one an
+// option word identifies (SEND MAP by MAP), else the verb alone, else the one IBM makes the default
+// (READQ is READQ TS).
 export function cicsCommand(options) {
   const verb = options[0]?.word;
   const words = new Set(options.slice(1).map((o) => o.word));
   const names = Object.keys(CICS_COMMANDS).filter((n) => n === verb || n.startsWith(`${verb} `));
-  return names.find((n) => CICS_COMMANDS[n].identify?.some((w) => words.has(w)))
+  const spelled = names.map((n) => n.split(' ')).filter((w) => w.length > 1 && w.slice(1).every((x, i) => options[i + 1]?.word === x))
+    .sort((a, b) => b.length - a.length)[0];
+  return spelled?.join(' ') || names.find((n) => CICS_COMMANDS[n].identify?.some((w) => words.has(w)))
     || names.find((n) => n === verb) || names.find((n) => CICS_COMMANDS[n].default) || null;
 }
 

package/lib/reach.mjs

@@ -14,11 +14,15 @@
 // lib/exploitability.mjs joins them with the route into a verdict.
 import { readFileSync, realpathSync } from 'node:fs';
 import { basename, sep, delimiter } from 'node:path';
+import { versionProblem, unknownKeys, ignoredKey } from './kernel/version-key.mjs';
 
 const str = (v) => typeof v === 'string' && v.length > 0;
 const isoDate = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(v);
 const up = (s) => String(s || '').toUpperCase();
 const ACCESS = new Set(['open', 'restricted']);
+export const REACH_VERSION = 1;
+const REACH_KEYS = new Set(['$schema', 'version', 'extract', 'retrieved', 'transactions', 'jobs', 'privileged']);
+const PRIVILEGED_KEYS = new Set(['transactions', 'jobs']);
 const arr = (v) => (Array.isArray(v) ? v.filter(str).map(up) : []);
 
 // The feeds a scan was given: the caller's list, or COBOLWORK_REACH read when the scan runs.
@@ -34,7 +38,7 @@ export function reachFeedPaths(opts = {}) {
 // derived from a RACF unload (who may start each transaction, and which run with elevated authority)
 // and the SIT (which face a network).
 export function loadReachFeed(path, { root = null } = {}) {
-  const feed = { file: basename(path), extract: null, retrieved: null, transactions: {}, jobs: {}, privileged: { transactions: [], jobs: [] }, refused: [], problem: null };
+  const feed = { file: basename(path), extract: null, retrieved: null, transactions: {}, jobs: {}, privileged: { transactions: [], jobs: [] }, refused: [], warnings: [], problem: null };
   let real;
   try { real = realpathSync(path); } catch (e) { feed.problem = `could not be read (${e.code || e.name})`; return feed; }
   if (root) {
@@ -43,6 +47,9 @@ export function loadReachFeed(path, { root = null } = {}) {
   }
   let doc;
   try { doc = JSON.parse(readFileSync(real, 'utf8')); } catch (e) { feed.problem = `is not JSON (${e.message})`; return feed; }
+  if (!doc || typeof doc !== 'object' || Array.isArray(doc)) { feed.problem = 'is not a JSON object'; return feed; }
+  feed.problem = versionProblem(doc, REACH_VERSION);
+  if (feed.problem) return feed;
   if (!str(doc.extract)) { feed.problem = 'does not say which extract it is'; return feed; }
   if (!isoDate(doc.retrieved)) { feed.problem = 'does not say when it was retrieved'; return feed; }
   feed.extract = doc.extract;
@@ -56,7 +63,9 @@ export function loadReachFeed(path, { root = null } = {}) {
     }
   }
   const pv = doc.privileged;
-  if (pv && typeof pv === 'object' && !Array.isArray(pv)) { feed.privileged.transactions = arr(pv.transactions); feed.privileged.jobs = arr(pv.jobs); }
+  const held = pv && typeof pv === 'object' && !Array.isArray(pv);
+  if (held) { feed.privileged.transactions = arr(pv.transactions); feed.privileged.jobs = arr(pv.jobs); }
+  feed.warnings = [...unknownKeys(doc, REACH_KEYS).map((k) => ignoredKey(feed.file, k)), ...(held ? unknownKeys(pv, PRIVILEGED_KEYS) : []).map((k) => ignoredKey(`${feed.file} privileged`, k))];
   return feed;
 }
 

package/lib/revision.json

@@ -1 +1 @@
-{"commit":"67c696c3d6f74d20137372b5f0a756e0b27753dd","tag":"v0.6.0"}
+{"commit":"4d7151cd03c0ac0850eec10a2529b07f0fc04aa1","tag":"v0.7.0"}

package/lib/sarif.mjs

@@ -45,13 +45,23 @@ const DECLARED = new Map([...Object.entries(ALL_RULES), ...Object.entries(DIFF_R
 const SET_OF = new Map();
 for (const s of REGISTRY) for (const id of Object.keys(s.rules)) SET_OF.set(id, s.name);
 
+// A related entry without a path is an advisory the finding rests on, not a place in the tree.
+const located = (f) => (f.related || []).filter((r) => r.path);
+
+const cweTaxon = (cwe) => /^CWE-(\d+)$/.exec(cwe || '')?.[1] ?? null;
+
+// The CWE taxonomy is run.taxonomies[0] whenever any rule names a CWE.
+const CWE_COMPONENT = { name: 'CWE', index: 0 };
+
 // SARIF's level cannot say what kind of claim a result is, so evidence rides in the property bags.
 const descriptorFor = (report) => (id) => {
   const properties = {
     ...(report.ruleCwe?.[id] ? { cwe: report.ruleCwe[id] } : {}),
     ...(report.ruleEvidence?.[id] ? { evidence: report.ruleEvidence[id] } : {}),
+    ...(report.ruleCompliance?.[id] ? { compliance: report.ruleCompliance[id] } : {}),
     ...(SECURITY_SEVERITY[DECLARED.get(id)] ? { 'security-severity': SECURITY_SEVERITY[DECLARED.get(id)] } : {}),
   };
+  const taxon = cweTaxon(report.ruleCwe?.[id]);
   return {
     id,
     shortDescription: { text: report.ruleText?.[id] || id },
@@ -59,10 +69,23 @@ const descriptorFor = (report) => (id) => {
     // rule. `help` is where GitHub code scanning shows a recommendation.
     ...(report.ruleImpact?.[id] ? { fullDescription: { text: textOf(report.ruleImpact[id]) } } : {}),
     ...(report.ruleRemedy?.[id] ? { help: { text: textOf(report.ruleRemedy[id]) } } : {}),
+    ...(taxon ? { relationships: [{ target: { id: taxon, toolComponent: CWE_COMPONENT } }] } : {}),
     ...(Object.keys(properties).length ? { properties } : {}),
   };
 };
 
+function cweTaxonomy(report, ruleIds) {
+  const taxa = [...new Set(ruleIds.map((id) => cweTaxon(report.ruleCwe?.[id])).filter(Boolean))].sort((a, b) => a - b);
+  if (!taxa.length) return [];
+  return [{
+    name: CWE_COMPONENT.name,
+    organization: 'MITRE',
+    shortDescription: { text: 'Common Weakness Enumeration' },
+    informationUri: 'https://cwe.mitre.org/',
+    taxa: taxa.map((id) => ({ id, helpUri: `https://cwe.mitre.org/data/definitions/${id}.html` })),
+  }];
+}
+
 function notifications(report) {
   const bySet = report.summary?.bySet || {};
   const execution = [];
@@ -111,6 +134,9 @@ function notifications(report) {
       properties: { set },
     });
   }
+  for (const [id, warnings] of [['cobolwork/site-key-ignored', report.summary?.siteWarnings], ['cobolwork/feed-key-ignored', report.summary?.feedWarnings]]) {
+    for (const w of warnings || []) configuration.push({ descriptor: { id }, level: 'warning', message: { text: textOf(w) } });
+  }
   // A diff whose two sides were scanned under different configuration.
   for (const { file, change } of report.summary?.configurationChanged || []) {
     configuration.push({
@@ -144,6 +170,16 @@ export function toSarif(report, { toolVersion = '0.1.0' } = {}) {
 
   const { execution, configuration } = notifications(report);
   const s = report.summary || {};
+  const taxonomies = cweTaxonomy(report, present);
+
+  // Every file a result or related location names, each listed once; locations point at it by index.
+  const results = [...report.findings, ...suppressed];
+  const uris = [...new Set(results.flatMap((f) => [f.path, ...located(f).map((r) => r.path)]).map(uriOf))].sort();
+  const artifactIndex = new Map(uris.map((uri, i) => [uri, i]));
+  const artifactLocation = (path) => {
+    const uri = uriOf(path);
+    return { uri, uriBaseId: '%SRCROOT%', index: artifactIndex.get(uri) };
+  };
 
   return {
     $schema: 'https://json.schemastore.org/sarif-2.1.0.json',
@@ -158,6 +194,7 @@ export function toSarif(report, { toolVersion = '0.1.0' } = {}) {
         },
         ...(extensions.length ? { extensions } : {}),
       },
+      ...(taxonomies.length ? { taxonomies } : {}),
       invocations: [{
         // The verdict, where a consumer that reads nothing else will still see it.
         executionSuccessful: s.nosrc !== true,
@@ -183,7 +220,8 @@ export function toSarif(report, { toolVersion = '0.1.0' } = {}) {
           },
         },
       }],
-      results: [...report.findings, ...suppressed].map((f) => {
+      ...(uris.length ? { artifacts: uris.map((uri) => ({ location: { uri, uriBaseId: '%SRCROOT%' } })) } : {}),
+      results: results.map((f) => {
         const set = SET_OF.get(f.rule);
         const component = set !== undefined ? indexOfSet.get(set) : undefined;
         const within = component === undefined ? null : extensions[component].rules.findIndex((r) => r.id === f.rule);
@@ -217,8 +255,8 @@ export function toSarif(report, { toolVersion = '0.1.0' } = {}) {
             // a subprogram no caller run is shown to pass.
             ...(f.abend?.inputFrom ? { inputFrom: f.abend.inputFrom } : {}),
           } } : {}),
-          locations: [{ physicalLocation: { artifactLocation: { uri: uriOf(f.path), uriBaseId: '%SRCROOT%' }, region: { startLine: f.line || 1 } } }],
-          ...(f.related ? { relatedLocations: f.related.map((r) => ({ physicalLocation: { artifactLocation: { uri: uriOf(r.path), uriBaseId: '%SRCROOT%' }, region: { startLine: r.line || 1 } }, message: { text: textOf(r.detail) } })) } : {}),
+          locations: [{ physicalLocation: { artifactLocation: artifactLocation(f.path), region: { startLine: f.line || 1 } } }],
+          ...(located(f).length ? { relatedLocations: located(f).map((r) => ({ physicalLocation: { artifactLocation: artifactLocation(r.path), region: { startLine: r.line || 1 } }, message: { text: textOf(r.detail) } })) } : {}),
           ...(f.suppressed ? { suppressions: [{ kind: 'external', status: 'accepted', justification: `${f.suppressed.action} by ${f.suppressed.who} until ${f.suppressed.expires}: ${f.suppressed.reason}` }] } : {}),
         };
       }),

package/lib/scan.mjs

@@ -66,6 +66,9 @@ function mergeEstate(reports) {
     effectNote: effectless.length ? `no authority facts declared in ${of(effectless)}, so no finding there was judged to run privileged: name privilegedTransactions/privilegedJobs in each repository's cobolwork.site.json, or bring them in the COBOLWORK_REACH extract` : null,
     witnessFeeds: first('witnessFeeds'), byWitness: addUp('byWitness'), witnessFeedProblems: union('witnessFeedProblems'),
     witnessUnmatched: unmatched, witnessOverruled: overruled,
+    executionFeeds: first('executionFeeds'), byExecution: addUp('byExecution'), executionFeedProblems: union('executionFeedProblems'),
+    siteWarnings: all.flatMap((x) => (x.s.siteWarnings || []).map((w) => `${x.repo}/${w}`)),
+    feedWarnings: union('feedWarnings'),
   };
   return Object.fromEntries(Object.entries(merged).filter(([, v]) => (Array.isArray(v) ? v.length : v)));
 }
@@ -102,7 +105,7 @@ function scanEach(root, opts) {
       filesScanned, programs: sum('programs'), programFiles: sum('programFiles'), copybookFiles: sum('copybookFiles'),
       jclFiles: sum('jclFiles'), copiesMissing: sum('copiesMissing'), filesUnreadable: sum('filesUnreadable'),
       filesEbcdic: sum('filesEbcdic'), filesOverBudget: sum('filesOverBudget'), crossProgram: sum('crossProgram'),
-      ruleSets: opts.only || RULE_SETS, nosrc: filesScanned === 0,
+      ruleSets: opts.only || RULE_SETS, flowModel: FLOW_MODEL, toolVersion: TOOL_VERSION, nosrc: filesScanned === 0,
       coverageIncomplete: reports.some(({ r }) => r.summary.coverageIncomplete),
       identity: { version: FINGERPRINT_VERSION, shared },
       ...(reports[0]?.r.summary.advisoryFeeds ? { advisoryFeeds: reports[0].r.summary.advisoryFeeds } : {}),
@@ -158,7 +161,10 @@ export function applyEstateFacts(findings, checked, root, opts = {}) {
   const { byExecution } = applyExecution(findings, root, executionFeeds);
   const driven = byExploitability ? byExploitability['attacker-driven'] : 0;
   const loaded = witnessFeeds.filter((f) => !f.problem);
+  const feedWarnings = [...reachFeeds, ...witnessFeeds].flatMap((f) => f.warnings).sort();
   return {
+    ...(site.warnings.length ? { siteWarnings: site.warnings } : {}),
+    ...(feedWarnings.length ? { feedWarnings } : {}),
     ...(byReach ? { byReach } : {}),
     ...(byEffect ? { byEffect } : {}),
     ...(reachFeeds.some(f => !f.problem) ? { reachFeeds: reachFeeds.filter(f => !f.problem).map(f => ({ file: f.file, extract: f.extract, retrieved: f.retrieved })) } : {}),

package/lib/sets/abend.mjs

@@ -63,6 +63,10 @@ const OPTIMIZED = {
 // reference modification's start, length, and start plus length.
 const RANGE_MESSAGES = /^IGZ00(06|07|72|73|74)S\b/;
 
+// Language Environment's message for a dynamic CALL of a module no library holds, which ironwork
+// ends U4038 (its assumption C450); earlier ironwork releases ended the same CALL S806.
+const MODULE_NOT_FOUND = /^CEE3501S The module .* was not found\.$/;
+
 export function abendRunPaths(opts = {}) {
   if (opts.abendRuns) return opts.abendRuns;
   return (process.env.COBOLWORK_ABENDS || '').split(delimiter).filter(Boolean);
@@ -72,13 +76,19 @@ export function abendRunPaths(opts = {}) {
 // code CICS reports as ASRA.
 const CICS_PROGRAM_CHECK = /\((S0C7|S0C4), which CICS reports as ASRA\)$/;
 
+// How ironwork ends the message of a Language Environment condition in a CICS task, U4038 in
+// batch, which CICS reports as transaction abend 4038 (its assumption C454).
+const CICS_LE_CONDITION = / \(U4038, which CICS reports as transaction abend 4038\)$/;
+
 export function abendRule(abend) {
-  const code = (abend.code === 'ASRA' && CICS_PROGRAM_CHECK.exec(abend.message || '')?.[1]) || abend.code;
+  const cicsLe = abend.code === '4038' && CICS_LE_CONDITION.test(abend.message || '');
+  const message = cicsLe ? abend.message.replace(CICS_LE_CONDITION, '') : abend.message || '';
+  const code = cicsLe ? 'U4038' : (abend.code === 'ASRA' && CICS_PROGRAM_CHECK.exec(message)?.[1]) || abend.code;
   if (code === 'S0C7') return 'input-causes-abend-s0c7';
   if (code === 'S0C4') return 'input-causes-abend-s0c4';
   if (code === 'S322') return 'input-causes-hang';
-  if (code === 'S806') return 'input-selects-program';
-  if (RANGE_MESSAGES.test(abend.message || '')) return 'input-causes-abend-subscript-range';
+  if (code === 'S806' || (code === 'U4038' && MODULE_NOT_FOUND.test(message))) return 'input-selects-program';
+  if (RANGE_MESSAGES.test(message)) return 'input-causes-abend-subscript-range';
   return 'input-causes-abend';
 }
 
@@ -208,10 +218,10 @@ export function loadAbendRun(dir, tree) {
     const where = isString(place.file) ? posix.normalize(posix.join(rootDir ?? posix.dirname(programFile), place.file.replace(/\\/g, '/'))) : programFile;
     const line = Number.isInteger(place.line) && place.line > 0 ? place.line : 1;
     const rule = abendRule(run.abend);
-    // An S806 is input choosing the program only where the run's own journal records the marker
-    // fuzz put in the input reaching that CALL's program load.
+    // A CALL of a missing program is input choosing the program only where the run's own journal
+    // records the marker fuzz put in the input reaching that CALL's program load.
     if (rule === 'input-selects-program' && !recorded.sinks.some((k) => k.sink === 'dynamic-program-load' && k.reached === true && k.file === place.file && k.line === place.line)) {
-      out.problems.push(`run ${run.journal}: its journal does not record the input reaching the CALL at ${place.file}:${place.line}, so its S806 is not reported`);
+      out.problems.push(`run ${run.journal}: its journal does not record the input reaching the CALL at ${place.file}:${place.line}, so its missing-program ending is not reported`);
       continue;
     }
     const used = (Array.isArray(run.input) ? run.input : []).map((id) => inputs.get(id)).filter(Boolean)

package/lib/sets/cics.mjs

@@ -7,6 +7,7 @@ import { parseSource } from '../parser.mjs';
 import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { buildControl, hasFact } from '../control.mjs';
 import { parseCsd } from '../csd.mjs';
+import { execReading } from '../exec-reading.mjs';
 
 
 // `impact` says who can do what once the construct is present, so a reader can tell a defect they
@@ -113,27 +114,6 @@ export function signOnTest(test) {
 }
 
 
-function execOptions(exec) {
-  const opts = new Map();
-  const words = [];
-  const toks = exec.toks;
-  for (let i = 0; i < toks.length; i++) {
-    const t = toks[i];
-    if (t.t !== 'word') continue;
-    words.push(t.u);
-    if (toks[i + 1] && toks[i + 1].t === 'sep' && toks[i + 1].v === '(') {
-      const inner = [];
-      let depth = 1;
-      for (let k = i + 2; k < toks.length && depth > 0; k++) {
-        if (toks[k].t === 'sep') { depth += toks[k].v === '(' ? 1 : -1; if (depth === 0) break; continue; }
-        inner.push(toks[k]);
-      }
-      if (!opts.has(t.u)) opts.set(t.u, inner);
-    }
-  }
-  return { opts, words };
-}
-
 const subtreeRead = (x) => x.directRefs > 0 || x.children.some(subtreeRead);
 
 // Whether EIBCALEN bounds what the program reads: a condition tests it, or a field it was moved or
@@ -164,8 +144,8 @@ function testsLength(p) {
 // Commands whose failure a program ordinarily lets pass: removing a temporary-storage queue that may
 // not exist, a line written to a transient-data log, the clock, a screen sent to a terminal that may
 // have gone, an ASSIGN, and the RETURN that ends the task anyway.
-const QUIET_FAILURES = (words) => (words[0] === 'DELETEQ' && words[1] === 'TS') || (words[0] === 'WRITEQ' && words[1] === 'TD')
-  || ['ASKTIME', 'FORMATTIME', 'SEND', 'ASSIGN', 'RETURN'].includes(words[0]);
+const QUIET_FAILURES = ({ verb, sub }) => (verb === 'DELETEQ' && sub === 'TS') || (verb === 'WRITEQ' && sub === 'TD')
+  || ['ASKTIME', 'FORMATTIME', 'SEND', 'ASSIGN', 'RETURN'].includes(verb);
 const SHOWN_IGNORED = 5;
 
 // Commands whose outcome the program cannot see: NOHANDLE with no RESP and no EIBRESP test before the
@@ -175,15 +155,16 @@ function conditionsIgnored(p, path) {
   const execs = p.execs.filter((e) => e.kind === 'CICS').sort((a, b) => a.line - b.line);
   const hits = [];
   execs.forEach((e, k) => {
-    const { opts: o, words } = execOptions(e);
-    if (words[0] === 'IGNORE' && words[1] === 'CONDITION') {
+    const reading = execReading(e);
+    const { opts: o, words, verb, sub } = reading;
+    if (verb === 'IGNORE' && sub === 'CONDITION') {
       hits.push({ e, what: `IGNORE CONDITION ${words.slice(2).join(' ')}`.trim() });
       return;
     }
-    if (!words.includes('NOHANDLE') || o.has('RESP') || QUIET_FAILURES(words)) return;
+    if (!words.includes('NOHANDLE') || o.has('RESP') || QUIET_FAILURES(reading)) return;
     const until = execs[k + 1] ? execs[k + 1].line : Infinity;
     const tested = p.statements.some((st) => st.line > e.line && st.line < until && (st.sources || []).some((t) => t.t === 'word' && t.u === 'EIBRESP'));
-    if (!tested) hits.push({ e, what: `${words[0]}${words[1] && words[1] !== 'NOHANDLE' && !o.has(words[1]) ? ` ${words[1]}` : ''} NOHANDLE` });
+    if (!tested) hits.push({ e, what: `${verb}${sub && sub !== 'NOHANDLE' && !o.has(sub) ? ` ${sub}` : ''} NOHANDLE` });
   });
   if (!hits.length) return null;
   const shown = hits.slice(0, SHOWN_IGNORED).map((h) => `${h.what} at line ${h.e.line}`).join(', ');
@@ -226,8 +207,7 @@ export function* scanCicsSteps(root, opts = {}) {
     const transfers = [];
     for (const e of p.execs) {
       if (e.kind !== 'CICS') continue;
-      const { opts: o, words: w } = execOptions(e);
-      const verb = w[0];
+      const { opts: o, verb } = execReading(e);
       if (!['LINK', 'XCTL', 'START'].includes(verb)) continue;
       const pgm = (o.get('PROGRAM') || o.get('TRANSID') || [])[0];
       const areaTok = (o.get('COMMAREA') || o.get('FROM') || [])[0];
@@ -275,7 +255,7 @@ export function* scanCicsSteps(root, opts = {}) {
   // asking the security manager, and one that folds a password's case first. Passwords are recognised
   // by field name, as the sign-on rules above recognise them, and each finding says so.
   function passwordChecks(p, path) {
-    if (p.execs.some((e) => e.kind === 'CICS' && /^(VERIFY|SIGNON)$/.test(execOptions(e).words[0] || ''))) return [];
+    if (p.execs.some((e) => e.kind === 'CICS' && /^(VERIFY|SIGNON)$/.test(execReading(e).verb || ''))) return [];
     const byName = new Map();
     for (const it of p.items) if (!byName.has(it.name)) byName.set(it.name, it);
     const resolve = (tok) => (tok && tok.t === 'word' ? p.resolved.get(tok) || byName.get(tok.u) || null : null);
@@ -286,8 +266,8 @@ export function* scanCicsSteps(root, opts = {}) {
     // FETCH fills, and everything under any of them. A copy made by MOVE first is not followed.
     const read = new Set();
     for (const e of p.execs) {
-      const { opts: o, words } = execOptions(e);
-      if (e.kind === 'CICS' && /^READ(NEXT|PREV)?$/.test(words[0] || '')) for (const t of o.get('INTO') || []) if (t.t === 'word') read.add(t.u);
+      const { opts: o, verb } = execReading(e);
+      if (e.kind === 'CICS' && /^READ(NEXT|PREV)?$/.test(verb || '')) for (const t of o.get('INTO') || []) if (t.t === 'word') read.add(t.u);
       if (e.kind === 'SQL' && (e.hostVariables || []).length) for (const h of e.hostVariables) if (h.written && h.tok) read.add(h.tok.u);
     }
     const stored = (name) => {
@@ -343,15 +323,15 @@ export function* scanCicsSteps(root, opts = {}) {
     const sends = [];
     for (const e of p.execs) {
       if (e.kind !== 'CICS') continue;
-      const { opts: o, words: w } = execOptions(e);
-      const opt = w[0] === 'XCTL' || w[0] === 'LINK' ? 'PROGRAM' : (w[0] === 'RETURN' || w[0] === 'START') ? 'TRANSID' : null;
+      const { opts: o, verb } = execReading(e);
+      const opt = verb === 'XCTL' || verb === 'LINK' ? 'PROGRAM' : (verb === 'RETURN' || verb === 'START') ? 'TRANSID' : null;
       const t = opt && (o.get(opt) || [])[0];
       if (!t || t.t !== 'lit') continue;
       const target = `${opt === 'PROGRAM' ? '' : 'transaction '}${String(t.v).trim().toUpperCase()}`;
       if (target === p.id) continue;
       const bits = ctl.facts.get(ctl.nodeOf.get(e));
       if (!bits) continue;
-      sends.push({ e, verb: w[0], target, signedOn: hasFact(bits, bit) });
+      sends.push({ e, verb, target, signedOn: hasFact(bits, bit) });
     }
     const granted = new Set(sends.filter((x) => x.signedOn).map((x) => x.target));
     const aid = p.refs.some((r) => r.tok.u === 'EIBAID');

package/lib/sets/compile.mjs

@@ -59,11 +59,13 @@ const REPLACE_STATEMENT = /\bREPLACE\b([\s\S]*?)(?:\.\s|\.$)/gim;
 
 // A header declares a paragraph only where the sentence before it has ended: after a line of code
 // ending in a period, or with no code before it. Without the period the name is read as part of the
-// statement before, and the compiler reports it undefined.
-function sentenceEndsBefore(text, at) {
-  const lines = text.slice(0, at).split(/\r?\n/);
-  lines.pop();
-  for (let i = lines.length - 1; i >= 0; i--) {
+// statement before, and the compiler reports it undefined. `lines` and `starts` are the text's lines
+// and where each begins.
+function sentenceEndsBefore(lines, starts, at) {
+  let lo = 0;
+  let hi = starts.length - 1;
+  while (lo < hi) { const mid = (lo + hi + 1) >>> 1; if (starts[mid] <= at) lo = mid; else hi = mid - 1; }
+  for (let i = lo - 1; i >= 0; i--) {
     const l = lines[i];
     if (/^.{6}[*/]/.test(l) || /^\s*\*>/.test(l)) continue;
     const code = l.slice(0, 72).replace(/\*>.*$/, '').replace(/^[0-9 ]{6}(?=[ \-D])/, '').trim();
@@ -72,12 +74,17 @@ function sentenceEndsBefore(text, at) {
   return true;
 }
 
-function declaredIn(text, names, into) {
-  const note = (w) => { const u = w.toUpperCase(); if (names.has(u)) into.add(u); };
+// Every name a text declares, upper-cased, whatever is being looked for.
+function declaredWords(text) {
+  const words = new Set();
+  const note = (w) => words.add(w.toUpperCase());
+  const lines = text.split(/\r?\n/);
+  const starts = [0];
+  for (const m of text.matchAll(/\r?\n/g)) starts.push(m.index + m[0].length);
   for (const re of [DECLARATION, HEADER]) {
     re.lastIndex = 0;
     let m;
-    while ((m = re.exec(text))) if (re !== HEADER || sentenceEndsBefore(text, m.index)) note(m[1]);
+    while ((m = re.exec(text))) if (re !== HEADER || sentenceEndsBefore(lines, starts, m.index)) note(m[1]);
   }
   for (const re of [COPY_STATEMENT, REPLACE_STATEMENT]) {
     re.lastIndex = 0;
@@ -87,9 +94,11 @@ function declaredIn(text, names, into) {
       for (const w of m[1].match(/[A-Za-z0-9][A-Za-z0-9-]*/g) || []) note(w);
     }
   }
-  return into;
+  return words;
 }
 
+const declaredIn = (words, names, into) => { for (const u of names) if (words.has(u)) into.add(u); return into; };
+
 // A reference cut at column 72 names a word the source does not hold.
 function truncated(ref, lines) {
   const line = (lines.get(ref.file) || [])[ref.line - 1] || '';
@@ -114,7 +123,23 @@ export function* scanCompileSteps(root, opts = {}) {
     because[why] = (because[why] || 0) + 1;
     undecided.push(`${tree.rel(f)} (${id}): ${why}: ${[...new Set(refs.map((x) => x.name))].slice(0, SHOWN).join(', ')}`);
   };
-  const textOf = (p) => (tree.contains(p) ? tree.text(p) : readSource(p)).text;
+  // A copybook's text, lines and declared names, read once for every program that copies it.
+  const remembered = (read) => {
+    const held = new Map();
+    return (p) => {
+      let h = held.get(p);
+      if (!h) {
+        try { const text = read(p); h = { text, lines: null, words: null }; } catch (e) { h = { error: e }; }
+        held.set(p, h);
+      }
+      if (h.error) throw h.error;
+      return h;
+    };
+  };
+  const copyOf = remembered((p) => (tree.contains(p) ? tree.text(p) : readSource(p)).text);
+  const answerOf = remembered((p) => tree.text(p).text);
+  const wordsOf = (h) => (h.words ||= declaredWords(h.text));
+  const linesOf = (h) => (h.lines ||= h.text.split(/\r?\n/));
   const candidates = [];
 
   const run = yield loopOver(files, (f) => {
@@ -126,6 +151,7 @@ export function* scanCompileSteps(root, opts = {}) {
     stats.filesScanned++;
     const blocked = whyUndecided(r, src);
     const copied = [...new Set(r.copies.filter((c) => c.path).map((c) => c.path))];
+    const own = { text: src, lines: null, words: null };
     for (const p of r.programs) {
       // A class or a method has no PROGRAM-ID, and the data its methods use belongs to no program here.
       if (!p.id || p.diags.some((d) => d.kind === 'no-procedure-division')) continue;
@@ -133,11 +159,11 @@ export function* scanCompileSteps(root, opts = {}) {
       if (!p.unresolvedRefs.length) continue;
       if (blocked) { undecide(f, p.id, blocked, p.unresolvedRefs); continue; }
       const names = new Set(p.unresolvedRefs.map((x) => x.name));
-      const texts = new Map([[f, src]]);
-      for (const q of copied) { try { texts.set(q, textOf(q)); } catch { /* the parse read it once; a second failure changes nothing */ } }
+      const texts = new Map([[f, own]]);
+      for (const q of copied) { try { texts.set(q, copyOf(q)); } catch { /* the parse read it once; a second failure changes nothing */ } }
       const seen = new Set();
-      for (const text of texts.values()) declaredIn(text, names, seen);
-      const lines = new Map([...texts].map(([q, t]) => [q, t.split(/\r?\n/)]));
+      for (const h of texts.values()) declaredIn(wordsOf(h), names, seen);
+      const lines = new Map([...texts].map(([q, h]) => [q, linesOf(h)]));
       const left = p.unresolvedRefs.filter((x) => !seen.has(x.name) && !truncated(x, lines));
       if (!left.length) { undecide(f, p.id, 'the parse missed a declaration the text holds', p.unresolvedRefs); continue; }
       candidates.push({ file: f, id: p.id, line: p.line, refs: left, copied: new Set(r.copies.map((c) => nameOf(c.name))) });
@@ -160,7 +186,7 @@ export function* scanCompileSteps(root, opts = {}) {
   for (const c of candidates) {
     const names = new Set(c.refs.map((x) => x.name));
     const seen = new Set();
-    for (const n of c.copied) for (const p of answering.get(n) || []) { try { declaredIn(tree.text(p).text, names, seen); } catch { /* unread declares nothing */ } }
+    for (const n of c.copied) for (const p of answering.get(n) || []) { try { declaredIn(wordsOf(answerOf(p)), names, seen); } catch { /* unread declares nothing */ } }
     const left = c.refs.filter((x) => !seen.has(x.name));
     if (!left.length) { undecide(c.file, c.id, 'another copybook of a name it copies declares them', c.refs); continue; }
     const firsts = new Map();

package/lib/sets/crypto.mjs

@@ -11,7 +11,7 @@ import { readFileSync } from 'node:fs';
 import { inScope, isProgram, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { constantsReaching, keywordsOf } from '../constants.mjs';
 
 const TABLE = JSON.parse(readFileSync(new URL('../../rules/icsf-services.json', import.meta.url), 'utf8'));
@@ -54,13 +54,13 @@ const argument = (call, service, parameter) => {
 };
 const shown = (values) => [...new Set(values.map((v) => (/^[A-Z-]+$/.test(v) ? v : `'${v.trim()}'`)))].join(', ');
 
-export function scanCrypto(root, opts = {}) {
+export function* scanCryptoSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const files = tree.list().filter(isProgram).filter(inScope(opts));
   const findings = [];
   const stats = { filesScanned: 0, filesUnreadable: 0, filesUnparsed: 0, icsfCalls: 0, argumentsUndecided: 0 };
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     if (!/CALL\s+['"]?CS[NF]/i.test(src)) return src.length;
@@ -108,3 +108,5 @@ export function scanCrypto(root, opts = {}) {
 
   return report('crypto', { rules: CRYPTO_RULES, findings, stats, run });
 }
+
+export const scanCrypto = (root, opts = {}) => drive(scanCryptoSteps(root, opts));