@portll/cobolwork 0.5.0 → 0.7.0

package/lib/sets/hlasm.mjs

@@ -6,16 +6,21 @@
 //
 // It also answers the question the COBOL side cannot: a CALL 'SUBRTN' with no COBOL program of that
 // name may be an assembler module, and a CSECT or ENTRY of that name here is where it is.
-import { inScope, isAssembler, isProgram, relPath } from '../sources.mjs';
+import { basename, extname } from 'node:path';
+import { inScope, isAssembler, isJcl, isProgram, relPath, HLASM_EXT, COPY_EXT } from '../sources.mjs';
+import { parseJcl } from '../jcl.mjs';
+import { optableIn, processOptions } from '../hlasm/optable.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
-import { readHlasm, OPERATIONS } from '../hlasm.mjs';
+import { readHlasm, OPERATIONS, isDocument } from '../hlasm.mjs';
+import { readHlasmStatements, parseHlasmStatement, NAMED_MACROS } from '../hlasm/read.mjs';
+import { macroOperands, splitOperands } from '../hlasm/operands.mjs';
 
 export const HLASM_RULES = {
   'hlasm-supervisor-state-change': {
     sev: 'crit', evidence: 'construct', cwe: 'CWE-250',
-    text: 'Assembler source switches to PSW key zero or supervisor state',
+    text: 'Assembler source switches to PSW key zero, another system key, or supervisor state',
     impact: 'The code after the switch runs with the operating system\'s authority: it can read and change any storage, so a flaw in it is a flaw in the system rather than in one job',
     remedy: 'Keep the authorized code to the instructions that need it, return to the caller\'s key and problem state straight after (MODESET KEY=NZERO, MODE=PROB), and APF-authorize the module only where it has to be',
   },
@@ -31,6 +36,12 @@ export const HLASM_RULES = {
     impact: 'A program call, a program transfer or a change of secondary address space moves execution or data access across address spaces, under the authority the linkage tables grant rather than the caller\'s own',
     remedy: 'Confirm the entry tables and authorization indexes grant only the address spaces and keys the service needs, and that every PC routine validates what its caller passes',
   },
+  'hlasm-runtime-module-name': {
+    sev: 'med', evidence: 'construct', cwe: 'CWE-470',
+    text: 'Assembler source loads or calls a module whose name it reads from storage at run time',
+    impact: 'EPLOC= and DE= give LINK, XCTL, LOAD and ATTACH the address of the name rather than the name, so whatever writes that storage chooses the program that runs, with the caller\'s authority',
+    remedy: 'Name the module with EP= where it is fixed; where it must vary, check the name against a list of the modules the program may run before the call',
+  },
   'hlasm-calls-security-product': { sev: 'info', evidence: 'context', cwe: null, text: 'Assembler source calls the security product directly' },
   'hlasm-provides-called-module': { sev: 'info', evidence: 'context', cwe: null, text: 'An assembler CSECT or ENTRY is a module a COBOL program in this tree CALLs' },
 };
@@ -45,8 +56,17 @@ const STATE = { KEY: 'PSW key zero', EXTKEY: 'PSW key zero', MODE: 'supervisor s
 
 const where = (use) => (use.inMacro ? ' It is in a macro definition, so it is in every program that uses the macro.' : '');
 
-function detail(use) {
-  const written = `${use.name}${use.field ? ` ${use.field}` : ''}`;
+// EX and EXRL name their target instruction; when it is labelled in the same file, the finding says
+// which instruction runs with the register's length or mask.
+function executed(use, labels) {
+  if (use.class !== 'built-instruction') return '';
+  const target = (splitOperands(use.field || '')[1] || '').trim().toUpperCase();
+  const t = /^[A-Z$#@_][A-Z0-9$#@_]*$/.test(target) ? labels.get(target) : null;
+  return t ? ` runs ${t.operation}${t.field ? ` ${t.field}` : ''}` : '';
+}
+
+function detail(use, labels) {
+  const written = `${use.name}${use.field ? ` ${use.field}` : ''}${executed(use, labels)}`;
   if (use.class === 'security-product') {
     const request = /\bREQUEST=([A-Z]+)/i.exec(use.field || '');
     return `${request ? `${use.name} REQUEST=${request[1].toUpperCase()}` : use.name} calls the security product from assembler rather than through a service that checks on the program's behalf.${where(use)}`;
@@ -67,31 +87,122 @@ function cobolCalls(source) {
   return calls;
 }
 
+// The repository's own definitions of macros the reader knows by name (IF, LINK, SAVE, ...): a
+// member so named whose first statement is MACRO. Only those names are read, so the cost is a few
+// files at most.
+function repositoryMacros(tree, files) {
+  const found = new Set();
+  for (const f of files) {
+    const name = basename(f, extname(f)).toUpperCase();
+    if (!NAMED_MACROS.has(name) || found.has(name)) continue;
+    let text;
+    try { text = tree.text(f).text; } catch { continue; }
+    const first = readHlasmStatements(text).statements.find((s) => s.kind === 'statement');
+    if (first?.operation === 'MACRO') found.add(name);
+  }
+  return found;
+}
+
+// The OPTABLE the estate's assembly steps give in PARM (ASMA90, the old IEV90, or a procedure such as
+// ASMACL), with where each was written. One level is the estate's; several leave each file to its own.
+function jclOptables(root, tree, files) {
+  const levels = new Map();
+  for (const f of files) {
+    let job;
+    try { job = parseJcl(tree.text(f).text, f); } catch { continue; }
+    for (const step of job.steps) {
+      if (!/^(?:ASMA90|IEV90)$/i.test(step.pgm || '') && !/^(?:ASMA|HLASM|IEV)/i.test(step.proc || '')) continue;
+      for (const [k, v] of step.keywords) {
+        const found = /^PARM(?:\.|$)/.test(k) ? optableIn(v) : null;
+        if (found && !levels.has(found.level)) levels.set(found.level, `${relPath(root, f)}:${step.line}`);
+      }
+    }
+  }
+  return levels;
+}
+
 export function scanHlasm(root, opts = {}) {
   const tree = treeFor(root, opts);
   const all = tree.list().filter(inScope(opts));
   const findings = [];
   const stats = { filesScanned: 0, filesUnreadable: 0, assemblerFiles: 0, bmsFiles: 0, imsFiles: 0, unrecognisedFiles: 0, readNotAssembled: true };
+  const members = new Set(all.filter((f) => [...HLASM_EXT, ...COPY_EXT].some((e) => f.toLowerCase().endsWith(e))).map((f) => basename(f, extname(f)).toUpperCase()));
+  const libraryMacros = repositoryMacros(tree, all.filter(isAssembler));
+  const forms = { mvs38: opts.mvs38Forms !== false };
+  const fromJcl = opts.hlasmOptable ? new Map() : jclOptables(root, tree, all.filter(isJcl));
+  const estateOptable = opts.hlasmOptable ?? (fromJcl.size === 1 ? [...fromJcl.keys()][0] : null);
+  if (opts.hlasmOptable) stats.hlasmOptable = { level: opts.hlasmOptable, from: 'option' };
+  else if (fromJcl.size === 1) stats.hlasmOptable = { level: estateOptable, from: [...fromJcl.values()][0] };
+  else if (fromJcl.size > 1) stats.hlasmOptableConflict = Object.fromEntries(fromJcl);
+  // The table a file is assembled with: the option given to the scan, then a *PROCESS OVERRIDE in
+  // the file, the estate's PARM, and the file's own *PROCESS, as HLASM ranks them.
+  const optableOf = (src) => {
+    if (opts.hlasmOptable) return opts.hlasmOptable;
+    const own = optableIn(processOptions(src));
+    return own?.override ? own.level : estateOptable ?? own?.level ?? null;
+  };
+  const refused = {};
+  const macroCalls = new Map();
+  const copyNotFound = new Set();
+  // Reads a file's statements: its labels, what the reader refused, and the macros and COPY members it
+  // uses, which count toward the coverage report unless the file is a document.
+  function readStatements(src, path) {
+    const labels = new Map();
+    const read = readHlasmStatements(src, { libraryMacros }).statements;
+    const statements = read.filter((st) => !st.inMacro);
+    const fileForms = { ...forms, optable: optableOf(src) };
+    const results = statements.map((st) => parseHlasmStatement(st, fileForms));
+    const refusals = results.filter((res) => res.status === 'unparsed' || res.status === 'unknown').length;
+    if (isDocument(read.filter((st) => st.kind === 'statement'), statements.length, refusals)) return null;
+    statements.forEach((st, i) => {
+      const res = results[i];
+      if (st.kind === 'statement' && st.name && !st.name.startsWith('&') && !st.name.startsWith('.')) labels.set(st.name.toUpperCase(), st);
+      if (res.node?.mvs38) stats.mvs38Forms = (stats.mvs38Forms || 0) + 1;
+      if (res.status === 'unparsed' || res.status === 'unknown') (refused[res.kind] ||= { count: 0, first: `${path}:${st.line}` }).count++;
+      const name = res.kind === 'MACRO CALL' ? st.operation : NAMED_MACROS.has(res.kind) ? res.kind : null;
+      if (name) {
+        const m = macroCalls.get(name) || { name, calls: 0, definedInTree: members.has(name) };
+        m.calls++;
+        macroCalls.set(name, m);
+      }
+      if (res.kind === 'COPY') {
+        const member = String(res.status === 'parsed' ? res.node.member : (/^\s*([A-Z$#@][A-Z0-9$#@_]*)/i.exec(st.field || '') || [])[1] || '').toUpperCase();
+        if (member && !members.has(member)) copyNotFound.add(member);
+      }
+    });
+    return labels;
+  }
   const defines = new Map();
 
   const run = eachWithinMemory(all.filter(isAssembler), (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     stats.filesScanned++;
-    const r = readHlasm(src);
+    const r = readHlasm(src, forms);
     if (r.kind === 'bms') { stats.bmsFiles++; return src.length; }
     if (r.kind === 'ims') { stats.imsFiles++; return src.length; }
     if (r.kind === 'unrecognised') { stats.unrecognisedFiles++; return src.length; }
-    stats.assemblerFiles++;
+    if (r.kind === 'listing') { stats.listingFiles = (stats.listingFiles || 0) + 1; return src.length; }
     const path = relPath(root, f);
+    const labels = readStatements(src, path);
+    if (!labels) { stats.documentFiles = (stats.documentFiles || 0) + 1; return src.length; }
+    stats.assemblerFiles++;
     for (const d of r.defines) if (!defines.has(d.name)) defines.set(d.name, { ...d, path });
     for (const use of r.operations) {
+      if (use.class === 'module-load') {
+        const { keywords } = macroOperands(use.field || '');
+        const at = keywords.has('EPLOC') ? 'EPLOC' : keywords.has('DE') ? 'DE' : null;
+        if (at) findings.push({ rule: 'hlasm-runtime-module-name', path, line: use.line, detail: `${use.name} ${at}=${keywords.get(at)} names the module by the address of its name, so the program that runs is whatever name that storage holds.${where(use)}` });
+        continue;
+      }
       if (use.name === 'MODESET') {
         if (use.stateChange) findings.push({ rule: 'hlasm-supervisor-state-change', path, line: use.line, detail: `MODESET ${use.stateChange} switches to ${STATE[use.stateChange.split('=')[0]]}.${where(use)}` });
+        // A system key writes storage problem-state keys cannot, short of key zero's reach.
+        else if (use.systemKey) findings.push({ rule: 'hlasm-supervisor-state-change', sev: 'high', path, line: use.line, detail: `MODESET ${use.systemKey.written} switches to PSW key ${use.systemKey.key}, a system key.${where(use)}` });
         continue;
       }
       const rule = RULE_OF_CLASS[use.class];
-      if (rule) findings.push({ rule, path, line: use.line, detail: detail(use) });
+      if (rule) findings.push({ rule, path, line: use.line, detail: detail(use, labels) });
     }
     return src.length;
   }, { label: 'hlasm', maxBytes: opts.maxSourceBytes ?? Infinity });
@@ -117,5 +228,10 @@ export function scanHlasm(root, opts = {}) {
     }
   }
 
+  // Each statement the reader refused, each macro whose expansion was not read and each COPY member
+  // the tree does not hold is named: an operation inside any of them is one this set did not see.
+  if (Object.keys(refused).length) { stats.statementsNotRead = refused; stats.coverageIncomplete = true; }
+  if (copyNotFound.size) { stats.copyNotFound = [...copyNotFound].sort(); stats.coverageIncomplete = true; }
+  if (macroCalls.size) stats.macrosNotExpanded = [...macroCalls.values()].sort((a, b) => b.calls - a.calls || a.name.localeCompare(b.name)).slice(0, 50);
   return report('hlasm', { rules: HLASM_RULES, findings, stats, run });
 }

package/lib/sets/ims.mjs

@@ -0,0 +1,139 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// IMS database and program definitions: what a PSB lets a program change, SENSEGs that name segments
+// their DBD does not define, and what DBDGEN, PSBGEN or ACBGEN refuses (lib/ims/rules.mjs). DBDs and
+// PSBs are assembler macros and are often named .asm, so assembler files are read too, and kept only
+// where they hold a DBD or PSB.
+import { inScope, isAssembler, isIms, isJcl, isProgram, relPath } from '../sources.mjs';
+import { report } from '../kernel/ruleset.mjs';
+import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
+import { eachWithinMemory } from '../kernel/memory.mjs';
+import { modelOf } from '../ims/model.mjs';
+import { IMS_RULES, checkModels, checkProcoptAgainstCalls } from '../ims/rules.mjs';
+import { DLI_ROUTINE, DLI_FUNCTIONS, DLI_SERVICES, psbOfParm } from '../ims/dli.mjs';
+import { parseJcl } from '../jcl.mjs';
+import { constantsReaching } from '../constants.mjs';
+import { execReading } from '../exec-reading.mjs';
+
+export { IMS_RULES };
+
+export function scanIms(root, opts = {}) {
+  const tree = treeFor(root, opts);
+  const files = tree.list().filter(inScope(opts)).filter((f) => isIms(f) || isAssembler(f));
+  const stats = { filesScanned: 0, filesUnreadable: 0, dbdFiles: 0, psbFiles: 0 };
+  const models = [];
+  const unread = {};
+  const run = eachWithinMemory(files, (f) => {
+    let src;
+    try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
+    const model = modelOf(src);
+    if (!model) return src.length;
+    stats.filesScanned++;
+    stats[model.kind === 'DBD' ? 'dbdFiles' : 'psbFiles']++;
+    const path = relPath(root, f);
+    for (const u of model.unread) (unread[u.operation || 'UNREADABLE'] ||= { count: 0, first: `${path}:${u.line}` }).count++;
+    models.push({ path, model });
+    return src.length;
+  }, { label: 'ims', maxBytes: opts.maxSourceBytes ?? Infinity });
+
+  // A statement the reader refused may hold a segment, a field or a PCB the rules never saw.
+  if (Object.keys(unread).length) { stats.statementsNotRead = unread; stats.coverageIncomplete = true; }
+
+  // Programs are read only when a PSB could be named for one, and only those that issue DL/I.
+  const programs = [];
+  const links = [];
+  let calls = null;
+  if (models.some(({ model }) => model.kind === 'PSB')) {
+    const sources = tree.list().filter(inScope(opts)).filter((f) => isProgram(f) || isJcl(f));
+    calls = eachWithinMemory(sources, (f) => {
+      let src;
+      try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
+      const path = relPath(root, f);
+      if (isJcl(f)) {
+        if (!/DFSRRC00/i.test(src)) return src.length;
+        let jcl;
+        try { jcl = parseJcl(src); } catch { return src.length; }
+        for (const step of jcl.steps || []) {
+          const l = String(step.pgm || '').toUpperCase() === 'DFSRRC00' ? psbOfParm(step.parm) : null;
+          if (l) links.push({ ...l, at: `${path}:${step.line}` });
+        }
+        return src.length;
+      }
+      if (!/CBLTDLI|AIBTDLI|EXEC\s+DLI/i.test(src)) return src.length;
+      let r;
+      try { r = tree.parse(f, src); } catch (e) { noteUnparsed(stats, tree, f, e); return src.length; }
+      for (const prog of r.programs) {
+        const read = dliCallsOf(prog);
+        const id = String(prog.id).toUpperCase();
+        programs.push({ id, path, line: prog.line || 1, functions: read.functions });
+        for (const psb of read.schedules) links.push({ program: id, psb, at: `${path}, its own scheduling call` });
+      }
+      return src.length;
+    }, { label: 'ims programs', maxBytes: opts.maxSourceBytes ?? Infinity });
+    stats.programsWithDli = programs.length;
+  }
+  const findings = [...checkModels(models), ...checkProcoptAgainstCalls(models, programs, links)];
+  return report('ims', { rules: IMS_RULES, findings, stats, run: calls ? bothRuns(run, calls) : run });
+}
+
+// The DL/I database functions a program issues itself, null when they cannot all be told: a function
+// code no constant decides, a CALL that passes on a PCB or a LINKAGE item, which may hold one, or a
+// CICS LINK or XCTL from a program that schedules a PSB, whose target can issue DL/I against it. And
+// the PSBs it schedules itself, by EXEC DLI SCHD or a PCB call, where a constant names one.
+function dliCallsOf(prog) {
+  let functions = new Set();
+  const schedules = new Set();
+  const ignore = { ignoreCall: (x) => DLI_ROUTINE.test(String(x.name)) };
+  const valuesOf = (a) => {
+    if (!a) return { values: [], open: true };
+    if (a.lit != null) return { values: [String(a.lit)], open: false };
+    return a.word ? constantsReaching(prog, a.word, ignore) : { values: [], open: true };
+  };
+  const named = (k) => (k.open ? null : k.values.map((v) => String(v).trim().toUpperCase()));
+  const known = (v) => Object.hasOwn(DLI_FUNCTIONS, v) || DLI_SERVICES.has(v);
+  const byName = new Map((prog.items || []).map((it) => [it.name, it]));
+  const topOf = (it) => { while (it && it.parent) it = it.parent; return it; };
+  // The function code is the first of the two arguments whose constants name one; the PCB or AIB follows it.
+  const functionAt = (using) => [0, 1].find((i) => valuesOf(using[i]).values.some((v) => known(String(v).trim().toUpperCase())));
+  const pcbs = new Set();
+  for (const c of prog.calls || []) {
+    if (c.kind !== 'L' || !DLI_ROUTINE.test(String(c.name))) continue;
+    const at = functionAt(c.using || []);
+    const pcb = at === undefined ? null : (c.using || [])[at + 1];
+    if (pcb && pcb.word) pcbs.add(pcb.word);
+  }
+  const passesOn = (a) => a.word && (pcbs.has(a.word) || (topOf(byName.get(a.word)) || {}).section === 'LINKAGE');
+  for (const c of prog.calls || []) {
+    if (c.kind !== 'L') { functions = null; continue; }
+    if (!DLI_ROUTINE.test(String(c.name))) {
+      if ((c.using || []).some(passesOn)) functions = null;
+      continue;
+    }
+    const using = c.using || [];
+    const at = functionAt(using);
+    const codes = at === undefined ? null : named(valuesOf(using[at]));
+    if (!codes || !codes.every(known)) { functions = null; continue; }
+    if (codes.includes('PCB')) for (const n of named(valuesOf(using[at + 1])) || []) schedules.add(n);
+    if (functions) for (const fn of codes) if (Object.hasOwn(DLI_FUNCTIONS, fn)) functions.add(fn);
+  }
+  for (const e of prog.execs || []) {
+    if (e.kind !== 'DLI') continue;
+    const { verb, opts } = execReading(e);
+    if (verb === 'SCHD') {
+      const tok = (opts.get('PSB') || []).find((t) => t.t === 'word' || t.t === 'lit');
+      const k = !tok ? { values: [], open: true } : tok.t === 'lit' ? { values: [tok.v], open: false } : constantsReaching(prog, tok.v, ignore);
+      for (const n of named(k) || []) schedules.add(n);
+    } else if (functions && Object.hasOwn(DLI_FUNCTIONS, verb)) functions.add(verb);
+  }
+  const transfers = (prog.execs || []).some((e) => e.kind === 'CICS' && ['LINK', 'XCTL'].includes(execReading(e).verb));
+  if (schedules.size && transfers) functions = null;
+  return { functions: functions && functions.size ? functions : null, schedules: [...schedules] };
+}
+
+function bothRuns(a, b) {
+  return {
+    skipped: [...a.skipped, ...b.skipped],
+    stoppedBy: a.stoppedBy || b.stoppedBy,
+    peakHeapBytes: Math.max(a.peakHeapBytes || 0, b.peakHeapBytes || 0) || undefined,
+    note: [a.note, b.note].filter(Boolean).join(' ') || undefined,
+  };
+}

package/lib/sets/log.mjs

@@ -23,7 +23,8 @@ import { parseJcl } from '../jcl.mjs';
 import { compileStepOptions } from '../options.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
+import { execReading } from '../exec-reading.mjs';
 
 
 export const LOG_RULES = {
@@ -107,17 +108,16 @@ function logSinks(p, where, path) {
   }
   for (const e of p.execs || []) {
     if (e.kind !== 'CICS') continue;
-    const w = e.toks.filter((t) => t.t === 'word').map((t) => t.u);
-    const isTd = w[0] === 'WRITEQ' && w[1] === 'TD';
-    const isJournal = w[0] === 'WRITE' && w.includes('JOURNALNAME');
+    const { verb, sub, words, opts } = execReading(e);
+    const isTd = verb === 'WRITEQ' && sub === 'TD';
+    const isJournal = verb === 'WRITE' && words.includes('JOURNALNAME');
     // The console, read by whoever is watching the region. A credential on it is as exposed as one
     // in SYSOUT, and it outlives the screen in the system log.
-    const isOperator = w[0] === 'WRITE' && w[1] === 'OPERATOR';
+    const isOperator = verb === 'WRITE' && sub === 'OPERATOR';
     if (!isTd && !isJournal && !isOperator) continue;
     // FROM names the area written, TEXT names the console message; that name is what this reads.
     const opt = isOperator ? 'TEXT' : 'FROM';
-    const i = e.toks.findIndex((t) => t.t === 'word' && t.u === opt);
-    const arg = i >= 0 ? e.toks.slice(i + 1).find((t) => t.t === 'word') : null;
+    const arg = (opts.get(opt) || []).find((t) => t.t === 'word') || null;
     const kind = isTd ? 'WRITEQ TD' : isOperator ? 'WRITE OPERATOR' : 'WRITE JOURNALNAME';
     if (arg) out.push({ kind, name: arg.u, ...where(e, path) });
   }
@@ -337,7 +337,7 @@ const memberOf = (file) => basename(String(file || ''), extname(String(file || '
 const runByJob = (batch, p, file) => batch.unsure
   || [String(p.id || '').toUpperCase(), memberOf(file)].some((n) => n && (batch.runs.has(n) || batch.runs.has(n.slice(0, 8))));
 
-export function scanLog(root, opts = {}) {
+export function* scanLogSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const files = tree.list().filter(isProgram).filter(inScope(opts));
   const findings = [];
@@ -364,7 +364,7 @@ export function scanLog(root, opts = {}) {
     return { rule: 'display-echoes-a-credential', detail: `${p.id} shows ${w.name} on the terminal to the person who typed it: not a log, but the screen and its scrollback keep it` };
   };
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     // Every EXEC CICS WRITE form, not the two this set started with. WRITE OPERATOR was added as a
@@ -404,3 +404,5 @@ export function scanLog(root, opts = {}) {
 
   return report('log', { rules: LOG_RULES, findings, stats, run });
 }
+
+export const scanLog = (root, opts = {}) => drive(scanLogSteps(root, opts));

package/lib/sets/opaque.mjs

@@ -74,21 +74,41 @@ export function scanOpaque(root, opts = {}) {
     const path = relPath(root, f);
 
     // Which names were declared as pointers, so a CALL through one can be told from an ordinary
-    // CALL of a variable - which the flow rules already report as a dynamic program load.
-    const pointers = new Set();
-    for (const m of text.matchAll(/\b(?:\d\d\s+)?([A-Z0-9-]+)\b[^.\n]{0,80}\bUSAGE\s+(?:IS\s+)?(?:PROCEDURE-POINTER|POINTER)\b/gi)) {
-      pointers.add(m[1].toUpperCase());
-    }
+    // CALL of a variable - which the flow rules already report as a dynamic program load. Found
+    // only once a CALL needs it, since the search tries every word of the program.
+    let pointers = null;
+    const pointerNames = () => {
+      if (pointers) return pointers;
+      pointers = new Set();
+      for (const m of text.matchAll(/\b(?:\d\d\s+)?([A-Z0-9-]+)\b[^.\n]{0,80}\bUSAGE\s+(?:IS\s+)?(?:PROCEDURE-POINTER|POINTER)\b/gi)) {
+        pointers.add(m[1].toUpperCase());
+      }
+      return pointers;
+    };
+
+    // How many newlines of the joined text come before an offset, which is the normalised entry
+    // holding it.
+    const starts = [0];
+    for (let i = text.indexOf('\n'); i >= 0; i = text.indexOf('\n', i + 1)) starts.push(i + 1);
+    const entryAt = (offset) => {
+      let lo = 0;
+      let hi = starts.length - 1;
+      while (lo < hi) {
+        const mid = (lo + hi + 1) >> 1;
+        if (starts[mid] <= offset) lo = mid; else hi = mid - 1;
+      }
+      return lo;
+    };
 
     let hit = false;
     for (const p of PATTERNS) {
       p.re.lastIndex = 0;
       let m;
       while ((m = p.re.exec(text)) !== null) {
-        if (p.needsPointerOperand && !pointers.has(String(m[1]).toUpperCase())) continue;
+        if (p.needsPointerOperand && !pointerNames().has(String(m[1]).toUpperCase())) continue;
         // The line number is the normalised entry's own, so a construct that came from a copybook
         // is reported where the compiler would see it.
-        const before = text.slice(0, m.index).split('\n').length - 1;
+        const before = entryAt(m.index);
         const line = norm.entries[before] ? norm.entries[before].line : 1;
         findings.push({ rule: p.rule, path, line, detail: p.detail(m) });
         hit = true;

package/lib/sets/pli.mjs

@@ -0,0 +1,40 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// PL/I programs read statement by statement (lib/pli): where they address storage through a pointer,
+// call through an entry variable or use the preprocessor, which a static reading cannot follow, and
+// ON units that swallow their condition. Data flow through PL/I is the flow set's; this set reports
+// what the program text itself shows, and counts every statement the reader could not parse.
+import { inScope, isPli, relPath } from '../sources.mjs';
+import { report } from '../kernel/ruleset.mjs';
+import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
+import { eachWithinMemory } from '../kernel/memory.mjs';
+import { readPli } from '../pli/lex.mjs';
+import { parseStatement } from '../pli/statements.mjs';
+import { PLI_RULES, checkProgram } from '../pli/rules/index.mjs';
+
+export { PLI_RULES };
+
+export function scanPli(root, opts = {}) {
+  const tree = treeFor(root, opts);
+  const files = tree.list().filter(inScope(opts)).filter(isPli);
+  const stats = { filesScanned: 0, filesUnreadable: 0, statements: 0, statementsParsed: 0 };
+  const notParsed = {};
+  const findings = [];
+  const run = eachWithinMemory(files, (f) => {
+    let src;
+    try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
+    const path = relPath(root, f);
+    const statements = readPli(src, { file: path }).statements.map((st) => ({ ...st, parsed: parseStatement(st) }));
+    stats.filesScanned++;
+    for (const st of statements) {
+      stats.statements++;
+      if (st.parsed.status === 'parsed') { stats.statementsParsed++; continue; }
+      (notParsed[st.parsed.kind || 'UNKNOWN'] ||= { count: 0, first: `${path}:${st.line}` }).count++;
+    }
+    findings.push(...checkProgram({ path, statements }));
+    return src.length;
+  }, { label: 'pli', maxBytes: opts.maxSourceBytes ?? Infinity });
+
+  // A statement the reader could not parse may hold a pointer, an entry call or an ON unit.
+  if (Object.keys(notParsed).length) { stats.statementsNotParsed = notParsed; stats.coverageIncomplete = true; }
+  return report('pli', { rules: PLI_RULES, findings, stats, run });
+}

package/lib/sets/priv.mjs

@@ -19,7 +19,7 @@
 import { inScope, isProgram, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { parseCsd } from '../csd.mjs';
 import { parseJcl } from '../jcl.mjs';
 import { loadSite } from '../site.mjs';
@@ -101,7 +101,7 @@ function under(dsn, prefixes) {
   });
 }
 
-export function scanPriv(root, opts = {}) {
+export function* scanPrivSteps(root, opts = {}) {
   const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts));
@@ -140,7 +140,7 @@ export function scanPriv(root, opts = {}) {
   const diagnostics = [];
   const uss = [];
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     const path = relPath(root, f);
@@ -320,3 +320,5 @@ export function scanPriv(root, opts = {}) {
 
   return report('priv', { rules: PRIV_RULES, findings, stats, run });
 }
+
+export const scanPriv = (root, opts = {}) => drive(scanPrivSteps(root, opts));

package/lib/sets/recon.mjs

@@ -14,7 +14,7 @@
 import { inScope, isJcl, isProgram, isCopybook, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { loadSite, classifyPath, productionQualifierOf, SITE_FILE } from '../site.mjs';
 import { parseJcl, dispositionOf } from '../jcl.mjs';
 import { statementCard } from '../cards.mjs';
@@ -161,7 +161,7 @@ function productionTouches(job, qualifiers) {
   return { found, claimed };
 }
 
-export function scanRecon(root, opts = {}) {
+export function* scanReconSteps(root, opts = {}) {
   const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts))
@@ -192,7 +192,7 @@ export function scanRecon(root, opts = {}) {
 
   // This set reads every program, copybook and job in the tree, which is the widest reading any
   // set does, so it is walked inside the guard like the rest.
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     stats.filesScanned++;
@@ -267,3 +267,5 @@ export function scanRecon(root, opts = {}) {
   // production means, as against files going unread. Both travel, and they are not merged.
   return report('recon', { rules: RECON_RULES, findings, stats, run });
 }
+
+export const scanRecon = (root, opts = {}) => drive(scanReconSteps(root, opts));