@portll/cobolwork 0.6.0 → 0.7.0

package/lib/dataflow.mjs

@@ -1,11 +1,19 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { inScope, isProgram, isBms, relPath } from './sources.mjs';
+import { inScope, isProgram, isBms, isPli, relPath } from './sources.mjs';
 import { parseBms, symbolicNames } from './bms.mjs';
 import { dirname, join } from 'node:path';
 import { parseSource, buildFileIndex } from './parser.mjs';
 import { parseJcl } from './jcl.mjs';
 import { holdsDecimal } from './layout.mjs';
 import { hostVariablesIn } from './embedded-sql.mjs';
+import { execReading } from './exec-reading.mjs';
+import { DLI_ROUTINE, dliFunction, isGet } from './ims/dli.mjs';
+import { CICS_EVERY_COMMAND } from './cics-commands.mjs';
+
+// RESP and RESP2, which every command returns and the response-code source already reads.
+const CICS_EVERY_COMMAND_OPTIONS = new Set(Object.keys(CICS_EVERY_COMMAND));
+import { parsePliSource } from './pli/program.mjs';
+import { membersOf, chooseMember } from './pli/include.mjs';
 import { tsoCommands } from './utilities.mjs';
 import { parseCsd, ddOfQueue } from './csd.mjs';
 import { loadSite } from './site.mjs';
@@ -14,12 +22,14 @@ import { isJcl } from './sources.mjs';
 import { watchMemoryBuffer } from './kernel/memory.mjs';
 import { drive, loopOver } from './kernel/shared-pass.mjs';
 import { keep, parseOrder, restore, sameAnalysis } from './control-reuse.mjs';
+import { CONTROL_WORKER_MIN_BYTES, controlAhead, controlWorkerCount, programFiles } from './control-workers.mjs';
+import { statSync } from 'node:fs';
 import { createHash } from 'node:crypto';
 
 // A flow finding is located by source and sink rather than by one path and line, so it keeps its
 // own comparator. The text comparison underneath it is the shared one.
 import { byText } from './kernel/findings.mjs';
-import { buildControl, creditOf, hasFact, sparseFacts, topOf, within, wholeTop } from './control.mjs';
+import { buildControl, creditOf, hasFact, internFacts, namesOf, topOf, within, wholeTop } from './control.mjs';
 import { directoryTree } from './kernel/source-tree.mjs';
 
 // Every reused control analysis checked against the program's own: for the equivalence runs and tests.
@@ -35,6 +45,7 @@ export const SOURCE_KINDS = {
   'cics-terminal': 'EXEC CICS RECEIVE',
   'cics-web': 'EXEC CICS WEB RECEIVE or EXTRACT',
   'file-record': 'record read from a file',
+  'cics-queue': 'an item EXEC CICS READQ TS or TD returns, or the data RETRIEVE returns from the START that began the task',
   'database': 'host variable filled by EXEC SQL',
   'jcl-parm': 'PARM= on a JCL EXEC statement',
   'jcl-instream': 'in-stream data on a JCL DD statement',
@@ -46,7 +57,7 @@ export const SINK_KINDS = {
   'dynamic-program-load': 'CALL whose target is a variable',
   'cics-dynamic-transfer': 'EXEC CICS LINK, XCTL or START with a variable program or transaction',
   'dynamic-sql': 'EXEC SQL EXECUTE IMMEDIATE or PREPARE from a host variable',
-  'dynamic-file-path': 'SELECT ... ASSIGN TO a variable',
+  'dynamic-file-path': 'SELECT ... ASSIGN TO a variable, or EXEC CICS READ, WRITE, DELETE or STARTBR with FILE or DATASET from a variable',
   'outbound-http': 'EXEC CICS WEB CONVERSE, WEB SEND on a client session, or the containers INVOKE SERVICE sends',
   'socket-send': 'CALL EZASOKET or EZACICAL with SEND, SENDTO or WRITE',
   'message-queue': 'CALL MQPUT or MQPUT1',
@@ -105,36 +116,6 @@ function consKey(v) {
   return `${v instanceof Set ? 'S' : parts ? 'A' : 'O'}[${keys.join(',')}]`;
 }
 
-// The facts a program keeps per statement: one copy of each distinct set, held as the numbers of the
-// facts in it, all in one buffer per program. Neighbouring statements mostly hold the same checks, and
-// few of a program's checks hold at any one statement: 26 large programs kept 1.5 GB of bitsets, 0.4%
-// of their bits set.
-function internFacts(facts) {
-  const byHash = new Map();
-  const kept = [];
-  const keptAt = new Map();
-  for (const [point, bits] of facts) {
-    let h = 2166136261;
-    for (let i = 0; i < bits.length; i++) h = Math.imul(h ^ bits[i], 16777619);
-    let same = byHash.get(h);
-    if (!same) byHash.set(h, same = []);
-    let k = same.find((x) => x.bits.length === bits.length && x.bits.every((v, i) => v === bits[i]));
-    if (!k) { same.push(k = { bits, held: sparseFacts(bits) }); kept.push(k); }
-    keptAt.set(point, k);
-  }
-  let size = 0;
-  for (const k of kept) if (k.held instanceof Uint16Array) size += k.held.length;
-  const pool = new Uint16Array(size);
-  let at = 0;
-  for (const k of kept) {
-    if (!(k.held instanceof Uint16Array)) continue;
-    pool.set(k.held, at);
-    k.held = pool.subarray(at, at += k.held.length);
-  }
-  for (const [point, k] of keptAt) facts.set(point, k.held);
-  return facts;
-}
-
 // Every literal a program holds that could be a program name: its VALUE clauses and the literals its
 // statements and commands use. A menu that XCTLs through a table of names starts every one of them.
 function literalNames(prog) {
@@ -186,18 +167,21 @@ function boundsOf(counter, until) {
 // nearly every subscript descends from a file record, and reporting those would be reporting batch.
 const FROM_OUTSIDE = ['argv-or-env', 'cics-terminal', 'cics-web', 'jcl-parm', 'jcl-instream'];
 // What can reach a CICS command: a batch job never issues one.
-const IN_CICS = ['cics-web', 'cics-terminal', 'file-record', 'database'];
+const IN_CICS = ['cics-web', 'cics-terminal', 'file-record', 'cics-queue', 'database'];
 const REFLECTED = ['cics-web', 'cics-terminal'];
 // What the system says about a failure: a command's RESP, the EIB's copy of it, the SQLCA. Sent to a
 // web client it describes the system behind the program (CWE-209); written anywhere else it is the
 // program's own business, so these reach only a response, and Db2's message text the terminal too.
 const SYSTEM_FIELDS = new Set(['EIBRESP', 'EIBRESP2', 'SQLCODE', 'SQLSTATE', 'SQLERRMC']);
+// PL/I's SQLCA holds Db2's message text in SQLERRM, a CHAR(70) VARYING, where COBOL's has SQLERRMC.
+const PLI_SYSTEM_FIELDS = new Set([...SYSTEM_FIELDS, 'SQLERRM']);
 const TO_CLIENT = ['web-response', 'http-header'];
 
 // Exfiltration is data at rest leaving the program. Terminal or web input reaching the same channel
 // is the program relaying its own caller's data, and pairing it would fire on every gateway.
 
 // A trace longer than this keeps its two ends and says how many hops it dropped.
+
 export const TRACE_MAX = 64;
 export const TRACE_KEEP = 24;
 // Edges one source's walk may examine, and all walks together: about fifteen times what the largest
@@ -205,7 +189,7 @@ export const TRACE_KEEP = 24;
 export const WALK_EDGES = 5_000_000;
 export const TOTAL_EDGES = 500_000_000;
 
-const DATA_AT_REST = ['database', 'file-record'];
+const DATA_AT_REST = ['database', 'file-record', 'cics-queue'];
 // The class tests a condition can make, which restrict what a field holds rather than compare it.
 const CLASS_TEST = new Set(['NUMERIC', 'ALPHABETIC', 'ALPHABETIC-LOWER', 'ALPHABETIC-UPPER', 'POSITIVE', 'NEGATIVE']);
 const SOCKET_ROUTINE = /^(EZASOKET|EZACICAL)$/i;
@@ -215,27 +199,6 @@ const SOCKET_SEND = /^(SEND|SENDTO|WRITE|SENDMSG|WRITEV)$/i;
 // MsgDesc, PutMsgOpts, BufferLength, Buffer, …): the buffer is the sixth argument of both.
 const MQ_BUFFER_ARG = { MQPUT: 5, MQPUT1: 5 };
 
-function execOptions(exec) {
-  const opts = new Map();
-  const toks = exec.toks;
-  const words = [];
-  for (let i = 0; i < toks.length; i++) {
-    const t = toks[i];
-    if (t.t !== 'word') continue;
-    words.push(t.u);
-    if (toks[i + 1] && toks[i + 1].t === 'sep' && toks[i + 1].v === '(') {
-      const inner = [];
-      let depth = 1;
-      for (let k = i + 2; k < toks.length && depth > 0; k++) {
-        if (toks[k].t === 'sep') { depth += toks[k].v === '(' ? 1 : -1; if (depth === 0) break; continue; }
-        inner.push(toks[k]);
-      }
-      if (!opts.has(t.u)) opts.set(t.u, inner);
-    }
-  }
-  return { opts, words };
-}
-
 export function* analyzeSteps(root, opts = {}) {
   const repos = opts.repos || [''];
   const stats = { files: 0, programs: 0, edges: 0, nodes: 0, threw: 0, overBudget: 0, unreadable: 0, ebcdic: 0,
@@ -335,16 +298,18 @@ export function* analyzeSteps(root, opts = {}) {
   const textKey = (src) => createHash('sha1').update(src).digest('base64');
   function repeatedTexts(tree, files) {
     const count = new Map();
+    const later = new Set();
     for (const f of files) {
       let src;
       try { src = tree.text(f).text; } catch { continue; }
       if (src.length < reuseMin) continue;
       const k = textKey(src);
+      if (count.has(k)) later.add(f);
       count.set(k, (count.get(k) || 0) + 1);
     }
-    const out = new Map();
-    for (const [k, n] of count) if (n > 1) out.set(k, { text: k, left: n, kept: new Map() });
-    return out;
+    const repeated = new Map();
+    for (const [k, n] of count) if (n > 1) repeated.set(k, { text: k, left: n, kept: new Map() });
+    return { repeated, later };
   }
   const copyText = new Map();
   const copyKey = (r, tree) => r.copies.map((c) => {
@@ -352,20 +317,21 @@ export function* analyzeSteps(root, opts = {}) {
     if (!copyText.has(c.path)) { let k = null; try { k = textKey(tree.text(c.path).text); } catch { /* unread: compared by status */ } copyText.set(c.path, k); }
     return `${c.status}:${copyText.get(c.path)}`;
   }).join('|');
-  const reuseOf = (copied, k, r, f, tree) => ({ kept: copied.kept, key: `${k}|${copyKey(r, tree)}`, files: [f, ...r.copies.filter((c) => c.path).map((c) => c.path)] });
-  // A reused analysis checked against the program's own, where COBOLWORK_VERIFY_REUSE asks.
-  function sameAsOwn(ctl, prog, resolver, file) {
+  const reuseOf = (copied, k, r, f, tree) => ({ kept: copied.kept, key: `${k}|${copyKey(r, tree)}`, files: programFiles(r, f) });
+  // A restored analysis checked against the program's own, where COBOLWORK_VERIFY_REUSE asks.
+  function sameAsOwn(ctl, prog, resolver, file, from) {
     let own = null;
-    try { own = buildControl(prog, resolver); } catch { /* the kept one threw too, or it differs */ }
+    try { own = buildControl(prog, resolver); } catch { /* the restored one threw too, or it differs */ }
     if (own) internFacts(own.facts);
-    if (!own !== !ctl || (own && !sameAnalysis(own, ctl, new Set(parseOrder(prog))))) throw new Error(`the control analysis reused for ${file} differs from its own`);
+    if (!own !== !ctl || (own && !sameAnalysis(own, ctl, new Set(parseOrder(prog))))) throw new Error(`the control analysis ${from} for ${file} differs from its own`);
   }
 
-  function summarise(prog, file, pk, options = [], reuse = null) {
+  // `built` is the program's analysis from a worker thread (lib/control-workers.mjs), with the files
+  // its references name.
+  function summarise(prog, file, pk, options = [], reuse = null, built = null) {
     programOf[pk] = { id: prog.id, file };
     const byItem = new Map();
-    const byName = new Map();
-    for (const it of prog.items) if (!byName.has(it.name)) byName.set(it.name, it);
+    const { byName, itemOfToken, indexNames, resolve: resolver } = namesOf(prog);
     const nodeOf = (item) => {
       let n = byItem.get(item);
       if (!n) {
@@ -394,10 +360,7 @@ export function* analyzeSteps(root, opts = {}) {
       }
       return n;
     };
-    // Names inside EXEC blocks never pass through the parser's reference resolution, so fall back to the name.
-    const itemOfToken = (tok) => (tok ? prog.resolved.get(tok) || (tok.t === 'word' ? byName.get(tok.u) || null : null) : null);
     // An INDEXED BY name is not a data item, so it needs a node of its own to carry taint.
-    const indexNames = new Set(prog.items.flatMap((it) => it.indexNames || []));
     const indexNodes = new Map();
     const indexNode = (name) => {
       let n = indexNodes.get(name);
@@ -412,10 +375,11 @@ export function* analyzeSteps(root, opts = {}) {
     // A system field the program never declares - the EIB's, or an SQLCA the precompiler supplies -
     // still holds a value, so it needs a node of its own, as an index name does.
     const systemNodes = new Map();
+    const systemFields = isPli(file) ? PLI_SYSTEM_FIELDS : SYSTEM_FIELDS;
     // A code on a terminal's error line is ordinary; Db2's message text names tables, columns and
     // constraints, so only SQLERRMC reaches the screen sink.
     const systemSource = (n, detail, at) => {
-      const onlyTo = /^SQLERRMC\b/.test(detail) ? [...TO_CLIENT, 'screen'] : TO_CLIENT;
+      const onlyTo = /^SQLERRMC?\b/.test(detail) ? [...TO_CLIENT, 'screen'] : TO_CLIENT;
       if (!n.sources.some((x) => x.kind === 'system-response')) addSource(n, { kind: 'system-response', onlyTo, noCredit: true, ...at, detail });
     };
     const systemNode = (tok) => {
@@ -434,30 +398,34 @@ export function* analyzeSteps(root, opts = {}) {
       if (it) return nodeOf(it);
       if (!tok || tok.t !== 'word') return null;
       if (indexNames.has(tok.u)) return indexNode(tok.u);
-      return SYSTEM_FIELDS.has(tok.u) ? systemNode(tok) : null;
+      return systemFields.has(tok.u) ? systemNode(tok) : null;
     };
     // The program's statement order, where its structure could be read. Without it a check is
     // credited as it always was, wherever its field is used, and never clears anything.
     let ctl = null;
-    const resolver = (tok) => itemOfToken(tok) || (tok && tok.t === 'word' && indexNames.has(tok.u) ? { index: tok.u } : null);
+    // A kept or worker-built analysis is { value }, { threw } or { none }, the last for a program
+    // with no procedure division, which is not counted as unordered.
     const kept = reuse && reuse.kept.get(reuse.key);
-    if (kept) {
-      ctl = kept.threw ? null : restore(kept.value, parseOrder(prog), reuse.files);
-      if (!ctl) stats.programsUnordered++;
-      if (verifyReuse()) sameAsOwn(ctl, prog, resolver, file);
+    const given = kept || (built && !built.local ? built : null);
+    if (given) {
+      ctl = given.value ? restore(given.value, parseOrder(prog), kept ? reuse.files : built.files) : null;
+      if (given.threw) stats.programsUnordered++;
+      if (verifyReuse()) sameAsOwn(ctl, prog, resolver, file, kept ? 'reused' : 'built in a worker');
+      if (reuse && !kept) reuse.kept.set(reuse.key, given);
     } else {
-      try { ctl = buildControl(prog, resolver); } catch { stats.programsUnordered++; }
+      let threw = false;
+      try { ctl = buildControl(prog, resolver); } catch { threw = true; stats.programsUnordered++; }
       if (ctl) internFacts(ctl.facts);
       if (reuse) {
         const value = ctl ? keep(ctl, parseOrder(prog), reuse.files) : null;
-        if (!ctl || value) reuse.kept.set(reuse.key, ctl ? { value } : { threw: true });
+        if (!ctl || value) reuse.kept.set(reuse.key, value ? { value } : threw ? { threw: true } : { none: true });
       }
     }
     const pointOf = (x) => (ctl ? ctl.nodeOf.get(x) : undefined);
     // Where a name inside a condition is read, which can know more than the statement does.
     const placeOf = (tok, st) => (ctl && ctl.posOf.get(tok)) ?? pointOf(st);
 
-    for (const it of prog.items) if (SYSTEM_FIELDS.has(it.name)) systemSource(nodeOf(it), `${it.name}, which the system sets`, { file: it.file || file, line: it.line });
+    for (const it of prog.items) if (systemFields.has(it.name)) systemSource(nodeOf(it), `${it.name}, which the system sets`, { file: it.file || file, line: it.line });
     // What a program in a CICS region DISPLAYs goes to its CESE and CESO queues, a log. Terminal or web
     // input written there unchecked can carry the control characters that start a line the program
     // did not write (CWE-117). A batch program's DISPLAY is its job's own log, which no terminal
@@ -512,15 +480,37 @@ export function* analyzeSteps(root, opts = {}) {
       const recordOf = (it) => { let top = it; while (top.parent) top = top.parent; return top; };
       for (const it of prog.items) { const top = recordOf(it); if (!byRecord.has(top)) byRecord.set(top, []); byRecord.get(top).push(it); }
       const extent = (it) => (it.offset != null && it.size != null ? [it.offset, it.offset + (it.contributes || it.size)] : null);
+      // Each record's items with a known extent, by where they start, so a check finds the items
+      // inside its field without testing every item of the record. They are then taken in the
+      // record's order, which is the order their nodes are made in.
+      const layouts = new Map();
+      const layoutOf = (top, items) => {
+        let l = layouts.get(top);
+        if (!l) {
+          const at = [];
+          items.forEach((it, i) => { if (it.level !== 88 && it.offset != null && it.size != null) at.push({ i, start: it.offset, end: it.offset + (it.contributes || it.size) }); });
+          at.sort((a, b) => a.start - b.start || a.i - b.i);
+          layouts.set(top, l = { at, position: new Map(items.map((it, i) => [it, i])) });
+        }
+        return l;
+      };
       for (const c of ctl.checks) {
         const entry = { x: c.x, t: c.t, f: c.f, tCons: sameCondition(c.tCons), fCons: sameCondition(c.fCons), file: c.file, line: c.line, item: c.field.index || c.field.name, ...(c.bounds ? { bounds: c.bounds } : {}), ...(c.mono ? { mono: true } : {}) };
         if (c.field.index) { (indexNode(c.field.index).checks ||= []).push(entry); continue; }
+        const top = recordOf(c.field);
+        const items = byRecord.get(top);
+        if (!items) { if (c.field.level !== 88) (nodeOf(c.field).checks ||= []).push(entry); continue; }
+        const { at, position } = layoutOf(top, items);
+        const inside = new Set();
+        if (c.field.level !== 88 && position.has(c.field)) inside.add(position.get(c.field));
         const span = extent(c.field);
-        for (const it of byRecord.get(recordOf(c.field)) || [c.field]) {
-          if (it.level === 88) continue;
-          const e = extent(it);
-          if (it === c.field || (span && e && e[0] >= span[0] && e[1] <= span[1])) (nodeOf(it).checks ||= []).push(entry);
+        if (span) {
+          let lo = 0;
+          let hi = at.length;
+          while (lo < hi) { const mid = (lo + hi) >>> 1; if (at[mid].start < span[0]) lo = mid + 1; else hi = mid; }
+          for (let k = lo; k < at.length && at[k].start <= span[1]; k++) if (at[k].end <= span[1]) inside.add(at[k].i);
         }
+        for (const i of [...inside].sort((a, b) => a - b)) (nodeOf(items[i]).checks ||= []).push(entry);
       }
     }
     // Without the order, a condition that restricts what an item may hold is credited wherever the
@@ -771,6 +761,13 @@ export function* analyzeSteps(root, opts = {}) {
         const a = c.using[0];
         const n = a && a.tok ? nodeOfToken(a.tok) : null;
         if (n) addSink(n, { kind: 'connection-target', onlyFrom: FROM_OUTSIDE, file: c.file, line: c.line, point, detail: `CALL '${c.name}' with queue manager ${a.word}, which decides the queue manager the program connects to` });
+      } else if (DLI_ROUTINE.test(c.name)) {
+        // A get call fills the I/O area, the argument after the PCB, with the segment it retrieves.
+        const valueOf = (a) => a.lit ?? (a.tok ? ((itemOfToken(a.tok) || {}).values || []).find(v => v.t === 'lit')?.v : null);
+        const call = dliFunction(c.using, valueOf);
+        const a = call && isGet(call.fn) ? c.using[call.index + 2] : null;
+        const n = a && a.tok ? nodeOfToken(a.tok) : null;
+        if (n) addSource(n, { kind: 'database', file: c.file, line: c.line, point, detail: `CALL '${c.name}' ${call.fn} fills ${a.word}` });
       } else if (String(c.name).toUpperCase() === 'CEEGTST') {
         // CEEGTST(heap id, size, address, feedback code): the size is the second argument.
         const a = c.using[1];
@@ -891,25 +888,24 @@ export function* analyzeSteps(root, opts = {}) {
     const changes = new Map();
     for (const e of prog.execs) {
       if (e.kind !== 'CICS') continue;
-      const { opts: o, words } = execOptions(e);
-      if (words[0] !== 'REWRITE' && !(words[0] === 'DELETE' && !o.has('RIDFLD'))) continue;
+      const { opts: o, verb } = execReading(e);
+      if (verb !== 'REWRITE' && !(verb === 'DELETE' && !o.has('RIDFLD'))) continue;
       const file = optionName(o, 'DATASET') || optionName(o, 'FILE');
-      if (file && !changes.has(file)) changes.set(file, words[0] === 'REWRITE' ? 'rewrites' : 'deletes');
+      if (file && !changes.has(file)) changes.set(file, verb === 'REWRITE' ? 'rewrites' : 'deletes');
     }
     for (const e of prog.execs) {
-      const { opts: o, words } = execOptions(e);
+      const { opts: o, words, verb, sub, command, direction } = execReading(e);
       const at = { file: e.file, line: e.line, point: pointOf(e) };
       if (e.kind === 'CICS') {
-        const verb = words[0];
         const intoTok = (o.get('INTO') || o.get('SET') || [])[0];
         for (const opt of ['RESP', 'RESP2']) {
           const r = nodeOfToken((o.get(opt) || [])[0]);
           if (r) systemSource(r, `the ${opt} of EXEC CICS ${verb}`, at);
         }
-        if (verb === 'WRITE' && ['OPERATOR', 'JOURNALNAME', 'JOURNALNUM'].includes(words[1])) {
-          const opt = words[1] === 'OPERATOR' ? 'TEXT' : 'FROM';
+        if (verb === 'WRITE' && ['OPERATOR', 'JOURNALNAME', 'JOURNALNUM'].includes(sub)) {
+          const opt = sub === 'OPERATOR' ? 'TEXT' : 'FROM';
           const n = nodeOfToken((o.get(opt) || [])[0]);
-          if (n) addSink(n, { kind: 'log', onlyFrom: REFLECTED, ...at, detail: `EXEC CICS WRITE ${words[1]} ${opt}(${n.name}), ${words[1] === 'OPERATOR' ? 'to the operator console' : 'to a journal'}` });
+          if (n) addSink(n, { kind: 'log', onlyFrom: REFLECTED, ...at, detail: `EXEC CICS WRITE ${sub} ${opt}(${n.name}), ${sub === 'OPERATOR' ? 'to the operator console' : 'to a journal'}` });
         }
         if (verb === 'RECEIVE') {
           const isWeb = words.includes('WEB');
@@ -924,12 +920,12 @@ export function* analyzeSteps(root, opts = {}) {
           if (!isWeb && o.has('MAP')) receiveMap(e, o, itemOfToken(intoTok), at);
         }
         // What the terminal shows: SEND TEXT's FROM, or the map's output record, <map>O without FROM.
-        if (verb === 'SEND' && (words[1] === 'TEXT' || words[1] === 'MAP')) {
+        if (verb === 'SEND' && (sub === 'TEXT' || sub === 'MAP')) {
           const from = nodeOfToken((o.get('FROM') || [])[0]);
-          const map = words[1] === 'MAP' ? optionName(o, 'MAP') : null;
+          const map = sub === 'MAP' ? optionName(o, 'MAP') : null;
           const sym = !from && map ? byName.get(`${map}O`) : null;
           const n = from || (sym ? nodeOf(sym) : null);
-          if (n) addSink(n, { kind: 'screen', onlyFrom: ['system-response'], ...at, detail: `EXEC CICS SEND ${words[1]}${map ? `(${map})` : ''}${from ? ` FROM(${n.name})` : ', from its symbolic map'}, to the terminal` });
+          if (n) addSink(n, { kind: 'screen', onlyFrom: ['system-response'], ...at, detail: `EXEC CICS SEND ${sub}${map ? `(${map})` : ''}${from ? ` FROM(${n.name})` : ', from its symbolic map'}, to the terminal` });
         }
         // RIDFLD decides which record a READ or DELETE reaches, or where a browse starts. A key the
         // user types is how a lookup works; a key the program put in a protected field and read back
@@ -956,11 +952,11 @@ export function* analyzeSteps(root, opts = {}) {
           if (name) channelOf.set(name, optionName(o, 'CHANNEL'));
         }
         // A service call sends every container on its channel, and its URI decides where they go.
-        if (verb === 'INVOKE' && (words[1] === 'SERVICE' || words[1] === 'WEBSERVICE')) {
-          invokes.push({ at, what: words[1], channel: optionName(o, 'CHANNEL') });
+        if (verb === 'INVOKE' && (sub === 'SERVICE' || sub === 'WEBSERVICE')) {
+          invokes.push({ at, what: sub, channel: optionName(o, 'CHANNEL') });
           for (const opt of ['URI', 'URIMAP']) {
             const n = nodeOfToken((o.get(opt) || [])[0]);
-            if (n) addSink(n, { kind: 'outbound-host', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS INVOKE ${words[1]} ${opt}(...), which decides where the request goes` });
+            if (n) addSink(n, { kind: 'outbound-host', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS INVOKE ${sub} ${opt}(...), which decides where the request goes` });
           }
         }
         if (verb === 'GET' && o.has('CONTAINER')) {
@@ -970,17 +966,17 @@ export function* analyzeSteps(root, opts = {}) {
         }
         // CONVERSE is always a client call; SEND is one only on a session this program opened. A
         // plain WEB SEND answers the request that started the program, which is its job.
-        if (verb === 'WEB' && (words[1] === 'CONVERSE' || (words[1] === 'SEND' && o.has('SESSTOKEN')))) {
+        if (verb === 'WEB' && (sub === 'CONVERSE' || (sub === 'SEND' && o.has('SESSTOKEN')))) {
           const n = nodeOfToken((o.get('FROM') || [])[0]);
-          if (n) addSink(n, { kind: 'outbound-http', onlyFrom: DATA_AT_REST, ...at, detail: `EXEC CICS WEB ${words[1]} to a remote server` });
+          if (n) addSink(n, { kind: 'outbound-http', onlyFrom: DATA_AT_REST, ...at, detail: `EXEC CICS WEB ${sub} to a remote server` });
           // The request may be a container rather than a field.
           const sent = o.has('CONTAINER') ? optionName(o, 'CONTAINER') : null;
-          if (sent) addSink(containerNode(sent), { kind: 'outbound-http', onlyFrom: DATA_AT_REST, ...at, detail: `EXEC CICS WEB ${words[1]} sending CONTAINER(${sent})` });
+          if (sent) addSink(containerNode(sent), { kind: 'outbound-http', onlyFrom: DATA_AT_REST, ...at, detail: `EXEC CICS WEB ${sub} sending CONTAINER(${sent})` });
         }
         // The response this program returns to its own caller, and the document built for it. CICS
         // escapes nothing on the way out - DOCUMENT SET's UNESCAPED option is about URL decoding,
         // not HTML - so whatever reaches here is what the browser is handed.
-        if (verb === 'WEB' && words[1] === 'SEND' && !o.has('SESSTOKEN')) {
+        if (verb === 'WEB' && sub === 'SEND' && !o.has('SESSTOKEN')) {
           const n = nodeOfToken((o.get('FROM') || [])[0]);
           // A stored value becomes script only where the reply is markup, and a web program
           // answering with a database row is the ordinary shape of one. Input echoed back is a
@@ -990,28 +986,37 @@ export function* analyzeSteps(root, opts = {}) {
           if (n) addSink(n, { kind: 'web-response', onlyFrom: [...(markup ? IN_CICS : REFLECTED), 'system-response'], ...at, detail: `EXEC CICS WEB SEND answering this program's caller${markup ? ` as ${String(media.v).trim()}` : ''}` });
         }
         if (verb === 'DOCUMENT') {
-          const what = words[1] === 'SET' ? ['VALUE', 'SYMBOLLIST'] : ['TEXT', 'FROM', 'BINARY'];
+          const what = sub === 'SET' ? ['VALUE', 'SYMBOLLIST'] : ['TEXT', 'FROM', 'BINARY'];
           for (const opt of what) {
             const n = nodeOfToken((o.get(opt) || [])[0]);
-            if (n) addSink(n, { kind: 'web-response', onlyFrom: [...IN_CICS, 'system-response'], ...at, detail: `EXEC CICS DOCUMENT ${words[1]} ${opt}(...), which is sent as the response` });
+            if (n) addSink(n, { kind: 'web-response', onlyFrom: [...IN_CICS, 'system-response'], ...at, detail: `EXEC CICS DOCUMENT ${sub} ${opt}(...), which is sent as the response` });
           }
         }
-        if (verb === 'WEB' && words[1] === 'WRITE' && o.has('HTTPHEADER')) {
+        if (verb === 'WEB' && sub === 'WRITE' && o.has('HTTPHEADER')) {
           for (const opt of ['HTTPHEADER', 'VALUE']) {
             const n = nodeOfToken((o.get(opt) || [])[0]);
             if (n) addSink(n, { kind: 'http-header', onlyFrom: [...IN_CICS, 'system-response'], ...at, detail: `EXEC CICS WEB WRITE HTTPHEADER ${opt}(...)` });
           }
         }
-        if (verb === 'WEB' && (words[1] === 'OPEN' || words[1] === 'CONVERSE')) {
+        if (verb === 'WEB' && (sub === 'OPEN' || sub === 'CONVERSE')) {
           for (const opt of ['HOST', 'URIMAP', 'PATH']) {
             const n = nodeOfToken((o.get(opt) || [])[0]);
-            if (n) addSink(n, { kind: 'outbound-host', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS WEB ${words[1]} ${opt}(...), which decides where the request goes` });
+            if (n) addSink(n, { kind: 'outbound-host', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS WEB ${sub} ${opt}(...), which decides where the request goes` });
+          }
+        }
+        // FILE or DATASET picks the file a command reads, writes or browses from every file the region
+        // defines; the commands that carry on from these act on the file already picked.
+        if (['READ', 'WRITE', 'DELETE', 'STARTBR'].includes(verb)) {
+          for (const opt of ['FILE', 'DATASET']) {
+            const t = (o.get(opt) || [])[0];
+            const n = t && t.t === 'word' ? nodeOfToken(t) : null;
+            if (n) addSink(n, { kind: 'dynamic-file-path', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS ${verb} ${opt}(${n.name}), which decides the file the command acts on` });
           }
         }
         // The queue a command acts on, as distinct from what it writes there: a name from input
         // reads, overwrites or deletes whatever queue the input asks for.
         if (['WRITEQ', 'READQ', 'DELETEQ'].includes(verb)) {
-          const ts = words[1] === 'TS' || !words.includes('TD');
+          const ts = sub === 'TS' || !words.includes('TD');
           for (const opt of ['QUEUE', 'QNAME']) {
             const n = nodeOfToken((o.get(opt) || [])[0]);
             if (n) addSink(n, { kind: 'queue-name', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS ${verb} ${ts ? 'TS' : 'TD'} ${opt}(...) names the queue` });
@@ -1024,23 +1029,40 @@ export function* analyzeSteps(root, opts = {}) {
           const n = nodeOfToken((o.get('SYSID') || [])[0]);
           if (n) addSink(n, { kind: 'cics-sysid', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS ${verb} SYSID(${n.name}), which decides the region the command is shipped to` });
         }
-        if (verb === 'SET' && words[1] && o.has(words[1])) {
-          const n = nodeOfToken((o.get(words[1]) || [])[0]);
-          if (n) addSink(n, { kind: 'cics-system-resource', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS SET ${words[1]}(${n.name}), which decides the resource the command changes` });
+        // The resource option is the one the table names the command by, or the synonym written (SET
+        // DATASET is SET FILE).
+        const resource = o.has(sub) ? sub : words[1];
+        if (verb === 'SET' && resource && o.has(resource)) {
+          const n = nodeOfToken((o.get(resource) || [])[0]);
+          if (n) addSink(n, { kind: 'cics-system-resource', onlyFrom: IN_CICS, ...at, detail: `EXEC CICS SET ${resource}(${n.name}), which decides the resource the command changes` });
         }
         if (verb === 'READ' || verb === 'READNEXT' || verb === 'READPREV') {
           const n = nodeOfToken(intoTok);
           if (n && (o.has('FILE') || o.has('DATASET'))) addSource(n, { kind: 'file-record', ...at, detail: `EXEC CICS ${verb} FILE ... INTO` });
         }
         // A CONVERSE response is the remote server's data, which is untrusted input like any other.
-        if ((verb === 'WEB' && words[1] !== 'SEND') || verb === 'EXTRACT') {
+        if ((verb === 'WEB' && sub !== 'SEND') || verb === 'EXTRACT') {
+          const n = nodeOfToken(intoTok);
+          if (n) addSource(n, { kind: 'cics-web', ...at, detail: `EXEC CICS ${[verb, sub].filter(Boolean).join(' ')}` });
+        }
+        // The request's form fields, query parameters and headers, and what WEB EXTRACT says about the
+        // request, come back in options other than INTO: each one CICS returns a value in.
+        if (/^(WEB (READ|READNEXT|EXTRACT)|EXTRACT WEB)/.test(command || '')) {
+          for (const [opt, toks] of o) {
+            if (opt === 'INTO' || !['receives', 'both'].includes(direction(opt)) || CICS_EVERY_COMMAND_OPTIONS.has(opt)) continue;
+            const n = nodeOfToken(toks[0]);
+            if (n) addSource(n, { kind: 'cics-web', ...at, detail: `EXEC CICS ${command} ${opt}` });
+          }
+        }
+        // A queue item, and the data a START passed, were stored by another task.
+        if (/^READQ /.test(command || '') || command === 'RETRIEVE') {
           const n = nodeOfToken(intoTok);
-          if (n) addSource(n, { kind: 'cics-web', ...at, detail: `EXEC CICS ${words.slice(0, 2).join(' ')}` });
+          if (n) addSource(n, { kind: 'cics-queue', ...at, detail: `EXEC CICS ${command} INTO` });
         }
         // Where a transient-data write lands is not in the program: the CSD maps the queue to a DD
         // and the region's JCL maps the DD to a destination. Collected here, resolved once every
         // definition has been read.
-        if (verb === 'WRITEQ' && words[1] === 'TD') {
+        if (verb === 'WRITEQ' && sub === 'TD') {
           const from = nodeOfToken((o.get('FROM') || [])[0]);
           const qTok = (o.get('QUEUE') || o.get('TDQUEUE') || [])[0];
           const qItem = qTok && qTok.t === 'word' ? itemOfToken(qTok) : null;
@@ -1068,11 +1090,16 @@ export function* analyzeSteps(root, opts = {}) {
           if (t && t.t === 'lit') startsTransactions.add(String(t.v).trim().toUpperCase());
         }
       }
+      if (e.kind === 'DLI' && isGet(verb)) {
+        for (const tok of o.get('INTO') || []) {
+          const n = nodeOfToken(tok);
+          if (n) addSource(n, { kind: 'database', ...at, detail: `EXEC DLI ${verb} INTO ${n.name}` });
+        }
+      }
       if (e.kind === 'SQL') {
         // A host variable names the item a qualified :G.F resolves to; the statement fills the ones
         // Db2 writes, and those hold what the database held.
         const hv = e.hostVariables || hostVariablesIn(e.toks);
-        const verb = words[0];
         for (const h of hv) {
           if (!h.written) continue;
           const n = nodeOfToken(h.tok);
@@ -1128,6 +1155,16 @@ export function* analyzeSteps(root, opts = {}) {
     const tree = (!repo && opts.tree) ? opts.tree : directoryTree(base, opts);
     const idx = tree.index;
     const files = tree.list().filter(isProgram).filter(inScope(opts));
+    // PL/I programs join the same graph unless the scan is told not to read them.
+    const readsPli = opts.pli ?? process.env.COBOLWORK_PLI !== '0';
+    if (readsPli) files.push(...tree.list().filter(isPli).filter(inScope(opts)));
+    // A PL/I %INCLUDE names a member by base name; any file of the tree may hold it.
+    const pliMembers = readsPli ? membersOf(tree.list()) : null;
+    const readMember = (name, from) => {
+      const path = chooseMember(pliMembers.get(name), from);
+      if (!path) return null;
+      try { return { path, text: tree.text(path).text }; } catch { return null; }
+    };
     for (const f of tree.list().filter(isJcl).filter(inScope(opts))) { jclPending.push(f); jclTrees.set(f, tree); }
     for (const f of tree.list().filter((p) => /\.csd$/i.test(p)).filter(inScope(opts))) {
       try { addCsd(tree.text(f).text, f); } catch (e) { stats.unreadable++; unread.push(`${tree.rel(f)}: ${e.code || e.name}`); }
@@ -1146,26 +1183,38 @@ export function* analyzeSteps(root, opts = {}) {
     // edges are proportional to data items and statements, not to bytes, so a repository of small
     // programs with large record layouts can exhaust a heap the budget believes is untouched. The
     // shared loop watches the heap itself, and the budget stays as the cheaper first line.
-    const repeated = repeatedTexts(tree, files);
-    const run = yield loopOver(files, (f) => {
+    const { repeated, later } = repeatedTexts(tree, files);
+    // The larger COBOL programs' control analyses are built in worker threads ahead of the loop, where
+    // the tree can say how it parses as data. A copy whose first is analysed already is restored instead.
+    const workers = tree.parseSpec ? (opts.controlWorkers ?? controlWorkerCount()) : 0;
+    const minWorkerBytes = opts.controlWorkerMinBytes ?? CONTROL_WORKER_MIN_BYTES;
+    const sizeOf = (f) => { try { return statSync(f).size; } catch { return 0; } };
+    const toWorkers = new Set(workers > 0 ? files.filter((f) => !isPli(f) && !later.has(f) && sizeOf(f) >= minWorkerBytes) : []);
+    const ahead = toWorkers.size ? controlAhead(tree.parseSpec, files, (f) => toWorkers.has(f), { workers }) : null;
+    const readProgram = (f, i) => {
       let src;
       try { const s = tree.text(f); src = s.text; if (s.encoding === 'ebcdic') stats.ebcdic++; } catch (e) { stats.unreadable++; unread.push(`${tree.rel(f)}: ${e.code || e.name}`); return 0; }
-      if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) return 0;
+      const pli = isPli(f);
+      if (!pli && !/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) return 0;
       if (held + src.length > budget) { stats.overBudget++; return 0; }
       held += src.length;
       let r;
       try {
-        r = tree.parse(f, src);
+        r = pli ? parsePliSource(src, f, { readMember }) : tree.parse(f, src);
       } catch (e) { stats.threw++; unparsed.push(`${tree.rel(f)}: ${e.code || e.name}`); return src.length; }
       stats.files++;
       const copied = src.length >= reuseMin && repeated.size ? repeated.get(textKey(src)) : undefined;
+      const answer = ahead ? ahead.take(f, i) : null;
+      const builtFiles = answer && answer.length === r.programs.length ? programFiles(r, f) : null;
       r.programs.forEach((p, k) => {
-        programs.push(summarise(p, f, programs.length, r.options, copied ? reuseOf(copied, k, r, f, tree) : null));
+        programs.push(summarise(p, f, programs.length, r.options, copied ? reuseOf(copied, k, r, f, tree) : null, builtFiles && { ...answer[k], files: builtFiles }));
         stats.programs++;
       });
       if (copied && --copied.left === 0) repeated.delete(copied.text);
       return src.length;
-    }, { label: 'flow', watcher: memoryWatcher });
+    };
+    let run;
+    try { run = yield loopOver(files, readProgram, { label: 'flow', watcher: memoryWatcher }); } finally { ahead?.close(); }
 
     // A file the loop never reached is not the same as one past the byte budget, and they are
     // counted separately so a report can say which limit it hit.
@@ -1565,12 +1614,12 @@ export function* analyzeSteps(root, opts = {}) {
         // stand in for it.
         const guarded = since.some((n) => n.checks || n.guard);
         if (t.partial) {
-          const key = `${to.id}|${guarded}|${t.ctx}`;
+          const key = atContext(to.id, t.ctx, guarded ? 1 : 0);
           const held = parts.get(key);
           if (held && held.some((r) => (r[2] ? t.vague : !t.vague && r[0] <= t.lo && t.hi <= r[1]))) continue;
           if (held) held.push([t.lo, t.hi, t.vague]); else parts.set(key, [[t.lo, t.hi, t.vague]]);
         } else if (guarded) {
-          const dk = `${to.id}|${t.ctx}`;
+          const dk = atContext(to.id, t.ctx);
           if (stampWhole[to.id] === walk || dirty.get(dk) === walk) continue;
           dirty.set(dk, walk);
         } else if (!firstVisit(to.id, t.ctx)) continue;
@@ -1594,10 +1643,12 @@ export function* analyzeSteps(root, opts = {}) {
   const stampWhole = new Int32Array(nodes.length);
   // A whole node reached with a call context, per walk. Reached with none, it stands in for every context.
   const stampCtx = new Map();
+  // A node and a call context as one number, with room for a second bit of state.
+  const atContext = (id, ctx, bit = 0) => (ctx * 2 + bit) * nodes.length + id;
   const firstVisit = (id, ctx) => {
     if (stampWhole[id] === walk) return false;
     if (!ctx) { stampWhole[id] = walk; return true; }
-    const key = `${id}|${ctx}`;
+    const key = atContext(id, ctx);
     if (stampCtx.get(key) === walk) return false;
     stampCtx.set(key, walk);
     return true;
@@ -1819,7 +1870,7 @@ export function* analyzeSteps(root, opts = {}) {
           if (crossedEdges) crossedEdges.push({ from: cur.id, to: to.id, e });
           const { partial, lo, hi, vague, ctx } = crossed;
           if (partial) {
-            const pk = `${to.id}|${ctx}`;
+            const pk = atContext(to.id, ctx);
             const held = partSeen.get(pk);
             if (held && held.some(r => (r[2] ? vague : !vague && r[0] <= lo && hi <= r[1]))) continue;
             if (held) held.push([lo, hi, vague]); else partSeen.set(pk, [[lo, hi, vague]]);

package/lib/db2/cursor.mjs

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// The Db2 statement cursor: the shared token cursor, failing with Db2Syntax.
+import { tokenCursor, split } from '../statement-cursor.mjs';
+
+export class Db2Syntax extends Error {
+  constructor(message, tok) {
+    super(message);
+    this.name = 'Db2Syntax';
+    this.line = tok ? tok.line : null;
+    this.col = tok ? tok.col : null;
+  }
+}
+
+export const cursor = (toks) => tokenCursor(toks, Db2Syntax);
+export { split };