@portll/cobolwork 0.2.117 → 0.2.150

package/lib/dataflow.mjs

@@ -1,9 +1,12 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { inScope, isProgram, isBms, readSource, relPath } from './sources.mjs';
+import { inScope, isProgram, isBms, relPath } from './sources.mjs';
 import { parseBms, symbolicNames } from './bms.mjs';
 import { dirname, join } from 'node:path';
 import { parseSource, buildFileIndex } from './parser.mjs';
 import { parseJcl } from './jcl.mjs';
+import { holdsDecimal } from './layout.mjs';
+import { hostVariablesIn } from './embedded-sql.mjs';
+import { tsoCommands } from './utilities.mjs';
 import { parseCsd, ddOfQueue } from './csd.mjs';
 import { loadSite } from './site.mjs';
 import { ssrangeAbends } from './options.mjs';
@@ -68,6 +71,7 @@ STANDARD-DEVIATION RANDOM SQRT FACTORIAL LOG LOG10 EXP EXP10 PI E ACOS ASIN ATAN
 SECONDS-PAST-MIDNIGHT`.split(/\s+/));
 const computes = (st) => ARITHMETIC.has(st.verb) || st.counts === true || (st.fns || []).some((f) => NUMERIC_FUNCTIONS.has(f));
 const computedOnRoute = (state) => { for (let s = state; s; s = s.prev) if (s.why && s.why.computes) return true; return false; };
+const typedOnRoute = (state) => { for (let s = state; s; s = s.prev) if (s.node.typed) return true; return false; };
 
 // Every literal a program holds that could be a program name: its VALUE clauses and the literals its
 // statements and commands use. A menu that XCTLs through a table of names starts every one of them.
@@ -89,15 +93,6 @@ function lastOption(name, list) {
   return null;
 }
 
-// Whether an item's bytes are read as decimal digits: zoned (DISPLAY with a numeric picture) or
-// packed. A group, an edited picture, binary and floating point are not.
-function holdsDecimal(it) {
-  if ((it.children || []).some((c) => c.level !== 88)) return false;
-  const usage = String(it.effectiveUsage || 'DISPLAY').replace('COMPUTATIONAL', 'COMP');
-  if (usage === 'COMP-3' || usage === 'PACKED-DECIMAL' || usage === 'COMP-6') return true;
-  return usage === 'DISPLAY' && !!it.picture && /9/.test(it.picture) && /^[S9VP()0-9]+$/i.test(it.picture);
-}
-
 const RELATION = new Set(['>', '<', '>=', '<=', '=', '<>']);
 const RELATION_WORDS = new Set(['GREATER', 'LESS', 'EQUAL']);
 const FILLER_WORDS = new Set(['IS', 'NOT', 'THAN', 'TO', 'OR', 'GREATER', 'LESS', 'EQUAL', 'OF', 'IN']);
@@ -178,27 +173,6 @@ function execOptions(exec) {
   return { opts, words };
 }
 
-function sqlHostVars(exec) {
-  const out = [];
-  for (let i = 0; i < exec.toks.length; i++) {
-    const t = exec.toks[i];
-    if (t.t === 'op' && t.v === ':' && exec.toks[i + 1] && exec.toks[i + 1].t === 'word') out.push({ tok: exec.toks[i + 1], at: i + 1 });
-  }
-  return out;
-}
-
-// The host variables a SELECT or FETCH fills are the ones in its INTO list, which ends at FROM.
-// Taking every variable at or after the INTO line made the input parameters in a WHERE clause read
-// as database values — and they are whatever the caller sent, under a rule that says data at rest.
-function sqlIntoRange(exec) {
-  const words = exec.toks.map(t => (t.t === 'word' ? t.u : ''));
-  const into = words.indexOf('INTO');
-  if (into < 0) return null;
-  let end = exec.toks.length;
-  for (let i = into + 1; i < words.length; i++) if (words[i] === 'FROM') { end = i; break; }
-  return [into, end];
-}
-
 export function analyze(root, opts = {}) {
   const repos = opts.repos || [''];
   const stats = { files: 0, programs: 0, edges: 0, nodes: 0, threw: 0, overBudget: 0, unreadable: 0, ebcdic: 0,
@@ -223,6 +197,7 @@ export function analyze(root, opts = {}) {
   const nodes = [];
   const programs = [];
   const jclPending = [];
+  const jclTrees = new Map();
   // CICS definitions, from CSD extracts and from DFHCSDUP job input, and the estate's own statement
   // of which DDs and queues reach the internal reader.
   const csd = { tdqueues: new Map(), transactions: new Map(), urimaps: new Map() };
@@ -235,7 +210,7 @@ export function analyze(root, opts = {}) {
   const jobSteps = [];
   // The maps of the repository being read, by MAPSET/MAP and by MAP alone.
   const bmsMaps = new Map();
-  const site = loadSite(root, opts.site || null);
+  const site = loadSite(root, opts.site || null, opts.tree);
   const relOf = new Map();
   const rel = (p) => { let r = relOf.get(p); if (r === undefined) { r = relPath(root, p); relOf.set(p, r); } return r; };
   // `at` is the statement an edge that is not a statement's own is read at: the CALL or LINK that
@@ -457,11 +432,14 @@ export function analyze(root, opts = {}) {
           usesOf.get(at.name).push(rec);
         }
         const offset = x.offset || 0;
+        // A constant length past the start, or a constant start before the length, takes that many bytes off the top.
+        const base = limitOf(host, table);
+        const limit = kind === 'reference-modification' && base != null && x.span ? base - x.span + 1 : base;
         const use = `${key}|${point ?? `s${st.at}`}|${offset}`;
         if (indexed.has(use)) continue;
         indexed.add(use);
         at.sinks.push({
-          kind, onlyFrom: FROM_OUTSIDE, file: st.file, line: st.line, point, group: `${pk}|${key}`, limit: limitOf(host, table), ...(offset ? { offset } : {}), ...(offset > 0 && idx && unsignedWhole(idx) ? { nonNeg: true } : {}), ...(ssrange ? { ssrange } : {}),
+          kind, onlyFrom: FROM_OUTSIDE, file: st.file, line: st.line, point, group: `${pk}|${key}`, limit, ...(offset ? { offset } : {}), ...(offset > 0 && idx && unsignedWhole(idx) ? { nonNeg: true } : {}), ...(ssrange ? { ssrange } : {}),
           detail: kind === 'subscript'
             ? `${at.name} subscripts ${host.name}, a table of ${table.occurs}`
             : `${at.name} sets the ${x.kind === 'refmod-offset' ? 'start' : 'length'} of a reference to ${host.name}, which is ${host.size} bytes`,
@@ -724,14 +702,16 @@ export function analyze(root, opts = {}) {
       walk(record);
       for (const field of map.fields) {
         if (!field.name) continue;
-        const marks = ['PROT', 'ASKIP', 'DRK', 'NUM'].filter((a) => field.effective.has(a));
-        if (!marks.length) continue;
         const iname = symbolicNames(map, field).find((n) => n.suffix === 'I');
         const item = iname && within.get(iname.name);
         if (!item) continue;
+        const marks = ['PROT', 'ASKIP', 'DRK', 'NUM'].filter((a) => field.effective.has(a));
+        const typed = !marks.includes('PROT') && !marks.includes('ASKIP');
+        if (typed) nodeOf(item).typed = true;
+        if (!marks.length) continue;
         const n = nodeOf(item);
         n.screen = { item: item.name, field: field.name, map: map.name, mapset: mapset.name, marks, declared: !!field.attrb };
-        if (!marks.includes('PROT') && !marks.includes('ASKIP')) continue;
+        if (typed) continue;
         stats.protectedFields++;
         const back = field.effective.has('FSET') ? ' with FSET' : '';
         n.sources.push({ kind: 'cics-protected-field', onlyTo: ['record-key', 'record-update'], ...at,
@@ -921,15 +901,14 @@ export function analyze(root, opts = {}) {
         }
       }
       if (e.kind === 'SQL') {
-        // The parser's host variables name the item a qualified :G.F resolves to, where the first
-        // word after the colon would name the group G and so every field in it.
-        const hv = e.hostVariables || sqlHostVars(e);
+        // A host variable names the item a qualified :G.F resolves to; the statement fills the ones
+        // Db2 writes, and those hold what the database held.
+        const hv = e.hostVariables || hostVariablesIn(e.toks);
         const verb = words[0];
-        const range = e.hostVariables ? null : ['SELECT', 'FETCH'].includes(verb) && sqlIntoRange(e);
         for (const h of hv) {
-          if (e.hostVariables ? !h.written : !range || h.at < range[0] || h.at > range[1]) continue;
+          if (!h.written) continue;
           const n = nodeOfToken(h.tok);
-          if (n) n.sources.push({ kind: 'database', ...at, detail: `EXEC SQL ${verb} INTO host variable` });
+          if (n) n.sources.push({ kind: 'database', ...at, detail: `EXEC SQL ${verb} ${verb === 'SELECT' || verb === 'FETCH' ? 'INTO' : 'writes'} host variable` });
         }
         // The location CONNECT TO or SET CONNECTION names, where a host variable holds it; what
         // follows USER and USING is the credential, not the target.
@@ -981,7 +960,7 @@ export function analyze(root, opts = {}) {
     const tree = (!repo && opts.tree) ? opts.tree : directoryTree(base, opts);
     const idx = tree.index;
     const files = tree.list().filter(isProgram).filter(inScope(opts));
-    for (const f of tree.list().filter(isJcl).filter(inScope(opts))) jclPending.push(f);
+    for (const f of tree.list().filter(isJcl).filter(inScope(opts))) { jclPending.push(f); jclTrees.set(f, tree); }
     for (const f of tree.list().filter((p) => /\.csd$/i.test(p)).filter(inScope(opts))) {
       try { addCsd(tree.text(f).text, f); } catch (e) { stats.unreadable++; unread.push(`${tree.rel(f)}: ${e.code || e.name}`); }
     }
@@ -1110,7 +1089,7 @@ export function analyze(root, opts = {}) {
   // ACCEPT FROM COMMAND-LINE is untrusted.
   for (const f of jclPending) {
     let src;
-    try { src = readSource(f).text; } catch (e) { stats.unreadable++; unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
+    try { src = jclTrees.get(f).text(f).text; } catch (e) { stats.unreadable++; unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
     let job;
     try { job = parseJcl(src, f); } catch (e) { stats.threw++; unparsed.push(`${rel(f)}: ${e.code || e.name}`); continue; }
     stats.jclFiles++;
@@ -1123,59 +1102,61 @@ export function analyze(root, opts = {}) {
         for (const dd of step.dds) if (dd.name && dd.name.toUpperCase() === 'SYSIN' && dd.inStream) addCsd(dd.inStream.map((l) => l.text).join('\n'), f, dd.inStream[0].line - 1);
       }
       jobSteps.push({ file: f, job: job.jobs[0]?.name || null, step: step.name, line: step.line, pgm: step.pgm.toUpperCase() });
-      // A TSO batch step starts a program by name in its commands: DSN ... RUN PROGRAM(X).
-      if (/^IKJEFT(01|1A|1B)$/i.test(step.pgm)) {
-        for (const dd of step.dds) for (const l of dd.inStream || []) {
-          const run = /\bRUN\s+PROGRAM\s*\(\s*([A-Z0-9$#@]{1,8})\s*\)/i.exec(l.text);
-          if (run) jobSteps.push({ file: f, job: job.jobs[0]?.name || null, step: step.name, line: l.line, pgm: run[1].toUpperCase() });
-        }
+      // A TSO batch step starts programs by name in its commands, TSO CALL and DSN RUN PROGRAM, and
+      // each is handed the step's DDs and its own parameter as EXEC PGM= would be.
+      const runs = [{ pgm: step.pgm.toUpperCase(), parm: step.parm, line: step.line, how: `PARM= on step ${step.name || '(unnamed)'} of ${rel(f)}` }];
+      for (const r of tsoCommands(step).runs) {
+        jobSteps.push({ file: f, job: job.jobs[0]?.name || null, step: step.name, line: r.line, pgm: r.program });
+        runs.push({ pgm: r.program, parm: r.parm, line: r.line, how: `the parameter ${r.via} passes in step ${step.name || '(unnamed)'} of ${rel(f)}` });
       }
-      const holders = allById.get(step.pgm.toUpperCase()) || [];
-      if (!holders.length) continue;            // a system utility, or a program not in this tree
-      stats.jclStepsResolved++;
-      // A program id held several times has no caller file to choose by, so each holder is a candidate.
-      const amb = holders.length > 1 ? `, any of ${holders.length} programs named ${step.pgm}` : '';
-      for (const callee of holders) {
+      for (const run of runs) {
+        const holders = allById.get(run.pgm) || [];
+        if (!holders.length) continue;            // a system utility, or a program not in this tree
+        stats.jclStepsResolved++;
+        // A program id held several times has no caller file to choose by, so each holder is a candidate.
+        const amb = holders.length > 1 ? `, any of ${holders.length} programs named ${run.pgm}` : '';
+        for (const callee of holders) {
 
-        // PARM arrives in the first PROCEDURE DIVISION USING item: on z/OS a halfword length
-        // followed by the text. A program with no USING cannot receive one, and saying it does
-        // would be a path nobody could follow.
-        if (step.parm !== null && callee.params && callee.params[0] && callee.params[0].node) {
-          callee.params[0].node.sources.push({
-            kind: 'jcl-parm', file: f, line: step.line,
-            detail: `PARM= on step ${step.name || '(unnamed)'} of ${rel(f)}, which runs ${step.pgm}${amb}`,
-          });
-          stats.jclCrossings++;
-        }
+          // PARM arrives in the first PROCEDURE DIVISION USING item: on z/OS a halfword length
+          // followed by the text. A program with no USING cannot receive one, and saying it does
+          // would be a path nobody could follow.
+          if (run.parm !== null && callee.params && callee.params[0] && callee.params[0].node) {
+            callee.params[0].node.sources.push({
+              kind: 'jcl-parm', file: f, line: run.line,
+              detail: `${run.how}, which runs ${run.pgm}${amb}`,
+            });
+            stats.jclCrossings++;
+          }
 
-        // In-stream data reaches whatever record the program reads from that DD. The COBOL says
-        // ASSIGN TO SYSIN and the job says //SYSIN DD *; neither half names the other, and the
-        // join is the whole point of reading both.
-        for (const dd of step.dds) {
-          // The same join in the other direction: what the program writes through SELECT ... ASSIGN
-          // TO a DD the job sends to the internal reader is submitted as a job.
-          if (dd.name && dd.sysout && /\bINTRDR\b/i.test(dd.sysout)) {
+          // In-stream data reaches whatever record the program reads from that DD. The COBOL says
+          // ASSIGN TO SYSIN and the job says //SYSIN DD *; neither half names the other, and the
+          // join is the whole point of reading both.
+          for (const dd of step.dds) {
+            // The same join in the other direction: what the program writes through SELECT ... ASSIGN
+            // TO a DD the job sends to the internal reader is submitted as a job.
+            if (dd.name && dd.sysout && /\bINTRDR\b/i.test(dd.sysout)) {
+              for (const m of callee.ddFiles || []) {
+                if (m.dd !== dd.name.toUpperCase()) continue;
+                for (const n of m.nodes) {
+                  n.sinks.push({ kind: 'internal-reader', file: m.file, line: m.line,
+                    detail: `records written through SELECT ${m.select} to //${dd.name}, which step ${step.name || '(unnamed)'} of ${rel(f)} sends to the internal reader${amb}` });
+                }
+              }
+            }
+            if (!dd.inStream || !dd.inStream.length || !dd.name) continue;
             for (const m of callee.ddFiles || []) {
               if (m.dd !== dd.name.toUpperCase()) continue;
               for (const n of m.nodes) {
-                n.sinks.push({ kind: 'internal-reader', file: m.file, line: m.line,
-                  detail: `records written through SELECT ${m.select} to //${dd.name}, which step ${step.name || '(unnamed)'} of ${rel(f)} sends to the internal reader${amb}` });
+                n.sources.push({
+                  kind: 'jcl-instream', file: f, line: dd.line,
+                  detail: `${dd.inStream.length} lines of in-stream data on //${dd.name} in step ${step.name || '(unnamed)'}, read through SELECT ${m.select}${amb}`,
+                });
+                stats.jclCrossings++;
               }
             }
-          }
-          if (!dd.inStream || !dd.inStream.length || !dd.name) continue;
-          for (const m of callee.ddFiles || []) {
-            if (m.dd !== dd.name.toUpperCase()) continue;
-            for (const n of m.nodes) {
-              n.sources.push({
-                kind: 'jcl-instream', file: f, line: dd.line,
-                detail: `${dd.inStream.length} lines of in-stream data on //${dd.name} in step ${step.name || '(unnamed)'}, read through SELECT ${m.select}${amb}`,
-              });
-              stats.jclCrossings++;
-            }
-          }
 
-      }
+          }
+        }
       }
     }
   }
@@ -1494,6 +1475,9 @@ export function analyze(root, opts = {}) {
           // arithmetic or a numeric function carries no bad bytes to the next one. Only this route is
           // dropped: a later one that copies the bytes still reports.
           if (sink.kind === 'arithmetic' && computedOnRoute(state)) continue;
+          // A protected field's value carried through a field the terminal may type into reaches the
+          // key as typed input, which the user could have entered anyway: the protection was no control.
+          if (src.kind === 'cics-protected-field' && typedOnRoute(state)) continue;
           seen.add(key);
           // The ends of a long path are what a reader uses: where the value came from, and what it
           // reached. Keeping every hop of a thousand-hop chain, for a thousand findings, is the

package/lib/diff.mjs

@@ -1,16 +1,15 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { existsSync, mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
-import { spawnSync } from 'node:child_process';
 import { tmpdir } from 'node:os';
-import { dirname, join, resolve, sep } from 'node:path';
-import { parseSource } from './parser.mjs';
-import { inScope, isProgram, readSource, relPath } from './sources.mjs';
+import { dirname, join, resolve } from 'node:path';
+import { git, revisionBlobs, treeOf, treePathParts } from './kernel/git.mjs';
+import { inScope, isProgram } from './sources.mjs';
 import { scanAll } from './scan.mjs';
 import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 // A diff finding is located by program rather than by line, so it orders on its own key. The text
 // comparison underneath it is the shared one.
 import { byText, evidenceMap } from './kernel/findings.mjs';
-import { directoryTree } from './kernel/source-tree.mjs';
+import { directoryTree, gitTree } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
 import { loadSite, SITE_FILE } from './site.mjs';
 import { loadBaseline, BASELINE_FILE, SUPPRESSING } from './baseline.mjs';
@@ -30,56 +29,18 @@ export const DIFF_RULES = {
 
 const MAX_LISTED = 8;
 
-// A reviewed repository's .git/config can name commands; with fsmonitor off and plumbing only, git runs none.
-const GIT_ENV = { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0' };
-const git = (repo, args, opts = {}) => spawnSync('git', ['-c', 'core.fsmonitor=false', '-C', repo, ...args], { env: GIT_ENV, maxBuffer: 256 * 1024 * 1024, ...opts });
-const refused = (what, ref, why) => Object.assign(new Error(`${what} ${ref} failed: ${why}`), { code: 'EDIFFREF' });
-
-function treeOf(repo, ref) {
-  if (!ref || String(ref).startsWith('-')) throw refused('git rev-parse', ref, 'a revision cannot be empty or start with "-"');
-  const r = git(repo, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{tree}`], { encoding: 'utf8' });
-  const oid = String(r.stdout || '').trim();
-  if (r.status !== 0 || !/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(oid)) throw refused('git rev-parse', ref, String(r.stderr || '').trim() || 'not a revision');
-  return oid;
-}
-
-// A tree path as a file under `dir`, or null if a part is empty, climbs, names a drive or stream, or is .git.
-function placeIn(dir, path) {
-  const parts = path.split('/');
-  if (parts.some((p) => !p || p === '.' || p === '..' || /[\\:\0]/.test(p) || p.toLowerCase() === '.git')) return null;
-  const to = resolve(dir, ...parts);
-  return to.startsWith(dir + sep) ? to : null;
-}
-
-// The revision as committed: blobs as stored, with no filter or line-ending conversion; links and submodules left out.
+// The revision as committed, written into a temporary directory, for a caller that hands the tree
+// to a program outside this process: build's compiler and ironwork read files, not blobs.
 function materialise(repo, ref) {
-  const tree = treeOf(repo, ref);
-  const listed = git(repo, ['ls-tree', '-r', '-z', '--full-tree', tree]);
-  if (listed.status !== 0) throw refused('git ls-tree', ref, String(listed.stderr || '').trim() || `exit ${listed.status}`);
-  const blobs = [];
-  for (const entry of listed.stdout.toString('utf8').split('\0')) {
-    const m = /^(\d{6}) blob ([0-9a-f]+)\t(.+)$/s.exec(entry);
-    if (m && m[1] !== '120000') blobs.push({ oid: m[2], path: m[3] });
-  }
+  const { blobs } = revisionBlobs(repo, treeOf(repo, ref), ref);
   const dir = realpathSync(mkdtempSync(join(tmpdir(), 'cobolwork-diff-')));
   try {
-    for (let i = 0; i < blobs.length; i += 500) {
-      const batch = blobs.slice(i, i + 500);
-      const r = git(repo, ['cat-file', '--batch'], { input: batch.map((b) => b.oid).join('\n') + '\n' });
-      if (r.status !== 0) throw refused('git cat-file', ref, String(r.stderr || '').trim() || `exit ${r.status}`);
-      let at = 0;
-      for (const b of batch) {
-        const nl = r.stdout.indexOf(0x0a, at);
-        const head = r.stdout.toString('utf8', at, nl).split(' ');
-        if (head[1] !== 'blob') throw refused('git cat-file', ref, `${b.oid} is ${head[1] || 'missing'}`);
-        const size = Number(head[2]);
-        const to = placeIn(dir, b.path);
-        if (to) {
-          mkdirSync(dirname(to), { recursive: true });
-          writeFileSync(to, r.stdout.subarray(nl + 1, nl + 1 + size));
-        }
-        at = nl + 1 + size + 1;
-      }
+    for (const b of blobs) {
+      const parts = treePathParts(b.path);
+      if (!parts) continue;
+      const to = resolve(dir, ...parts);
+      mkdirSync(dirname(to), { recursive: true });
+      writeFileSync(to, b.bytes);
     }
   } catch (e) {
     rmSync(dir, { recursive: true, force: true });
@@ -102,9 +63,9 @@ const qualified = (it) => { const names = []; for (let x = it; x; x = x.parent)
 
 // A program that cannot be read or parsed is named in `unread`, never dropped: dropped, it would
 // read as removed from the tree, and every change inside it would go unreported.
-function facts(root, opts = {}) {
-  // Each side of a diff is its own tree: the base is a revision materialised into a temporary
-  // directory, the head is the working tree, and they are walked separately.
+function facts(side, opts = {}) {
+  // Each side of a diff is its own tree: a revision read out of git, or the working tree, and they
+  // are walked separately.
   //
   // This used to build its own parse options with systemDirs: [], where every other caller passed
   // the caller's. Nobody decided that - it is what five copies of one object literal do. The
@@ -112,24 +73,24 @@ function facts(root, opts = {}) {
   // change inside one was invisible to the command whose whole job is what a change reaches. Both
   // sides now take the same configuration, which for a caller that passes no systemDirs is
   // exactly the old behaviour.
-  const tree = directoryTree(root, opts);
-  const idx = tree.index;
+  const tree = side.tree || directoryTree(side.root, opts);
+  const rel = (p) => tree.rel(p);
   const out = new Map();
   out.unread = [];
   for (const f of tree.list().filter(isProgram).filter(inScope(opts))) {
     let src;
-    try { src = readSource(f).text; } catch (e) { out.unread.push(`${relPath(root, f)}: ${e.code || e.name}`); continue; }
+    try { src = tree.text(f).text; } catch (e) { out.unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
     if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) continue;
     let r;
     try {
       r = tree.parse(f, src);
-    } catch (e) { out.unread.push(`${relPath(root, f)}: ${e.code || e.name}`); continue; }
-    const file = relPath(root, f);
+    } catch (e) { out.unread.push(`${rel(f)}: ${e.code || e.name}`); continue; }
+    const file = rel(f);
     for (const p of r.programs) {
       const items = new Map();
       for (const it of p.items) {
         if (!it.name || it.name === 'FILLER' || it.level === 88 || it.level === 66 || it.level === 78) continue;
-        items.set(qualified(it), { size: it.size, offset: it.offset, from: it.file ? relPath(root, it.file) : file, section: it.section, level: it.level });
+        items.set(qualified(it), { size: it.size, offset: it.offset, from: it.file ? rel(it.file) : file, section: it.section, level: it.level });
       }
       const calls = new Set();
       const dynamic = new Set();
@@ -149,7 +110,7 @@ function facts(root, opts = {}) {
         if (arg) commareas.add(arg.u);
       }
       const using = new Set(p.calls.flatMap(c => c.using.filter(a => a.word).map(a => a.word)));
-      const copied = new Set(r.copies.filter((c) => c.status === 'resolved' && c.path).map((c) => relPath(root, c.path)));
+      const copied = new Set(r.copies.filter((c) => c.status === 'resolved' && c.path).map((c) => rel(c.path)));
       out.set(`${file}#${p.id}`, { id: p.id, file, text: src, items, calls, dynamic, commareas, using, copied });
     }
   }
@@ -168,12 +129,17 @@ function countByKey(findings) {
   return m;
 }
 
-export function diffTrees(baseRoot, headRoot, opts = {}) {
-  // The allow list describes the working tree, so it never applies to a revision checked out into
-  // a temporary directory: applied there it would match nothing and the base would read as empty.
+// Each side is a directory, or a source tree: a git revision read without writing it out.
+const sideOf = (x) => (typeof x === 'string' ? { root: x, tree: null } : { root: x.root, tree: x });
+
+export function diffTrees(baseSide, headSide, opts = {}) {
+  const baseAt = sideOf(baseSide);
+  const headAt = sideOf(headSide);
+  // The allow list describes the working tree, so it never applies to a revision: applied there it
+  // would match nothing and the base would read as empty.
   const baseOpts = { ...opts, allow: null };
-  const base = facts(baseRoot, baseOpts);
-  const head = facts(headRoot, opts);
+  const base = facts(baseAt, baseOpts);
+  const head = facts(headAt, opts);
   const findings = [];
   const reach = { programsCompared: 0, programsAdded: 0, programsRemoved: 0, programsWithLayoutChange: 0, reachedOnlyThroughCopybooks: 0, copybooksImplicated: new Set() };
 
@@ -214,8 +180,9 @@ export function diffTrees(baseRoot, headRoot, opts = {}) {
 
   const listing = { listSinks: opts.listSinks === true, listSources: opts.listSources === true };
   const shared = { only: opts.only, fullTrace: opts.fullTrace, systemDirs: opts.systemDirs || [], ...listing };
-  const before = scanAll(baseRoot, shared);
-  const after = scanAll(headRoot, { ...shared, allow: opts.allow });
+  const withTree = (at) => (at.tree ? { tree: at.tree } : {});
+  const before = scanAll(baseAt.root, { ...shared, ...withTree(baseAt) });
+  const after = scanAll(headAt.root, { ...shared, ...withTree(headAt), allow: opts.allow });
   // A file neither tree could read cannot be compared. Its findings are neither introduced nor
   // resolved, and calling them resolved would report a scan failure as a fix.
   const unreadable = new Set([...base.unread, ...head.unread].map(u => u.split(': ')[0]));
@@ -237,7 +204,7 @@ export function diffTrees(baseRoot, headRoot, opts = {}) {
     return n > (afterCounts.get(k) || 0);
   });
   const uncomparable = [...new Set([...before.findings, ...after.findings].filter(f => !comparable(f)).map(f => f.path))];
-  const configurationChanged = configurationChanges(baseRoot, headRoot);
+  const configurationChanged = configurationChanges(baseSide, headSide);
 
   for (const f of findings) { const m = DIFF_RULES[f.rule]; f.sev = m.sev; f.cwe = m.cwe; f.evidence = m.evidence; }
   findings.sort((a, b) => byText(a.rule, b.rule) || byText(a.path, b.path) || byText(String(a.program), String(b.program)));
@@ -272,13 +239,16 @@ export function diffTrees(baseRoot, headRoot, opts = {}) {
 const SITE_LISTS = ['productionQualifiers', 'productionJobPaths', 'nonProductionJobPaths', 'systemNames', 'vendorPacks',
   'internalReaderDds', 'internalReaderQueues', 'compilerOptions', 'apfLibraries', 'restrictedDatasets', 'surrogateUsers'];
 
-function siteChange(baseRoot, headRoot) {
-  const was = existsSync(join(baseRoot, SITE_FILE));
-  const now = existsSync(join(headRoot, SITE_FILE));
+// Whether a side holds a file at its top: in the source tree it was read from, or on disk.
+const holds = (at, name) => (at.tree && at.tree.kind !== 'directory' ? at.tree.contains(resolve(at.tree.root, name)) : existsSync(join(at.root, name)));
+
+function siteChange(baseAt, headAt) {
+  const was = holds(baseAt, SITE_FILE);
+  const now = holds(headAt, SITE_FILE);
   if (!was && !now) return null;
   if (!now) return { file: SITE_FILE, change: 'removed, so the rules that need it do not run' };
-  const b = loadSite(baseRoot);
-  const h = loadSite(headRoot);
+  const b = loadSite(baseAt.root, null, baseAt.tree);
+  const h = loadSite(headAt.root, null, headAt.tree);
   const parts = was ? [] : ['added'];
   for (const k of SITE_LISTS) {
     const gone = b[k].filter((x) => !h[k].includes(x)).map((x) => `-${x}`);
@@ -291,16 +261,19 @@ function siteChange(baseRoot, headRoot) {
   return parts.length ? { file: SITE_FILE, change: printable(parts.join('; '), 400) } : null;
 }
 
-export const configurationChanges = (baseRoot, headRoot) =>
-  [siteChange(baseRoot, headRoot), baselineChange(baseRoot, headRoot)].filter(Boolean);
+export const configurationChanges = (baseSide, headSide) => {
+  const baseAt = sideOf(baseSide);
+  const headAt = sideOf(headSide);
+  return [siteChange(baseAt, headAt), baselineChange(baseAt, headAt)].filter(Boolean);
+};
 
-function baselineChange(baseRoot, headRoot) {
-  const was = existsSync(join(baseRoot, BASELINE_FILE));
-  const now = existsSync(join(headRoot, BASELINE_FILE));
+function baselineChange(baseAt, headAt) {
+  const was = holds(baseAt, BASELINE_FILE);
+  const now = holds(headAt, BASELINE_FILE);
   if (!was && !now) return null;
-  const held = (root, there) => new Map((there ? loadBaseline(root)?.entries || [] : []).map((e) => [`${e.fingerprint}|${e.rule}`, e]));
-  const b = held(baseRoot, was);
-  const h = held(headRoot, now);
+  const held = (at, there) => new Map((there ? loadBaseline(at.root, { tree: at.tree })?.entries || [] : []).map((e) => [`${e.fingerprint}|${e.rule}`, e]));
+  const b = held(baseAt, was);
+  const h = held(headAt, now);
   const added = [...h.keys()].filter((k) => !b.has(k));
   const suppressing = added.filter((k) => SUPPRESSING.includes(h.get(k).action)).length;
   const removed = [...b.keys()].filter((k) => !h.has(k)).length;
@@ -334,11 +307,14 @@ export function withRefs(repo, baseRef, headRef, opts, fn) {
   }
 }
 
+// Reads each revision out of git without writing it anywhere; the head is the working tree when
+// headRef is null.
 export function diffRefs(repo, baseRef, headRef = null, opts = {}) {
-  return withRefs(repo, baseRef, headRef, opts, (baseDir, headDir, scoped) => {
-    const res = diffTrees(baseDir, headDir, scoped);
-    res.summary.base = baseRef;
-    res.summary.head = headRef || WORKING_TREE;
-    return res;
-  });
+  const trees = { systemDirs: opts.systemDirs || [] };
+  const base = gitTree(repo, baseRef, trees);
+  const head = headRef ? gitTree(repo, headRef, trees) : repo;
+  const res = diffTrees(base, head, headRef ? opts : { ...opts, allow: gitScope(repo) });
+  res.summary.base = baseRef;
+  res.summary.head = headRef || WORKING_TREE;
+  return res;
 }

package/lib/embedded-sql.mjs

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// EXEC SQL as Db2 reads it, whichever language embeds it: a statement's words, host variables and
+// literals, and which host variables it reads and which it writes.
+// The words, host variables and literals of a statement. A host variable is :NAME, qualified as
+// :GROUP.NAME, and may carry an indicator variable, :NAME:IND or :NAME INDICATOR :IND.
+export function sqlTokens(sql) {
+  const toks = [];
+  const re = /'(?:[^']|'')*'|"(?:[^"]|"")*"|:\s*[A-Z0-9_$#@-]+(?:\.[A-Z0-9_$#@-]+)*|[A-Z0-9_$#@-]+|[(),=<>+*/;.]/gi;
+  for (const m of sql.matchAll(re)) {
+    const t = m[0];
+    if (t.startsWith(':')) toks.push({ host: t.slice(1).trim().toUpperCase() });
+    else if (/^['"]/.test(t)) toks.push({ lit: t });
+    else toks.push({ word: t.toUpperCase() });
+  }
+  return toks;
+}
+
+// Clauses that end an INTO list.
+const AFTER_INTO = new Set(['FROM', 'WHERE', 'GROUP', 'HAVING', 'ORDER', 'FETCH', 'FOR', 'WITH', 'OPTIMIZE', 'QUERYNO', 'SKIP', 'UNION', 'USING', 'VALUES']);
+
+// Which host variables a statement reads and which it writes, as Db2's SQL reference gives them. A
+// variable both read and written is in both lists. INTO a host variable writes it (INSERT INTO and
+// MERGE INTO name a table); so do SET's targets, GET DIAGNOSTICS' and ASSOCIATE LOCATORS' list. A
+// procedure's argument that is a lone variable may be IN, OUT or INOUT, which only the server knows.
+export function sqlRoles(toks) {
+  const sending = [];
+  const receiving = [];
+  const read = (h) => { if (!sending.includes(h)) sending.push(h); };
+  const write = (h) => { if (!receiving.includes(h)) receiving.push(h); };
+  const verb = toks.find((t) => t.word)?.word;
+  if (verb === 'CALL') return callRoles(toks, read, write), { sending, receiving };
+  // DESCRIBE and PREPARE read the SQLDA's SQLN and write the rest of it.
+  const intoBoth = verb === 'DESCRIBE' || verb === 'PREPARE';
+  let into = false;
+  let assigning = verb === 'SET';
+  let depth = 0;
+  for (let i = 0; i < toks.length; i++) {
+    const t = toks[i];
+    const next = toks[i + 1];
+    if (t.word === '(') depth++;
+    else if (t.word === ')') depth--;
+    else if (verb === 'SET' && t.word === '=' && depth === 0) assigning = false;
+    else if (verb === 'SET' && t.word === ',' && depth === 0) assigning = true;
+    if (t.word === 'INTO' && next && (next.host || next.word === 'DESCRIPTOR')) { into = true; continue; }
+    if (into && t.word && AFTER_INTO.has(t.word)) into = false;
+    if (t.word === 'DESCRIPTOR' && next?.host) {
+      // FETCH ... INTO DESCRIPTOR and USING DESCRIPTOR are synonyms: the program fills the SQLDA and Db2 writes where it points.
+      if (verb === 'FETCH' || intoBoth) { read(next.host); write(next.host); } else read(next.host);
+      i++;
+      continue;
+    }
+    if (!t.host) continue;
+    const written = into || assigning
+      || (verb === 'GET' && next?.word === '=')
+      || (verb === 'ASSOCIATE' && !toks.slice(0, i).some((x) => x.word === 'WITH'));
+    if (!written || (into && intoBoth)) read(t.host);
+    if (written) write(t.host);
+  }
+  return { sending, receiving };
+}
+
+// CALL :name reads the name. Each argument that is a lone variable, with or without its indicator,
+// is read and may be written; any other argument is an expression, IN only.
+function callRoles(toks, read, write) {
+  let depth = 0;
+  let arg = [];
+  const close = () => {
+    const hosts = arg.filter((t) => t.host);
+    const lone = hosts.length && arg.every((t) => t.host || t.word === 'INDICATOR');
+    for (const h of hosts) { read(h.host); if (lone) write(h.host); }
+    arg = [];
+  };
+  for (let i = 1; i < toks.length; i++) {
+    const t = toks[i];
+    if (t.word === 'DESCRIPTOR' && toks[i + 1]?.host) { read(toks[i + 1].host); write(toks[i + 1].host); i++; continue; }
+    if (t.word === '(') { if (depth++ > 0) arg.push(t); continue; }
+    if (t.word === ')') { if (--depth > 0) arg.push(t); else close(); continue; }
+    if (depth === 0) { if (t.host) read(t.host); continue; }
+    if (depth === 1 && t.word === ',') { close(); continue; }
+    arg.push(t);
+  }
+}
+
+export function hostVariableRoles(sql) {
+  return sqlRoles(sqlTokens(sql));
+}
+
+// The host variables in a parsed EXEC SQL block: a colon, then a name qualified by joined periods as
+// :GROUP.FIELD. Each carries its path, the token naming the item, where it stands, and whether the
+// statement writes it by sqlRoles.
+export function hostVariablesIn(toks) {
+  const found = [];
+  const sql = [];
+  for (let k = 0; k < toks.length; k++) {
+    const t = toks[k];
+    if (t.t === 'op' && t.v === ':' && toks[k + 1] && toks[k + 1].t === 'word') {
+      const path = [toks[k + 1]];
+      let j = k + 2;
+      for (; j + 1 < toks.length && toks[j].t === 'period' && toks[j].joined && toks[j + 1].t === 'word'; j += 2) path.push(toks[j + 1]);
+      const host = path.map((p) => p.u).join('.');
+      found.push({ path, tok: path[path.length - 1], at: k + 1, host });
+      sql.push({ host });
+      k = j - 1;
+      continue;
+    }
+    sql.push(t.t === 'lit' ? { lit: t.v } : { word: String(t.u ?? t.v).toUpperCase() });
+  }
+  const { receiving } = sqlRoles(sql);
+  return found.map((h) => ({ ...h, written: receiving.includes(h.host) }));
+}