@portll/cobolwork 0.5.0 → 0.7.0

package/lib/compliance.mjs

@@ -71,6 +71,17 @@ for (const d of loaded) {
 
 export const clausesFor = (ruleId) => byRule.get(ruleId) || [];
 
+// The clauses each kind of evidence the tools write answers (feed/catalogue.mjs EVIDENCE).
+const byEvidence = new Map();
+for (const d of loaded) {
+  for (const r of d.evidence || []) {
+    if (!byEvidence.has(r.evidenceId)) byEvidence.set(r.evidenceId, []);
+    byEvidence.get(r.evidenceId).push({ framework: r.framework, clause: r.clause, ...(r.title ? { title: r.title } : {}) });
+  }
+}
+
+export const evidenceClausesFor = (evidenceId) => byEvidence.get(evidenceId) || [];
+
 // The whole map, in the shape the report's other rule maps use.
 export function complianceMap(ruleIds) {
   const out = {};

package/lib/consequence.mjs

@@ -59,6 +59,7 @@ const RULE_CLASSES = {
   'cics-transfer-to-variable-program': [],
   // A failure carried on from: what follows depends on the program, and severity decides.
   'cics-condition-ignored': [],
+  'pli-error-condition-ignored': [],
   // Disclosure of stored secrets and a weaker check; severity decides.
   'program-checks-stored-password': [],
   'password-case-folded-before-compare': [],
@@ -69,6 +70,7 @@ const RULE_CLASSES = {
   'vendor-privileged-command': [ESCALATES],
   'jcl-instream-security-command': [ESCALATES],
   'jcl-instream-credential': [ESCALATES],
+  'credential-in-source': [ESCALATES],
   'web-uri-carries-a-credential': [ESCALATES],
   'log-writes-a-credential': [ESCALATES],
   'zowe-config-secret-in-clear': [ESCALATES],
@@ -122,6 +124,8 @@ const RULE_CLASSES = {
   // have landed, where the input chose.
   'input-causes-abend-s0c7': [],
   'input-causes-abend': [],
+  'input-causes-hang': [],
+  'input-selects-program': [ESCALATES],
   'input-causes-abend-s0c4': [MUTATES],
   'input-causes-abend-subscript-range': [MUTATES],
   // Key zero or supervisor state is the operating system's authority, taken by the code itself.
@@ -130,6 +134,9 @@ const RULE_CLASSES = {
   // the linkage tables grant that, and the source does not show them. Severity decides.
   'hlasm-executes-built-instruction': [],
   'hlasm-cross-memory-service': [],
+  'hlasm-runtime-module-name': [ESCALATES],
+  // A module FETCH loads by a name held in a variable runs with the program's authority.
+  'pli-fetch-title-variable': [ESCALATES],
   // Disclosure in transit; severity decides.
   'web-client-opens-cleartext': [],
   'web-receive-length-exceeds-area': [MUTATES],
@@ -137,6 +144,16 @@ const RULE_CLASSES = {
   'icsf-single-length-des-key': [],
   'icsf-weak-hash': [],
   'icsf-fixed-initialization-vector': [],
+  // A definition the generation step or the scheduler refuses stops the program; severity decides.
+  'ims-senseg-unknown-segment': [],
+  'ims-definition-inconsistent': [],
+  // An option granted and not used is authority the program holds, not a change any input makes.
+  'ims-procopt-broader-than-used': [],
+  // A grant decides who holds a privilege, not that anyone uses it; the PUBLIC grant of a write or an
+  // authority is crit, and severity decides the rest.
+  'db2-grant-to-public': [],
+  'db2-grant-with-grant-option': [],
+  'db2-system-authority-granted': [],
 };
 
 // Coverage and context assert no defect, so they carry no consequence.

package/lib/control-reuse.mjs

@@ -0,0 +1,113 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// A program's control analysis, kept so that a byte-identical copy of it elsewhere in the repository
+// is not analysed again.
+//
+// The analysis refers to objects of the program's parse: statements, items, tokens. Kept, each such
+// reference is the object's position in a walk of the parse that visits it in one fixed order, so the
+// copy's parse, walked the same way, supplies the copy's own objects. A string equal to one of the
+// program's file paths is kept as that file's place in the program's list of files, and becomes the
+// copy's path. Anything else the analysis holds is copied as it is. A path inside a longer string, or
+// a value that is not plain data, makes the analysis not kept at all: the copy is analysed afresh.
+
+const FAIL = Symbol('not kept');
+
+// The objects reachable from root, breadth first: array elements, Map keys and values, Set values and
+// own enumerable properties, in their order. Typed arrays are not entered.
+export function parseOrder(root) {
+  const order = [root];
+  const seen = new Set(order);
+  for (let i = 0; i < order.length; i++) {
+    const o = order[i];
+    const visit = (v) => { if (v !== null && typeof v === 'object' && !seen.has(v)) { seen.add(v); order.push(v); } };
+    if (ArrayBuffer.isView(o)) continue;
+    if (Array.isArray(o)) for (const v of o) visit(v);
+    else if (o instanceof Map) for (const [k, v] of o) { visit(k); visit(v); }
+    else if (o instanceof Set) for (const v of o) visit(v);
+    else for (const k of Object.keys(o)) visit(o[k]);
+  }
+  return order;
+}
+
+const plain = (o) => {
+  const proto = Object.getPrototypeOf(o);
+  return (proto === Object.prototype || proto === null)
+    && Object.getOwnPropertySymbols(o).length === 0
+    && Object.getOwnPropertyNames(o).length === Object.keys(o).length;
+};
+
+// `files` lists the program's file and its resolved copybooks, in the parse's order.
+export function keep(value, order, files) {
+  const at = new Map(order.map((o, i) => [o, i]));
+  const fileAt = new Map(files.map((f, i) => [f, i]));
+  const table = [];
+  const index = new Map();
+  const enc = (v) => {
+    if (typeof v === 'string') {
+      if (fileAt.has(v)) return { f: fileAt.get(v) };
+      for (const f of files) if (f && v.includes(f)) throw FAIL;
+      return v;
+    }
+    if (v === null || typeof v !== 'object') {
+      if (typeof v === 'function' || typeof v === 'symbol') throw FAIL;
+      return v;
+    }
+    if (at.has(v)) return { p: at.get(v) };
+    if (index.has(v)) return { r: index.get(v) };
+    const i = table.length;
+    index.set(v, i);
+    const entry = {};
+    table.push(entry);
+    if (ArrayBuffer.isView(v)) { entry.t = v; return { r: i }; }
+    if (Array.isArray(v)) {
+      if (Object.keys(v).length !== v.length || !plainArray(v)) throw FAIL;
+      entry.a = v.map(enc);
+    } else if (v instanceof Map) entry.m = [...v].map(([k, x]) => [enc(k), enc(x)]);
+    else if (v instanceof Set) entry.s = [...v].map(enc);
+    else {
+      if (!plain(v)) throw FAIL;
+      entry.o = Object.keys(v).map((k) => [k, enc(v[k])]);
+      if (Object.getPrototypeOf(v) === null) entry.n = true;
+    }
+    return { r: i };
+  };
+  try { return { root: enc(value), table }; } catch (e) { if (e === FAIL) return null; throw e; }
+}
+
+const plainArray = (a) => Object.getPrototypeOf(a) === Array.prototype && Object.getOwnPropertySymbols(a).length === 0;
+
+export function restore(kept, order, files) {
+  const made = kept.table.map((e) => (e.t ? e.t : e.a ? [] : e.m ? new Map() : e.s ? new Set() : e.n ? Object.create(null) : {}));
+  const dec = (x) => {
+    if (x === null || typeof x !== 'object') return x;
+    if ('p' in x) return order[x.p];
+    if ('f' in x) return files[x.f];
+    return made[x.r];
+  };
+  kept.table.forEach((e, i) => {
+    const o = made[i];
+    if (e.a) for (const x of e.a) o.push(dec(x));
+    else if (e.m) for (const [k, x] of e.m) o.set(dec(k), dec(x));
+    else if (e.s) for (const x of e.s) o.add(dec(x));
+    else if (e.o) for (const [k, x] of e.o) o[k] = dec(x);
+  });
+  return dec(kept.root);
+}
+
+// Whether two analyses are the same, objects of the parse compared as themselves.
+export function sameAnalysis(a, b, parse) {
+  const pairs = new Map();
+  const eq = (x, y) => {
+    if (x === y) return true;
+    if (x === null || y === null || typeof x !== 'object' || typeof y !== 'object') return Object.is(x, y);
+    if (parse.has(x) || parse.has(y)) return false;
+    if (pairs.has(x)) return pairs.get(x) === y;
+    pairs.set(x, y);
+    if (Object.getPrototypeOf(x) !== Object.getPrototypeOf(y)) return false;
+    if (ArrayBuffer.isView(x)) return x.length === y.length && x.every((v, i) => v === y[i]);
+    if (x instanceof Map) { if (x.size !== y.size) return false; const ys = [...y]; return [...x].every(([k, v], i) => eq(k, ys[i][0]) && eq(v, ys[i][1])); }
+    if (x instanceof Set) { if (x.size !== y.size) return false; const ys = [...y]; return [...x].every((v, i) => eq(v, ys[i])); }
+    const kx = Object.keys(x), ky = Object.keys(y);
+    return kx.length === ky.length && kx.every((k, i) => k === ky[i] && eq(x[k], y[k]));
+  };
+  return eq(a, b);
+}

package/lib/control-workers.mjs

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Control analyses of the programs ahead of the one the flow set is summarising, built in worker
+// threads.
+//
+// A worker parses a file as the directory tree does (tree.parseSpec), runs buildControl on each
+// program in it and posts each analysis kept as lib/control-reuse.mjs keeps a copy's: references into
+// the parse as positions in a fixed walk of it. The main thread restores that onto its own parse of the
+// same file, so the analysis it uses is made of its own objects. The scan stays synchronous: waiting
+// for an answer blocks on a counter the workers raise as they post.
+//
+// A file no worker has started when the main thread reaches it is taken back, and the main thread
+// analyses it itself. Only a started one is waited for.
+import { availableParallelism } from 'node:os';
+import { getHeapStatistics } from 'node:v8';
+import { MessageChannel, Worker, isMainThread, parentPort, receiveMessageOnPort, workerData } from 'node:worker_threads';
+import { buildControl, internFacts, namesOf } from './control.mjs';
+import { keep, parseOrder } from './control-reuse.mjs';
+import { parseInDirectory } from './kernel/source-tree.mjs';
+
+export const CONTROL_WORKERS_ENV = 'COBOLWORK_CONTROL_WORKERS';
+// Below this much program text, restoring an analysis on the main thread costs about what building
+// it does: on ACAS, 410 programs, moving every one saved 1 s of 3.3 s.
+export const CONTROL_WORKER_MIN_BYTES = 32 * 1024;
+
+// A task is the index of a wanted file. Workers claim tasks in order through a shared counter and
+// start one only once the main thread's limit has passed it, so a task never waits behind a busy
+// worker while another is idle.
+const QUEUED = 0;
+const STARTED = 1;
+const CANCELLED = 2;
+const CLAIM = 0;
+const LIMIT = 1;
+
+// The program's file and the copybooks its parse resolved, which the kept analysis names by position.
+export const programFiles = (r, file) => [file, ...r.copies.filter((c) => c.path).map((c) => c.path)];
+
+// Each program of a parsed file: { value }, its analysis kept; { none }, no procedure division;
+// { threw }, buildControl threw; { local }, the analysis holds something keep() does not carry.
+export function analyseFile(r, file) {
+  const files = programFiles(r, file);
+  return r.programs.map((prog) => {
+    let ctl;
+    try { ctl = buildControl(prog, namesOf(prog).resolve); } catch { return { threw: true }; }
+    if (!ctl) return { none: true };
+    internFacts(ctl.facts);
+    const value = keep(ctl, parseOrder(prog), files);
+    return value ? { value } : { local: true };
+  });
+}
+
+function serve({ spec, port, signal, control, states, tasks }) {
+  for (;;) {
+    const t = Atomics.add(control, CLAIM, 1);
+    if (t >= tasks.length) return;
+    for (let limit = Atomics.load(control, LIMIT); t >= limit; limit = Atomics.load(control, LIMIT)) Atomics.wait(control, LIMIT, limit);
+    if (Atomics.compareExchange(states, t, QUEUED, STARTED) !== QUEUED) continue;
+    let programs = null;
+    try { programs = analyseFile(parseInDirectory(spec, tasks[t]), tasks[t]); } catch { /* the main thread's own parse reports it */ }
+    try { port.postMessage({ t, programs }); } catch { port.postMessage({ t, programs: null }); }
+    Atomics.add(signal, 0, 1);
+    Atomics.notify(signal, 0);
+  }
+}
+
+if (!isMainThread && workerData?.controlWorker) serve(workerData);
+
+
+// How many workers a scan starts: COBOLWORK_CONTROL_WORKERS, else one fewer than the machine's
+// threads, at most four. Each holds one program's parse and analysis at a time, up to 1.3 GB on
+// programs of 15 MB.
+export function controlWorkerCount() {
+  const n = Number(process.env[CONTROL_WORKERS_ENV]);
+  if (process.env[CONTROL_WORKERS_ENV] !== undefined && Number.isInteger(n) && n >= 0) return n;
+  return Math.max(0, Math.min(4, availableParallelism() - 1));
+}
+
+// Analyses of `files` built ahead of the main thread, for those `wanted` selects. take(file, index)
+// returns the file's answer, one entry per program, or null where the main thread must analyse it.
+// Files are asked for in order. Workers start at most `ahead` wanted files past the one the main
+// thread is on, since an answer waiting to be taken holds the whole analysis. Twice the workers by
+// default: a large program's analysis takes as long as the main thread's own work on several files,
+// and on cnafbadboy's Batch, scheduled from its measured costs, one more than the workers leaves the
+// main thread waiting a fifth of its loop and twice the workers about 1%.
+//
+// Every failure a worker can catch is answered. One that runs out of heap dies without answering, and
+// nothing the main thread can read while it waits says so. A worker's heap is as large as the main
+// thread's, so a program that exhausts it would exhaust the main thread's too: the wait for a started
+// analysis is long rather than tuned, and past it the main thread analyses the program itself.
+export function controlAhead(spec, files, wanted, { workers, ahead = 2 * workers, waitMs = 30 * 60_000 }) {
+  const tasks = [];
+  const indexOf = [];
+  files.forEach((f, i) => { if (wanted(f)) { tasks.push(f); indexOf.push(i); } });
+  const signal = new Int32Array(new SharedArrayBuffer(4));
+  const control = new Int32Array(new SharedArrayBuffer(8));
+  const states = new Int32Array(new SharedArrayBuffer(4 * Math.max(1, tasks.length)));
+  const maxOldGenerationSizeMb = Math.ceil(getHeapStatistics().heap_size_limit / (1024 * 1024));
+  const startWorker = () => {
+    const { port1, port2 } = new MessageChannel();
+    const worker = new Worker(new URL(import.meta.url), {
+      workerData: { controlWorker: true, spec, port: port2, signal, control, states, tasks },
+      transferList: [port2],
+      resourceLimits: { maxOldGenerationSizeMb },
+    });
+    worker.on('error', () => {});
+    worker.unref();
+    port1.unref();
+    return { worker, port: port1 };
+  };
+  const pool = tasks.length ? Array.from({ length: workers }, startWorker) : [];
+  // The first task the main thread has neither taken nor passed, and the answers from it on.
+  let next = 0;
+  const answers = new Map();
+  const drain = () => {
+    for (const w of pool) {
+      for (let m = receiveMessageOnPort(w.port); m; m = receiveMessageOnPort(w.port)) {
+        if (m.message.t >= next) answers.set(m.message.t, m.message.programs);
+      }
+    }
+  };
+  const pass = () => {
+    Atomics.compareExchange(states, next, QUEUED, CANCELLED);
+    answers.delete(next++);
+  };
+  const offer = (upTo) => {
+    const limit = Math.min(tasks.length, upTo);
+    if (limit <= Atomics.load(control, LIMIT)) return;
+    Atomics.store(control, LIMIT, limit);
+    Atomics.notify(control, LIMIT);
+  };
+  return {
+    take(file, index) {
+      drain();
+      while (next < tasks.length && indexOf[next] < index) pass();
+      if (next >= tasks.length || indexOf[next] !== index) { offer(next + ahead); return null; }
+      const t = next;
+      const notStarted = Atomics.compareExchange(states, t, QUEUED, CANCELLED) === QUEUED;
+      offer(t + 1 + ahead);
+      let programs = null;
+      if (!notStarted) {
+        const deadline = Date.now() + waitMs;
+        for (;;) {
+          const seen = Atomics.load(signal, 0);
+          drain();
+          if (answers.has(t) || Date.now() > deadline) break;
+          Atomics.wait(signal, 0, seen, 200);
+        }
+        programs = answers.get(t) ?? null;
+      }
+      answers.delete(t);
+      next = t + 1;
+      return programs;
+    },
+    close() {
+      for (let t = next; t < tasks.length; t++) Atomics.compareExchange(states, t, QUEUED, CANCELLED);
+      Atomics.store(control, LIMIT, tasks.length);
+      Atomics.notify(control, LIMIT);
+      for (const w of pool) { w.port.close(); w.worker.terminate(); }
+      answers.clear();
+    },
+  };
+}

package/lib/control.mjs

@@ -21,6 +21,7 @@
 // paragraph, for the same reason.
 import { segmentEnd } from './parser.mjs';
 import { parser, attempt, candidates, leaves, storedRange } from './arith.mjs';
+import { execReading } from './exec-reading.mjs';
 
 const WORD = (t, u) => t && t.t === 'word' && t.u === u;
 const PHRASED = new Set(['READ', 'WRITE', 'REWRITE', 'DELETE', 'START', 'RETURN', 'CALL', 'COMPUTE', 'ADD', 'SUBTRACT',
@@ -267,7 +268,7 @@ export function buildControl(prog, resolve, { extra = null } = {}) {
   const twice = new Set();
   for (const l of prog.labels) { if (seenLabel.has(l.name)) twice.add(l.name); seenLabel.add(l.name); }
   const sure = (name) => labelNames.has(name) && !twice.has(name);
-  const wordsOf = (e) => e.toks.filter((x) => x.t === 'word').map((x) => x.u);
+  const wordsOf = (e) => execReading(e).words;
   // Once IGNORE CONDITION has run, a RETURN or XCTL that fails comes back to the next statement, and
   // whether it has run by then is the order of execution, not of the source. A HANDLE CONDITION sends
   // the failure to its label, but which one is in force is as uncertain, so reach assumes it may not.
@@ -698,17 +699,23 @@ export function buildControl(prog, resolve, { extra = null } = {}) {
         }
       }
     } else if (n.e) {
-      // A command may fill what any option is given directly, except what it sends FROM and the
+      // A command fills the options the precompile table says CICS returns a value in. One the table
+      // does not hold may fill what any option is given directly, except what it sends FROM and the
       // data-values it reads. A name inside that argument's own parentheses - a subscript, a
       // reference modification - is read.
       const reads = (n.e.kind === 'CICS' && CICS_READS[n.e.toks[0]?.u]) || [];
+      const direction = n.e.kind === 'CICS' ? execReading(n.e).direction : () => null;
+      const fills = (option) => {
+        const d = direction(option);
+        return d ? d === 'receives' || d === 'both' : option !== 'FROM' && !reads.includes(option);
+      };
       let depth = 0;
       let option = null;
       for (let i = 0; i < n.e.toks.length; i++) {
         const t = n.e.toks[i];
         if (t.t === 'sep') { depth += t.v === '(' ? 1 : -1; continue; }
         if (depth === 0) { option = t.t === 'word' ? t.u : option; continue; }
-        if (depth === 1 && t.t === 'word' && option !== 'FROM' && !reads.includes(option)) add(t);
+        if (depth === 1 && t.t === 'word' && fills(option)) add(t);
       }
     }
     return out;
@@ -2950,7 +2957,73 @@ function readersOf(tree) {
   return out;
 }
 
-export const hasFact = (bits, x) => x != null && bits != null && (bits[x >>> 5] & (1 << (x & 31))) !== 0;
+// A fact set is a Uint32Array with one bit per fact, or, where a program keeps it after its analysis,
+// a Uint16Array holding the sorted numbers of the facts that hold (sparseFacts).
+export const hasFact = (bits, x) => x != null && bits != null
+  && (bits instanceof Uint16Array ? holdsSorted(bits, x) : (bits[x >>> 5] & (1 << (x & 31))) !== 0);
+function holdsSorted(list, x) {
+  let lo = 0;
+  let hi = list.length - 1;
+  while (lo <= hi) {
+    const mid = (lo + hi) >>> 1;
+    if (list[mid] === x) return true;
+    if (list[mid] < x) lo = mid + 1; else hi = mid - 1;
+  }
+  return false;
+}
+
+// The facts a set holds as their sorted numbers, where every number fits 16 bits; the set itself otherwise.
+export function sparseFacts(bits) {
+  if (bits.length * 32 > 65536) return bits;
+  let n = 0;
+  for (let i = 0; i < bits.length; i++) for (let w = bits[i]; w; w &= w - 1) n++;
+  const out = new Uint16Array(n);
+  let k = 0;
+  for (let i = 0; i < bits.length; i++) for (let w = bits[i]; w; w &= w - 1) out[k++] = i * 32 + (31 - Math.clz32(w & -w));
+  return out;
+}
+
+// The facts a program keeps per statement: one copy of each distinct set, held as the numbers of the
+// facts in it, all in one buffer per program. Neighbouring statements mostly hold the same checks, and
+// few of a program's checks hold at any one statement: 26 large programs kept 1.5 GB of bitsets, 0.4%
+// of their bits set.
+export function internFacts(facts) {
+  const byHash = new Map();
+  const kept = [];
+  const keptAt = new Map();
+  for (const [point, bits] of facts) {
+    let h = 2166136261;
+    for (let i = 0; i < bits.length; i++) h = Math.imul(h ^ bits[i], 16777619);
+    let same = byHash.get(h);
+    if (!same) byHash.set(h, same = []);
+    let k = same.find((x) => x.bits.length === bits.length && x.bits.every((v, i) => v === bits[i]));
+    if (!k) { same.push(k = { bits, held: sparseFacts(bits) }); kept.push(k); }
+    keptAt.set(point, k);
+  }
+  let size = 0;
+  for (const k of kept) if (k.held instanceof Uint16Array) size += k.held.length;
+  const pool = new Uint16Array(size);
+  let at = 0;
+  for (const k of kept) {
+    if (!(k.held instanceof Uint16Array)) continue;
+    pool.set(k.held, at);
+    k.held = pool.subarray(at, at += k.held.length);
+  }
+  for (const [point, k] of keptAt) facts.set(point, k.held);
+  return facts;
+}
+
+// The name resolution the flow analysis hands buildControl: the parser's reference resolution, then the
+// first item of the name, since names inside EXEC blocks never pass through the parser's, then an
+// INDEXED BY name, which is not a data item.
+export function namesOf(prog) {
+  const byName = new Map();
+  for (const it of prog.items) if (!byName.has(it.name)) byName.set(it.name, it);
+  const itemOfToken = (tok) => (tok ? prog.resolved.get(tok) || (tok.t === 'word' ? byName.get(tok.u) || null : null) : null);
+  const indexNames = new Set(prog.items.flatMap((it) => it.indexNames || []));
+  const resolve = (tok) => itemOfToken(tok) || (tok && tok.t === 'word' && indexNames.has(tok.u) ? { index: tok.u } : null);
+  return { byName, itemOfToken, indexNames, resolve };
+}
 
 // What a node's checks say at a point, given the facts that hold there: 2 when the value it holds is
 // safe for the sink, 1 when a check has run on every route to the point, 0 when neither. The check