@portll/cobolwork 0.6.0 → 0.7.0

package/lib/compliance.mjs

@@ -71,6 +71,17 @@ for (const d of loaded) {
 
 export const clausesFor = (ruleId) => byRule.get(ruleId) || [];
 
+// The clauses each kind of evidence the tools write answers (feed/catalogue.mjs EVIDENCE).
+const byEvidence = new Map();
+for (const d of loaded) {
+  for (const r of d.evidence || []) {
+    if (!byEvidence.has(r.evidenceId)) byEvidence.set(r.evidenceId, []);
+    byEvidence.get(r.evidenceId).push({ framework: r.framework, clause: r.clause, ...(r.title ? { title: r.title } : {}) });
+  }
+}
+
+export const evidenceClausesFor = (evidenceId) => byEvidence.get(evidenceId) || [];
+
 // The whole map, in the shape the report's other rule maps use.
 export function complianceMap(ruleIds) {
   const out = {};

package/lib/consequence.mjs

@@ -59,6 +59,7 @@ const RULE_CLASSES = {
   'cics-transfer-to-variable-program': [],
   // A failure carried on from: what follows depends on the program, and severity decides.
   'cics-condition-ignored': [],
+  'pli-error-condition-ignored': [],
   // Disclosure of stored secrets and a weaker check; severity decides.
   'program-checks-stored-password': [],
   'password-case-folded-before-compare': [],
@@ -134,6 +135,8 @@ const RULE_CLASSES = {
   'hlasm-executes-built-instruction': [],
   'hlasm-cross-memory-service': [],
   'hlasm-runtime-module-name': [ESCALATES],
+  // A module FETCH loads by a name held in a variable runs with the program's authority.
+  'pli-fetch-title-variable': [ESCALATES],
   // Disclosure in transit; severity decides.
   'web-client-opens-cleartext': [],
   'web-receive-length-exceeds-area': [MUTATES],
@@ -141,6 +144,16 @@ const RULE_CLASSES = {
   'icsf-single-length-des-key': [],
   'icsf-weak-hash': [],
   'icsf-fixed-initialization-vector': [],
+  // A definition the generation step or the scheduler refuses stops the program; severity decides.
+  'ims-senseg-unknown-segment': [],
+  'ims-definition-inconsistent': [],
+  // An option granted and not used is authority the program holds, not a change any input makes.
+  'ims-procopt-broader-than-used': [],
+  // A grant decides who holds a privilege, not that anyone uses it; the PUBLIC grant of a write or an
+  // authority is crit, and severity decides the rest.
+  'db2-grant-to-public': [],
+  'db2-grant-with-grant-option': [],
+  'db2-system-authority-granted': [],
 };
 
 // Coverage and context assert no defect, so they carry no consequence.

package/lib/control-workers.mjs

@@ -0,0 +1,161 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Control analyses of the programs ahead of the one the flow set is summarising, built in worker
+// threads.
+//
+// A worker parses a file as the directory tree does (tree.parseSpec), runs buildControl on each
+// program in it and posts each analysis kept as lib/control-reuse.mjs keeps a copy's: references into
+// the parse as positions in a fixed walk of it. The main thread restores that onto its own parse of the
+// same file, so the analysis it uses is made of its own objects. The scan stays synchronous: waiting
+// for an answer blocks on a counter the workers raise as they post.
+//
+// A file no worker has started when the main thread reaches it is taken back, and the main thread
+// analyses it itself. Only a started one is waited for.
+import { availableParallelism } from 'node:os';
+import { getHeapStatistics } from 'node:v8';
+import { MessageChannel, Worker, isMainThread, parentPort, receiveMessageOnPort, workerData } from 'node:worker_threads';
+import { buildControl, internFacts, namesOf } from './control.mjs';
+import { keep, parseOrder } from './control-reuse.mjs';
+import { parseInDirectory } from './kernel/source-tree.mjs';
+
+export const CONTROL_WORKERS_ENV = 'COBOLWORK_CONTROL_WORKERS';
+// Below this much program text, restoring an analysis on the main thread costs about what building
+// it does: on ACAS, 410 programs, moving every one saved 1 s of 3.3 s.
+export const CONTROL_WORKER_MIN_BYTES = 32 * 1024;
+
+// A task is the index of a wanted file. Workers claim tasks in order through a shared counter and
+// start one only once the main thread's limit has passed it, so a task never waits behind a busy
+// worker while another is idle.
+const QUEUED = 0;
+const STARTED = 1;
+const CANCELLED = 2;
+const CLAIM = 0;
+const LIMIT = 1;
+
+// The program's file and the copybooks its parse resolved, which the kept analysis names by position.
+export const programFiles = (r, file) => [file, ...r.copies.filter((c) => c.path).map((c) => c.path)];
+
+// Each program of a parsed file: { value }, its analysis kept; { none }, no procedure division;
+// { threw }, buildControl threw; { local }, the analysis holds something keep() does not carry.
+export function analyseFile(r, file) {
+  const files = programFiles(r, file);
+  return r.programs.map((prog) => {
+    let ctl;
+    try { ctl = buildControl(prog, namesOf(prog).resolve); } catch { return { threw: true }; }
+    if (!ctl) return { none: true };
+    internFacts(ctl.facts);
+    const value = keep(ctl, parseOrder(prog), files);
+    return value ? { value } : { local: true };
+  });
+}
+
+function serve({ spec, port, signal, control, states, tasks }) {
+  for (;;) {
+    const t = Atomics.add(control, CLAIM, 1);
+    if (t >= tasks.length) return;
+    for (let limit = Atomics.load(control, LIMIT); t >= limit; limit = Atomics.load(control, LIMIT)) Atomics.wait(control, LIMIT, limit);
+    if (Atomics.compareExchange(states, t, QUEUED, STARTED) !== QUEUED) continue;
+    let programs = null;
+    try { programs = analyseFile(parseInDirectory(spec, tasks[t]), tasks[t]); } catch { /* the main thread's own parse reports it */ }
+    try { port.postMessage({ t, programs }); } catch { port.postMessage({ t, programs: null }); }
+    Atomics.add(signal, 0, 1);
+    Atomics.notify(signal, 0);
+  }
+}
+
+if (!isMainThread && workerData?.controlWorker) serve(workerData);
+
+
+// How many workers a scan starts: COBOLWORK_CONTROL_WORKERS, else one fewer than the machine's
+// threads, at most four. Each holds one program's parse and analysis at a time, up to 1.3 GB on
+// programs of 15 MB.
+export function controlWorkerCount() {
+  const n = Number(process.env[CONTROL_WORKERS_ENV]);
+  if (process.env[CONTROL_WORKERS_ENV] !== undefined && Number.isInteger(n) && n >= 0) return n;
+  return Math.max(0, Math.min(4, availableParallelism() - 1));
+}
+
+// Analyses of `files` built ahead of the main thread, for those `wanted` selects. take(file, index)
+// returns the file's answer, one entry per program, or null where the main thread must analyse it.
+// Files are asked for in order. Workers start at most `ahead` wanted files past the one the main
+// thread is on, since an answer waiting to be taken holds the whole analysis. Twice the workers by
+// default: a large program's analysis takes as long as the main thread's own work on several files,
+// and on cnafbadboy's Batch, scheduled from its measured costs, one more than the workers leaves the
+// main thread waiting a fifth of its loop and twice the workers about 1%.
+//
+// Every failure a worker can catch is answered. One that runs out of heap dies without answering, and
+// nothing the main thread can read while it waits says so. A worker's heap is as large as the main
+// thread's, so a program that exhausts it would exhaust the main thread's too: the wait for a started
+// analysis is long rather than tuned, and past it the main thread analyses the program itself.
+export function controlAhead(spec, files, wanted, { workers, ahead = 2 * workers, waitMs = 30 * 60_000 }) {
+  const tasks = [];
+  const indexOf = [];
+  files.forEach((f, i) => { if (wanted(f)) { tasks.push(f); indexOf.push(i); } });
+  const signal = new Int32Array(new SharedArrayBuffer(4));
+  const control = new Int32Array(new SharedArrayBuffer(8));
+  const states = new Int32Array(new SharedArrayBuffer(4 * Math.max(1, tasks.length)));
+  const maxOldGenerationSizeMb = Math.ceil(getHeapStatistics().heap_size_limit / (1024 * 1024));
+  const startWorker = () => {
+    const { port1, port2 } = new MessageChannel();
+    const worker = new Worker(new URL(import.meta.url), {
+      workerData: { controlWorker: true, spec, port: port2, signal, control, states, tasks },
+      transferList: [port2],
+      resourceLimits: { maxOldGenerationSizeMb },
+    });
+    worker.on('error', () => {});
+    worker.unref();
+    port1.unref();
+    return { worker, port: port1 };
+  };
+  const pool = tasks.length ? Array.from({ length: workers }, startWorker) : [];
+  // The first task the main thread has neither taken nor passed, and the answers from it on.
+  let next = 0;
+  const answers = new Map();
+  const drain = () => {
+    for (const w of pool) {
+      for (let m = receiveMessageOnPort(w.port); m; m = receiveMessageOnPort(w.port)) {
+        if (m.message.t >= next) answers.set(m.message.t, m.message.programs);
+      }
+    }
+  };
+  const pass = () => {
+    Atomics.compareExchange(states, next, QUEUED, CANCELLED);
+    answers.delete(next++);
+  };
+  const offer = (upTo) => {
+    const limit = Math.min(tasks.length, upTo);
+    if (limit <= Atomics.load(control, LIMIT)) return;
+    Atomics.store(control, LIMIT, limit);
+    Atomics.notify(control, LIMIT);
+  };
+  return {
+    take(file, index) {
+      drain();
+      while (next < tasks.length && indexOf[next] < index) pass();
+      if (next >= tasks.length || indexOf[next] !== index) { offer(next + ahead); return null; }
+      const t = next;
+      const notStarted = Atomics.compareExchange(states, t, QUEUED, CANCELLED) === QUEUED;
+      offer(t + 1 + ahead);
+      let programs = null;
+      if (!notStarted) {
+        const deadline = Date.now() + waitMs;
+        for (;;) {
+          const seen = Atomics.load(signal, 0);
+          drain();
+          if (answers.has(t) || Date.now() > deadline) break;
+          Atomics.wait(signal, 0, seen, 200);
+        }
+        programs = answers.get(t) ?? null;
+      }
+      answers.delete(t);
+      next = t + 1;
+      return programs;
+    },
+    close() {
+      for (let t = next; t < tasks.length; t++) Atomics.compareExchange(states, t, QUEUED, CANCELLED);
+      Atomics.store(control, LIMIT, tasks.length);
+      Atomics.notify(control, LIMIT);
+      for (const w of pool) { w.port.close(); w.worker.terminate(); }
+      answers.clear();
+    },
+  };
+}

package/lib/control.mjs

@@ -21,6 +21,7 @@
 // paragraph, for the same reason.
 import { segmentEnd } from './parser.mjs';
 import { parser, attempt, candidates, leaves, storedRange } from './arith.mjs';
+import { execReading } from './exec-reading.mjs';
 
 const WORD = (t, u) => t && t.t === 'word' && t.u === u;
 const PHRASED = new Set(['READ', 'WRITE', 'REWRITE', 'DELETE', 'START', 'RETURN', 'CALL', 'COMPUTE', 'ADD', 'SUBTRACT',
@@ -267,7 +268,7 @@ export function buildControl(prog, resolve, { extra = null } = {}) {
   const twice = new Set();
   for (const l of prog.labels) { if (seenLabel.has(l.name)) twice.add(l.name); seenLabel.add(l.name); }
   const sure = (name) => labelNames.has(name) && !twice.has(name);
-  const wordsOf = (e) => e.toks.filter((x) => x.t === 'word').map((x) => x.u);
+  const wordsOf = (e) => execReading(e).words;
   // Once IGNORE CONDITION has run, a RETURN or XCTL that fails comes back to the next statement, and
   // whether it has run by then is the order of execution, not of the source. A HANDLE CONDITION sends
   // the failure to its label, but which one is in force is as uncertain, so reach assumes it may not.
@@ -698,17 +699,23 @@ export function buildControl(prog, resolve, { extra = null } = {}) {
         }
       }
     } else if (n.e) {
-      // A command may fill what any option is given directly, except what it sends FROM and the
+      // A command fills the options the precompile table says CICS returns a value in. One the table
+      // does not hold may fill what any option is given directly, except what it sends FROM and the
       // data-values it reads. A name inside that argument's own parentheses - a subscript, a
       // reference modification - is read.
       const reads = (n.e.kind === 'CICS' && CICS_READS[n.e.toks[0]?.u]) || [];
+      const direction = n.e.kind === 'CICS' ? execReading(n.e).direction : () => null;
+      const fills = (option) => {
+        const d = direction(option);
+        return d ? d === 'receives' || d === 'both' : option !== 'FROM' && !reads.includes(option);
+      };
       let depth = 0;
       let option = null;
       for (let i = 0; i < n.e.toks.length; i++) {
         const t = n.e.toks[i];
         if (t.t === 'sep') { depth += t.v === '(' ? 1 : -1; continue; }
         if (depth === 0) { option = t.t === 'word' ? t.u : option; continue; }
-        if (depth === 1 && t.t === 'word' && option !== 'FROM' && !reads.includes(option)) add(t);
+        if (depth === 1 && t.t === 'word' && fills(option)) add(t);
       }
     }
     return out;
@@ -2976,6 +2983,48 @@ export function sparseFacts(bits) {
   return out;
 }
 
+// The facts a program keeps per statement: one copy of each distinct set, held as the numbers of the
+// facts in it, all in one buffer per program. Neighbouring statements mostly hold the same checks, and
+// few of a program's checks hold at any one statement: 26 large programs kept 1.5 GB of bitsets, 0.4%
+// of their bits set.
+export function internFacts(facts) {
+  const byHash = new Map();
+  const kept = [];
+  const keptAt = new Map();
+  for (const [point, bits] of facts) {
+    let h = 2166136261;
+    for (let i = 0; i < bits.length; i++) h = Math.imul(h ^ bits[i], 16777619);
+    let same = byHash.get(h);
+    if (!same) byHash.set(h, same = []);
+    let k = same.find((x) => x.bits.length === bits.length && x.bits.every((v, i) => v === bits[i]));
+    if (!k) { same.push(k = { bits, held: sparseFacts(bits) }); kept.push(k); }
+    keptAt.set(point, k);
+  }
+  let size = 0;
+  for (const k of kept) if (k.held instanceof Uint16Array) size += k.held.length;
+  const pool = new Uint16Array(size);
+  let at = 0;
+  for (const k of kept) {
+    if (!(k.held instanceof Uint16Array)) continue;
+    pool.set(k.held, at);
+    k.held = pool.subarray(at, at += k.held.length);
+  }
+  for (const [point, k] of keptAt) facts.set(point, k.held);
+  return facts;
+}
+
+// The name resolution the flow analysis hands buildControl: the parser's reference resolution, then the
+// first item of the name, since names inside EXEC blocks never pass through the parser's, then an
+// INDEXED BY name, which is not a data item.
+export function namesOf(prog) {
+  const byName = new Map();
+  for (const it of prog.items) if (!byName.has(it.name)) byName.set(it.name, it);
+  const itemOfToken = (tok) => (tok ? prog.resolved.get(tok) || (tok.t === 'word' ? byName.get(tok.u) || null : null) : null);
+  const indexNames = new Set(prog.items.flatMap((it) => it.indexNames || []));
+  const resolve = (tok) => itemOfToken(tok) || (tok && tok.t === 'word' && indexNames.has(tok.u) ? { index: tok.u } : null);
+  return { byName, itemOfToken, indexNames, resolve };
+}
+
 // What a node's checks say at a point, given the facts that hold there: 2 when the value it holds is
 // safe for the sink, 1 when a check has run on every route to the point, 0 when neither. The check
 // named is the one that decided it: one whose outcome alone makes the value safe, if there is one.