@portll/cobolwork 0.2.150 → 0.4.0

package/lib/sets/abend.mjs

@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Abends an input caused, from the runs ironwork's fuzzing harness kept (docs/spec/evidence.md §13.6).
+//
+// A fuzz run is a directory: manifest.json names the program, the inputs and each run that ended in
+// an abend, and evidence/ holds each kept run's hash-chained journal. A finding is reported only
+// where the evidence verifies and the run's own journal records the abend the manifest claims, so
+// the finding rests on a record of the run rather than on the manifest's word.
+import { readFileSync, existsSync } from 'node:fs';
+import { delimiter, join, posix, basename } from 'node:path';
+import { report } from '../kernel/ruleset.mjs';
+import { verifyEvidence } from '../evidence/verify.mjs';
+
+export const ABEND_RULES = {
+  'input-causes-abend-s0c7': {
+    sev: 'med', evidence: 'execution', cwe: 'CWE-20',
+    text: 'An input the program accepts ends its run with a data exception (S0C7)',
+    impact: 'Whoever supplies that input can stop the program at will: a batch step abends and the job behind it does not finish, a transaction abends at the terminal',
+    remedy: 'Test the field with IS NUMERIC, or validate the record, before the arithmetic or MOVE the journal names, and reject the input rather than letting the data exception end the run',
+  },
+  'input-causes-abend-s0c4': {
+    sev: 'high', evidence: 'execution', cwe: 'CWE-119',
+    text: 'An input the program accepts ends its run with a protection exception (S0C4)',
+    impact: 'The input made the program address storage it does not own; the same input under another layout can read or overwrite other data before anything stops it',
+    remedy: 'Find the subscript, reference modification, pointer or parameter length the input controls at the line the journal names, and bound it before use',
+  },
+  'input-causes-abend-subscript-range': {
+    sev: 'high', evidence: 'execution', cwe: 'CWE-129',
+    text: 'An input the program accepts drives a subscript or reference modification out of range, and the run abends',
+    impact: 'SSRANGE caught the overrun and ended the run; compiled without SSRANGE, as production builds often are, the same input reads or writes storage beside the table or field',
+    remedy: 'Check the index against both ends of the table or field before the statement the journal names, and keep SSRANGE on for the build',
+  },
+  'input-causes-abend': {
+    sev: 'med', evidence: 'execution', cwe: 'CWE-248',
+    text: 'An input the program accepts ends its run with an abend',
+    impact: 'Whoever supplies that input can stop the program at will, and the abend code says what the program failed to handle',
+    remedy: 'Read the abend code and the line the journal names, and handle the condition the input raises before it ends the run',
+  },
+};
+
+// IBM's messages for an SSRANGE failure: a subscript or index, an OCCURS DEPENDING ON object, and a
+// reference modification's start, length, and start plus length.
+const RANGE_MESSAGES = /^IGZ00(06|07|72|73|74)S\b/;
+
+export function abendRunPaths(opts = {}) {
+  if (opts.abendRuns) return opts.abendRuns;
+  return (process.env.COBOLWORK_ABENDS || '').split(delimiter).filter(Boolean);
+}
+
+export function abendRule(abend) {
+  if (abend.code === 'S0C7') return 'input-causes-abend-s0c7';
+  if (abend.code === 'S0C4') return 'input-causes-abend-s0c4';
+  if (RANGE_MESSAGES.test(abend.message || '')) return 'input-causes-abend-subscript-range';
+  return 'input-causes-abend';
+}
+
+// The abend a kept run's own journal records, or null.
+function journalAbend(evidenceDir, runId) {
+  if (typeof runId !== 'string' || !/^[0-9TZ]+-[0-9a-f]{16}$/.test(runId)) return null;
+  const file = join(evidenceDir, 'runs', `${runId}.jsonl`);
+  if (!existsSync(file)) return null;
+  for (const line of readFileSync(file, 'utf8').split('\n')) {
+    if (!line) continue;
+    const rec = JSON.parse(line);
+    if (rec.kind === 'abend') return rec;
+  }
+  return null;
+}
+
+const isString = (x) => typeof x === 'string' && x.length > 0;
+
+// One fuzz run: its findings, and what kept any of it from counting.
+export function loadAbendRun(dir) {
+  const out = { dir: basename(dir), program: null, findings: [], problems: [], counts: null, notModelled: 0 };
+  let doc;
+  try { doc = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf8')); } catch (e) {
+    out.problems.push(e.code ? `manifest.json could not be read (${e.code})` : `manifest.json is not JSON (${e.message})`);
+    return out;
+  }
+  if (!doc || doc.tool !== 'ironwork-fuzz' || !doc.program || !isString(doc.program.file) || !Array.isArray(doc.inputs) || !Array.isArray(doc.runs)) {
+    out.problems.push('manifest.json is not an ironwork-fuzz manifest with a program, inputs and runs');
+    return out;
+  }
+  const programFile = posix.normalize(doc.program.file.replace(/\\/g, '/'));
+  if (programFile.startsWith('../') || posix.isAbsolute(programFile)) {
+    out.problems.push(`the program ${doc.program.file} is outside the scanned tree`);
+    return out;
+  }
+  out.program = { file: programFile, id: isString(doc.program.id) ? doc.program.id : null };
+  out.counts = doc.counts && typeof doc.counts === 'object' ? doc.counts : null;
+
+  const evidenceDir = join(dir, 'evidence');
+  const verdict = verifyEvidence(evidenceDir);
+  if (!verdict.verified) {
+    out.problems.push(`its evidence does not verify: ${verdict.broken.map((b) => `${b.file}:${b.line} ${b.check}`).join(', ') || 'nothing is recorded'}`);
+    return out;
+  }
+  const inputs = new Map(doc.inputs.filter((i) => i && isString(i.id)).map((i) => [i.id, i]));
+  const seen = new Map();
+  for (const run of doc.runs) {
+    if (!run || run.outcome !== 'abend' || !run.abend || !isString(run.abend.code)) continue;
+    if (run.abend.code === 'IRONWORK') { out.notModelled++; continue; }
+    const recorded = journalAbend(evidenceDir, run.journal);
+    if (!recorded || recorded.code !== run.abend.code) {
+      out.problems.push(`run ${run.journal ?? '(unnamed)'}: its journal does not record the abend ${run.abend.code} the manifest gives`);
+      continue;
+    }
+    // Where the journal names the abend's place, that is the place, and a manifest that says otherwise is not believed.
+    const claimed = { file: run.abend.file, line: run.abend.line };
+    const place = isString(recorded.file) && Number.isInteger(recorded.line) ? { file: recorded.file, line: recorded.line } : claimed;
+    if (place !== claimed && ((isString(claimed.file) && claimed.file !== place.file) || (claimed.line != null && claimed.line !== place.line))) {
+      out.problems.push(`run ${run.journal}: its journal records the abend at ${place.file}:${place.line}, not where the manifest puts it`);
+      continue;
+    }
+    const where = isString(place.file) ? posix.normalize(posix.join(posix.dirname(programFile), place.file.replace(/\\/g, '/'))) : programFile;
+    const line = Number.isInteger(place.line) && place.line > 0 ? place.line : 1;
+    const rule = abendRule(run.abend);
+    const used = (Array.isArray(run.input) ? run.input : []).map((id) => inputs.get(id)).filter(Boolean)
+      .map((i) => ({ kind: i.kind, name: i.name, bytes: i.bytes, ...(i.minimized ? { minimized: true } : {}) }));
+    const key = `${rule}|${where}|${line}`;
+    const size = used.reduce((n, i) => n + String(i.bytes || '').length, 0);
+    const held = seen.get(key);
+    if (held && held.size <= size) continue;
+    const finding = {
+      rule, path: where, line, program: out.program.id,
+      detail: `${out.program.id || programFile} ended with ${run.abend.code}${isString(run.abend.message) ? ` (${run.abend.message})` : ''} on an input the fuzz run ${out.dir} kept`,
+      abend: { code: run.abend.code, ...(isString(run.abend.message) ? { message: run.abend.message } : {}) },
+      input: used, run: { dir: out.dir, journal: run.journal },
+    };
+    seen.set(key, { size, finding });
+  }
+  out.findings = [...seen.values()].map((s) => s.finding);
+  return out;
+}
+
+export function scanAbend(root, opts = {}) {
+  const dirs = abendRunPaths(opts);
+  const runs = dirs.map((d) => loadAbendRun(d));
+  const findings = runs.flatMap((r) => r.findings.filter((f) => existsSync(join(root, f.path))));
+  const stats = {
+    filesScanned: 0, filesUnreadable: 0,
+    abendRuns: runs.map((r) => ({ dir: r.dir, program: r.program ? r.program.file : null, findings: r.findings.length, ...(r.notModelled ? { notModelled: r.notModelled } : {}), ...(r.counts ? { counts: r.counts } : {}) })),
+    ...(runs.some((r) => r.problems.length) ? { abendRunProblems: runs.flatMap((r) => r.problems.map((p) => `${r.dir}: ${p}`)) } : {}),
+    ...(runs.some((r) => r.findings.some((f) => !existsSync(join(root, f.path)))) ? { abendRunsElsewhere: runs.flatMap((r) => r.findings.filter((f) => !existsSync(join(root, f.path))).map((f) => `${r.dir}: ${f.path}`)) } : {}),
+    setIncomplete: runs.some((r) => r.problems.length > 0),
+  };
+  return report('abend', { rules: ABEND_RULES, findings, stats, run: null });
+}
+

package/lib/sets/compile.mjs

@@ -93,7 +93,8 @@ function declaredIn(text, names, into) {
 // A reference cut at column 72 names a word the source does not hold.
 function truncated(ref, lines) {
   const line = (lines.get(ref.file) || [])[ref.line - 1] || '';
-  const re = new RegExp(`(?<![A-Za-z0-9-])${ref.name.replace(/[$#@]/g, '\\$&')}`, 'gi');
+  // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- the name is escaped
+  const re = new RegExp(`(?<![A-Za-z0-9-])${ref.name.replace(/[.*+?^${}()|[\]\\#@]/g, '\\$&')}`, 'gi');
   let whole = false;
   let any = false;
   let m;

package/lib/sets/hidden.mjs

@@ -127,7 +127,7 @@ const SEQUENCE_NUMBER = /^[A-Za-z]{0,4}\d{2,6}$/;
 const FIXED_INDICATOR = /^[ */\-Dd$]$/;
 const NOT_CODE = /^(\*>|\/\/|\/\*)/;
 // A directive that makes the rest of the file free format, where columns 1-6 are ordinary text.
-const TO_FREE = />>\s*SOURCE\s+(?:FORMAT\s+)?(?:IS\s+)?FREE\b|SOURCEFORMAT\s*\(?\s*["']?FREE/i;
+const TO_FREE = />>\s*SOURCE\s+(?:FORMAT\s+)?(?:IS\s+)?FREE\b|SOURCEFORMAT\s*(?:\(\s*)?["']?FREE/i;
 
 function sequencePayload(lines, format) {
   const rows = [];

package/lib/sets/jcl.mjs

@@ -154,6 +154,7 @@ const DESTRUCTIVE = /^\s*(DELETE(?!\s+FROM\b)|ALTER\s+\S+\s+NEWNAME|REPRO\b[^\n]
 // symbol ends it. An empty qualifier is a symbol the reader already filled with an empty default.
 const dsnKey = (dsn) => String(dsn || '').replace(/^['(]+|[')]+$/g, '').replace(/\(.*$/, '').toUpperCase()
   .replace(/&[A-Z0-9@#$]+\.?/g, '*').replace(/<[^>]*>/g, '*').replace(/\.(?=\.)/g, '.*');
+// nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- k is escaped, and * widened to one qualifier
 const dsnPattern = (k) => new RegExp(`^${k.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^.]+')}$`);
 
 // The programs whose in-stream data can delete a catalogued dataset. A step running another known

package/lib/sets/recon.mjs

@@ -107,6 +107,7 @@ function routableAddresses(text, jcl) {
 // A qualifier matches on a whole dotted component, never on a substring: PROD matches PROD.MASTER
 // and PRODLIB.X only if PRODLIB was itself listed. Matching on substrings is how a rule like this
 // starts reporting every line in the estate.
+// nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- q is escaped
 const qualifierPattern = (q) => new RegExp('(^|[^A-Z0-9$#@.])' + q.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?=[.\\s,)\'"]|$)', 'i');
 
 // JOBLIB and STEPLIB are where the system looks for the program, so what they hold is code rather

package/lib/utilities.mjs

@@ -118,6 +118,7 @@ function commands(lines) {
 // The values of the first of `names` given as a keyword in a command, as in INFILE(IN) or
 // OUTDDNAME(TAPE1,TAPE2). A name only counts at the start of a word, so INDD is not LOGINDD.
 function param(text, names) {
+  // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- names are literal keyword lists
   const m = new RegExp(`(?:^|[\\s,])(?:${names.join('|')})\\s*\\(([^)]*)\\)`, 'i').exec(text);
   return m ? m[1].split(/[\s,]+/).filter(Boolean).map((v) => upper(v.replace(/^'|'$/g, ''))) : null;
 }
@@ -341,6 +342,7 @@ function icetoolCopies(step, dds) {
   for (const st of statements) {
     if (!['COPY', 'SORT', 'MERGE'].includes(st.op)) continue;
     const named = (kw) => st.operands.flatMap((o) => {
+      // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- kw is a literal keyword
       const m = new RegExp(`^${kw}\\((.*)\\)$`, 'i').exec(o);
       return m ? m[1].split(',').map((n) => upper(n.trim())).filter(Boolean) : [];
     });

package/lib/verify.mjs

@@ -18,7 +18,8 @@ const END_TASK = 'end the task from CEDF or CEDX, or stop the program in the deb
 // What goes in, and what reading it at the operation shows. `run` says what letting the operation run
 // shows; null means it would act on something, so the plan stops before it (`stopBefore`).
 const SINKS = {
-  arithmetic: (x) => ({ value: `a letter where ${x.item} expects a digit, such as A in its first position`, reads: `${x.item} holding the letter`, run: 'the task abends ASRA, a data exception (S0C7 in batch), at the operation' }),
+  // A letter's low half is 1 to 9 and its zone a valid sign, so zoned arithmetic reads it as a digit.
+  arithmetic: (x) => ({ value: `an asterisk in every position of ${x.item}, whose digits it expects: a letter will not do, since zoned decimal reads a letter as a digit`, reads: `${x.item} holding the asterisks`, run: 'the task abends ASRA, a data exception (S0C7 in batch), at the operation' }),
   subscript: (x) => bounds(x, `${x.table ? `${x.table + 1}, one more than the ${x.table} entries in the table` : 'one more than the entries in the table'}, or 0`),
   'loop-bound': (x) => bounds(x, `${x.table ? `${x.table + 1}, one more than the ${x.table} entries in the table` : 'one more than the entries in the table'}`),
   'reference-modification': (x) => bounds(x, 'one more than the length of the field, or 0'),