@portll/cobolwork 0.2.150 → 0.4.0

package/lib/exploitability.mjs

@@ -6,12 +6,13 @@
 // running system confirms one, so a verdict says confirmed only where the estate brings the result
 // of its own test as a witness feed. See docs/spec/reach.md §9.
 import { readFileSync, realpathSync } from 'node:fs';
-import { basename, sep, delimiter } from 'node:path';
+import { basename, dirname, join, resolve, sep, delimiter } from 'node:path';
 import { ALL_RULES } from './kernel/registry.mjs';
 import { EXPLOITABILITY } from './kernel/findings.mjs';
 import { kindsOf } from './consequence.mjs';
 import { WHO } from './sets/flow.mjs';
 import { entryName, accessOfEntry, accessFromEntry, privilegedFromEntry } from './reach.mjs';
+import { verifyEvidence } from './evidence/verify.mjs';
 
 export const HANDLING = 'This report joins routes an attacker can drive with the estate\'s own access facts. It holds no source '
   + 'text, but it is a list of what to attack first: keep it where the access records it was built from may go, and do not '
@@ -62,9 +63,22 @@ const WITHOUT_A_TEST = {
 // response code is not input, so no test on it clears a route; the fix is not to send it.
 function fixAt(f, kinds) {
   const last = f.trace?.length ? f.trace[f.trace.length - 1] : null;
+  const instead = WITHOUT_A_TEST[kinds.sink];
+  // A value a STRING built holds the input as one part among literals, where no allow-list or class
+  // test describes the whole. The test belongs on the field folded in, before the STRING.
+  const built = f.trace ? f.trace.findLastIndex((h) => /^STRING at /.test(h.via || '')) : -1;
+  const part = built > 0 ? f.trace[built - 1] : null;
+  const where = part?.item && /\bat (.+):(\d+)$/.exec(f.trace[built].via);
+  if (where && kinds.source !== 'system-response' && TESTS[kinds.sink]) {
+    const [, path, line] = where;
+    return {
+      program: f.trace[built].program || f.program || null, path, line: Number(line), item: part.item,
+      builtInto: { item: last.item, path: f.path, line: f.line },
+      test: `before the STRING at ${path}:${line} that builds ${last.item}, on every route to it, test ${TESTS[kinds.sink](part.item)}, and let only a value that passes reach it${instead ? `; ${instead} needs no test` : ''}`,
+    };
+  }
   const item = last?.item || 'the value';
   const test = kinds.source === 'system-response' ? null : TESTS[kinds.sink]?.(item) || null;
-  const instead = WITHOUT_A_TEST[kinds.sink];
   return {
     program: f.program || null, path: f.path, line: f.line, item,
     ...(test ? { test: `before the operation at ${f.path}:${f.line}, on every route to it, test ${test}, and let only a value that passes reach it${instead ? `; ${instead} needs no test` : ''}` }
@@ -156,6 +170,28 @@ const OUTCOMES = new Set(['reproduced', 'not-reproduced']);
 const str = (v) => typeof v === 'string' && v.length > 0;
 const isoDate = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(v);
 
+// What an ironwork run a result cites shows: each operation the marker reached and the abend the run
+// ended with, from a journal in an evidence directory that verifies. A string says why it shows nothing.
+function witnessedRun(ev, base, verdicts) {
+  if (!ev || typeof ev !== 'object' || !str(ev.dir) || !str(ev.run)) return 'cites evidence without a directory and a run';
+  if (!/^[0-9TZ]+-[0-9a-f]{16}$/.test(ev.run)) return `cites run ${ev.run}, which is not a run id`;
+  const dir = resolve(base, ev.dir);
+  if (!verdicts.has(dir)) verdicts.set(dir, verifyEvidence(dir));
+  const v = verdicts.get(dir);
+  if (!v.verified) return `cites evidence in ${ev.dir} that does not verify`;
+  if (v.unrecorded.includes(ev.run)) return `cites run ${ev.run}, which the ledger in ${ev.dir} does not record`;
+  let text;
+  try { text = readFileSync(join(dir, 'runs', `${ev.run}.jsonl`), 'utf8'); } catch { return `cites run ${ev.run}, which ${ev.dir} does not hold`; }
+  const records = [];
+  for (const line of text.split('\n')) {
+    if (!line) continue;
+    const rec = JSON.parse(line);
+    if (rec.kind === 'sink' && rec.reached === true) records.push({ kind: 'sink', sink: rec.sink, file: rec.file, line: rec.line });
+    else if (rec.kind === 'abend' && str(rec.file) && Number.isInteger(rec.line)) records.push({ kind: 'abend', code: rec.code, file: rec.file, line: rec.line });
+  }
+  return records.length ? { run: ev.run, records } : `cites run ${ev.run}, whose journal shows no operation the marker reached and no abend at a line`;
+}
+
 // The results of the estate's own tests, keyed by the fingerprint of the finding each tried, loaded
 // for one scan and refused from inside the scanned tree the way a reach feed is: a list of what was
 // reproduced is a list of what to attack. Shape:
@@ -176,6 +212,7 @@ export function loadWitnessFeed(path, { root = null } = {}) {
   if (!doc.results || typeof doc.results !== 'object' || Array.isArray(doc.results)) { feed.problem = 'holds no results'; return feed; }
   feed.witness = doc.witness;
   feed.recorded = doc.recorded;
+  const verdicts = new Map();
   for (const [fingerprint, r] of Object.entries(doc.results)) {
     const why = !r || typeof r !== 'object' ? 'is not an object'
       : !OUTCOMES.has(r.outcome) ? `says outcome ${JSON.stringify(r.outcome)}, not "reproduced" or "not-reproduced"`
@@ -183,7 +220,9 @@ export function loadWitnessFeed(path, { root = null } = {}) {
           : !isoDate(r.on) ? 'does not say when the test ran'
             : !str(r.system) ? 'does not say which system it ran on' : null;
     if (why) { feed.refused.push({ fingerprint, why: `the result for ${fingerprint} ${why}` }); continue; }
-    feed.results.set(fingerprint, { outcome: r.outcome, by: r.by, on: r.on, system: r.system, ...(str(r.reference) ? { reference: r.reference } : {}) });
+    const evidence = r.evidence === undefined ? null : witnessedRun(r.evidence, dirname(real), verdicts);
+    if (typeof evidence === 'string') { feed.refused.push({ fingerprint, why: `the result for ${fingerprint} ${evidence}` }); continue; }
+    feed.results.set(fingerprint, { outcome: r.outcome, by: r.by, on: r.on, system: r.system, ...(str(r.reference) ? { reference: r.reference } : {}), ...(evidence ? { evidence } : {}) });
   }
   return feed;
 }
@@ -214,8 +253,11 @@ export function applyWitness(findings, checked, feeds) {
     const x = f.exploitability;
     const hit = x && f.fingerprint ? newest(good, f.fingerprint) : null;
     if (!hit) return false;
-    matched.add(f.fingerprint);
     const { r, feed } = hit;
+    // A cited run witnesses this finding only where its journal places the operation or the abend at the finding's line.
+    const at = r.evidence ? r.evidence.records.find((e) => e.line === f.line && basename(e.file) === basename(f.path)) : null;
+    if (r.evidence && !at) return false;
+    matched.add(f.fingerprint);
     const cite = `${feed.file} (${feed.witness}, recorded ${feed.recorded})`;
     x.witness = { ...r, from: cite };
     byWitness[r.outcome]++;
@@ -227,6 +269,9 @@ export function applyWitness(findings, checked, feeds) {
     const was = x.verdict;
     x.verdict = 'confirmed';
     x.because.push(`reproduced on ${r.system} by ${r.by} on ${r.on}${r.reference ? ` (${r.reference})` : ''}, per ${cite}`);
+    if (at) x.because.push(at.kind === 'sink'
+      ? `run ${r.evidence.run}'s verified journal records the marker reaching the ${at.sink} operation at ${at.file}:${at.line}`
+      : `run ${r.evidence.run}'s verified journal records the run ending with ${at.code} at ${at.file}:${at.line}`);
     delete x.unknown;
     if (!x.fixAt) x.fixAt = fixAt(f, kindsOf(f.rule));
     if (f.guardedFrom) f.sev = f.guardedFrom;

package/lib/gate.mjs

@@ -20,7 +20,9 @@ const MAX_EDIT = 2000;
 const MAX_LINES = 200000;
 const MAX_COMPILES = 50;
 const COMPILE_BUDGET_MS = 10 * 60 * 1000;
-const PASSING = new Set(['cleared-by-check', 'statement-removed', 'source-removed', 'no-longer-fires']);
+// A fix keeps the program's statements and stops the route: deleting the flagged statement or the
+// source statement removes what the program did, and fails.
+const PASSING = new Set(['cleared-by-check', 'no-longer-fires']);
 const UNDECIDED = new Set(['lowered-by-check', 'gone-unexplained']);
 
 const pathText = (p) => printable(String(p ?? ''), 120);
@@ -115,15 +117,18 @@ export function findCobc({ explicit = null, repo, env = process.env, platform =
   return found.path ? found : { path: null, why: `--cobc ${pathText(resolve(explicit))} is not a file outside the repository` };
 }
 
-// A compiler run as `cobc -fsyntax-only`, from outside the tree, for a bounded time.
+// A compiler run as `cobc -fsyntax-only`, from outside the tree, for a bounded time. Its error lines
+// are kept so a drafter whose patch stopped a program compiling is told where.
 export function cobcCompiler(cobc) {
   return (file, { includeDirs = [], free = false } = {}) => {
     const r = spawnSync(cobc, ['-fsyntax-only', ...(free ? ['-free'] : []), ...includeDirs.flatMap((d) => ['-I', d]), file],
-      { cwd: tmpdir(), timeout: 60000, windowsHide: true, stdio: 'ignore' });
-    return r.status === 0;
+      { cwd: tmpdir(), timeout: 60000, windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'], encoding: 'latin1', maxBuffer: 1 << 20 });
+    return { ok: r.status === 0, messages: String(r.stderr || '').split(/\r?\n/).filter((l) => /: error: /.test(l)) };
   };
 }
 
+const MAX_COMPILER_LINES = 3;
+
 // Level 1 matches across both lists first; each looser level pairs only what the level above left.
 export function pairLists(base, head, baseKeys, headKeys) {
   const headOf = new Array(base.length).fill(-1);
@@ -222,6 +227,10 @@ function targetReason(t, r) {
       return `the target is still reported, lowered from ${t.sev} to ${r.head.sev} by a check this patch added at ${where({ path: r.guard.file, line: r.guard.line })}; a check that lowers rather than stops may not turn away everything it should, so a person decides`;
     case 'program-removed':
       return `the patch removes ${pathText(t.path)}${t.program ? ` or its program ${printable(t.program, 40)}` : ''}, which held the target; removing the program is not a fix the gate accepts`;
+    case 'statement-removed':
+      return `the patch deletes the target's statement at ${where(t)}; a fix keeps the statement and stops the route to it, so deleting it is not a fix the gate accepts`;
+    case 'source-removed':
+      return `the patch deletes the statement the target's input comes from; a fix keeps it and stops the route from it, so deleting it is not a fix the gate accepts`;
     case 'gone-unexplained':
       return `the target, ${at}, is gone, but the patch neither removed its statement or its source nor added a check that stops its route; the route may be cut between its ends, or sent through something the engine does not follow, and a person decides`;
     default:
@@ -246,16 +255,29 @@ function compileCheck({ compiler, baseRoot, headRoot, baseProgs, headProgs, chan
     try { free = detectFormat(readFileSync(join(root, file), 'latin1')) === 'free'; } catch { /* unreadable: compiled as fixed */ }
     return { includeDirs: [...dirs].sort(), free };
   };
+  // A compiler answers true or false, or { ok, messages }.
+  const compile = (root, progs, file) => {
+    const r = compiler.run(join(root, file), optsFor(root, progs, file));
+    return r && typeof r === 'object' ? { ok: !!r.ok, messages: r.messages || [] } : { ok: !!r, messages: [] };
+  };
   const regressed = [], neverCompiled = [];
   const deadline = Date.now() + (compiler.budgetMs ?? COMPILE_BUDGET_MS);
   for (const file of [...files].sort()) {
     if (Date.now() > deadline) return { ok: null, compiled: `not compiled: the compiler ran past the gate's ${Math.round((compiler.budgetMs ?? COMPILE_BUDGET_MS) / 60000)}-minute budget`, reasons: [] };
-    const now = compiler.run(join(headRoot, file), optsFor(headRoot, headProgs, file));
-    if (now) continue;
-    const had = existsSync(join(baseRoot, file)) && compiler.run(join(baseRoot, file), optsFor(baseRoot, baseProgs, file));
-    (had ? regressed : neverCompiled).push(file);
+    const now = compile(headRoot, headProgs, file);
+    if (now.ok) continue;
+    const had = existsSync(join(baseRoot, file)) && compile(baseRoot, baseProgs, file).ok;
+    if (had) regressed.push({ file, messages: now.messages });
+    else neverCompiled.push(file);
+  }
+  // The compiler names the file by where the head was written; the reader knows it by its path.
+  const asInRepo = (m) => m.split(join(headRoot, sep)).join('');
+  if (regressed.length) {
+    return { ok: false, compiled: `${regressed.length} program(s) compiled before the patch and do not after`, reasons: regressed.flatMap(({ file, messages }) => [
+      `${pathText(file)} compiled before the patch and does not after`,
+      ...messages.slice(0, MAX_COMPILER_LINES).map((m) => `the compiler: ${asInRepo(m).trim()}`),
+    ]) };
   }
-  if (regressed.length) return { ok: false, compiled: `${regressed.length} program(s) compiled before the patch and do not after`, reasons: regressed.map((f) => `${pathText(f)} compiled before the patch and does not after`) };
   if (neverCompiled.length) return { ok: null, compiled: `${neverCompiled.length} program(s) do not compile before or after the patch, or are new and do not compile, which the gate cannot tell from a missing copy library`, reasons: [] };
   return { ok: true, compiled: `${files.size} program(s) compile`, reasons: [] };
 }

package/lib/kernel/findings.mjs

@@ -38,6 +38,7 @@ export const EVIDENCE = Object.freeze({
   advisory: 'a component version matches a published advisory against it',
   exposure: 'information about the estate is written into the source',
   change: 'a change moves an interface or adds a call target, which a reviewer has to look at',
+  execution: 'a run of the program on a recorded input ended this way, and the run\'s verified journal is the record of it',
   coverage: 'the analysis stopped following here; a limit of the tool, not a defect in the code',
   context: 'describes the estate - an entry point, a product in use - and asserts no defect',
 });
@@ -60,6 +61,7 @@ export const WHO_ACTS = Object.freeze({
   advisory: 'whoever owns the build',
   exposure: 'the owner',
   change: 'the reviewer of that change',
+  execution: "the program's owner",
   coverage: "nobody's code: read more, or accept the limit",
   context: 'nobody: it asserts no defect',
 });

package/lib/kernel/registry.mjs

@@ -27,6 +27,7 @@ import { scanLog, LOG_RULES } from '../sets/log.mjs';
 import { scanCompile, COMPILE_RULES } from '../sets/compile.mjs';
 import { scanSemantics, SEMANTICS_RULES } from '../sets/semantics.mjs';
 import { scanZowe, ZOWE_RULES } from '../sets/zowe.mjs';
+import { scanAbend, ABEND_RULES } from '../sets/abend.mjs';
 import { toolName } from './ruleset.mjs';
 
 // The order is the order a scan runs them and the order --only lists them. It is not significant
@@ -47,6 +48,7 @@ export const REGISTRY = [
   { name: 'log', scan: scanLog, rules: LOG_RULES },
   { name: 'semantics', scan: scanSemantics, rules: SEMANTICS_RULES },
   { name: 'zowe', scan: scanZowe, rules: ZOWE_RULES },
+  { name: 'abend', scan: scanAbend, rules: ABEND_RULES },
 ];
 
 // Re-exported from the kernel's ruleset module, which is where it has to live: this file imports

package/lib/kernel/source-tree.mjs

@@ -154,6 +154,7 @@ function heldTree({ kind, root, store, rel, systemDirs = [], copyDirs = null, cl
 // Buffer. Paths are used exactly as given, so a test reads the way it writes.
 export function memoryTree(files, { root = '/memory', systemDirs = [] } = {}) {
   const store = new Map(Object.entries(files).map(([p, v]) => [p, Buffer.isBuffer(v) ? v : Buffer.from(v, 'latin1')]));
+  // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- root is escaped
   const rel = (p) => String(p).replace(/\\/g, '/').replace(new RegExp('^' + root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '/?'), '');
   return heldTree({ kind: 'memory', root, store, rel, systemDirs });
 }

package/lib/options.mjs

@@ -433,7 +433,11 @@ export function compilerTasks(text) {
   };
   const valueOf = (v) => (v && typeof v === 'object' && !Array.isArray(v) ? v.value : v);
   const folder = (v) => String(v).replace(/\$\{(?:workspaceFolder|workspaceRoot)(?::[^}]*)?\}/g, '.').replace(/\$\{(?:\/|pathSeparator)\}/g, '/');
-  const quoted = (a) => { const s = String(valueOf(a) ?? ''); return /[\s"]/.test(s) || s === '' ? `"${s.replace(/"/g, '\\"')}"` : s; };
+  const quoted = (a, dialect) => {
+    const s = String(valueOf(a) ?? '');
+    const esc = ESCAPE[dialect];
+    return /[\s"]/.test(s) || s === '' ? `"${s.replace(/./gs, (c) => (c === '"' || c === esc ? esc + c : c))}"` : s;
+  };
   const out = [];
   const tasks = [doc, ...(Array.isArray(doc && doc.tasks) ? doc.tasks : [])];
   for (const task of tasks) {
@@ -448,7 +452,7 @@ export function compilerTasks(text) {
       const line = lineOf(JSON.stringify(command).slice(1, 41), task.label && JSON.stringify(task.label));
       const state = { dir };
       const found = [];
-      commandLine(folder([command, ...args.map(quoted)].join(' ')), dialect, state, line, found);
+      commandLine(folder([command, ...args.map((a) => quoted(a, dialect))].join(' ')), dialect, state, line, found);
       out.push(...found);
     }
   }

package/lib/packs.mjs

@@ -113,8 +113,10 @@ export function packProblems(pack) {
     const scopes = r.appliesTo || ['jcl-instream'];
     for (const sc of scopes) if (!SCOPES.includes(sc)) problems.push(`${pack.name}:${r.id}: unknown scope '${sc}'`);
     if (!scopes.length) problems.push(`${pack.name}:${r.id}: no scope, so it can never match anything`);
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- a shipped pack's pattern, checked here to compile
     try { new RegExp(r.pattern, r.flags || 'i'); } catch (e) { problems.push(`${pack.name}:${r.id}: ${e.message}`); }
     if (r.setsContext && !r.setsContextPattern) problems.push(`${pack.name}:${r.id}: setsContext without a pattern that sets it`);
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- a shipped pack's pattern, checked here to compile
     if (r.setsContextPattern) { try { new RegExp(r.setsContextPattern, 'i'); } catch (e) { problems.push(`${pack.name}:${r.id}: setsContextPattern ${e.message}`); } }
     // A rule that waits for a context nothing in the pack sets can never fire, which is the same
     // failure as an unknown scope and is caught the same way.
@@ -139,10 +141,12 @@ export function compilePack(pack) {
     pack: pack.name,
     vendor: pack.vendor,
     product: pack.product,
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- a shipped pack's pattern; test/hostile times each one
     re: new RegExp(r.pattern, r.flags || 'i'),
     appliesTo: r.appliesTo || ['jcl-instream'],
     // A rule may declare that it only means what it says once an earlier line has put the stream
     // into a mode, and a rule may be the thing that sets that mode.
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- a shipped pack's pattern; test/hostile times each one
     setsContextRe: r.setsContext && r.setsContextPattern ? new RegExp(r.setsContextPattern, 'i') : null,
   }));
 }

package/lib/parser.mjs

@@ -8,7 +8,8 @@ import {
   EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS,
 } from './words.mjs';
 import { CICS_COMMANDS, CICS_EVERY_COMMAND } from './cics-commands.mjs';
-import { cicsCommand } from './precompile-cics.mjs';
+import { cicsCommand, symbolicMapCopybook } from './precompile-cics.mjs';
+import { parseBms } from './bms.mjs';
 import { BINARY_SIZE, literalBytes, computeSizes, layoutReport } from './layout.mjs';
 import { hostVariablesIn } from './embedded-sql.mjs';
 import { join, dirname, basename, resolve, isAbsolute, sep, delimiter } from 'node:path';
@@ -88,7 +89,7 @@ export function detectFormat(src) {
 function codePastColumn72(src) {
   const lines = src.split(/\r?\n/, 2001);
   for (let i = 0; i < Math.min(lines.length, 2000); i++) {
-    const l = expandTabs(lines[i]).replace(/\s+$/, '');
+    const l = expandTabs(lines[i]).trimEnd();
     if (l.length <= 72 || l[6] === '*' || l[6] === '/' || /^\s*\*>/.test(l) || COMMENT_ENTRY.test(l.slice(7))) continue;
     const { quote, comment } = stateAtColumn72(l);
     if (comment) continue;
@@ -138,9 +139,42 @@ function scanQuotes(text, open) {
 
 const PREDEFINED = new Map([['P64', 'SET']]);
 
+// Cuts a floating comment where /\*>.*$/ would: at the first *> after the last line break.
+function withoutInlineComment(s) {
+  let from = 0;
+  for (const b of ['\n', '\r', '\u2028', '\u2029']) from = Math.max(from, s.lastIndexOf(b) + 1);
+  const at = s.indexOf('*>', from);
+  return at < 0 ? s : s.slice(0, at);
+}
+
+// The name, value and OVERRIDE of a >>DEFINE body as /^(?:CONSTANT\s+)?([A-Za-z0-9_-]+)\s+(?:AS\s+)?(.*?)\s*(OVERRIDE)?\s*$/i
+// reads them, in time linear in the body.
+function defineParts(body) {
+  const space = (c) => c !== undefined && /\s/.test(c);
+  const skip = (i) => { while (space(body[i])) i++; return i; };
+  const from = (start) => {
+    let j = start;
+    while (j < body.length && /[A-Za-z0-9_-]/.test(body[j])) j++;
+    if (j === start || !space(body[j])) return null;
+    let v = skip(j);
+    if (/^AS$/i.test(body.slice(v, v + 2)) && space(body[v + 2])) v = skip(v + 2);
+    let end = body.length;
+    while (end > v && space(body[end - 1])) end--;
+    let override;
+    if (end - 8 >= v && /^OVERRIDE$/i.test(body.slice(end - 8, end))) {
+      override = body.slice(end - 8, end);
+      end -= 8;
+      while (end > v && space(body[end - 1])) end--;
+    }
+    const value = body.slice(v, end);
+    return /[\n\r\u2028\u2029]/.test(value) ? null : [body, body.slice(start, j), value, override];
+  };
+  return (/^CONSTANT$/i.test(body.slice(0, 8)) && space(body[8]) && from(skip(8))) || from(0);
+}
+
 function defineDirective(directive, defines) {
-  const body = directive.replace(/\*>.*$/, '').replace(/^>>\s*DEFINE\s+/i, '').trim();
-  const m = body.match(/^(?:CONSTANT\s+)?([A-Za-z0-9_-]+)\s+(?:AS\s+)?(.*?)\s*(OVERRIDE)?\s*$/i);
+  const body = withoutInlineComment(directive).replace(/^>>\s*DEFINE\s+/i, '').trim();
+  const m = defineParts(body);
   if (!m) return;
   const name = m[1].toUpperCase();
   const raw = m[2].trim();
@@ -153,17 +187,17 @@ function defineDirective(directive, defines) {
 function setDirective(text, defines) {
   const c = /\bCONSTANT\s+([A-Za-z0-9_-]+)\s+(?:(["'])(.*?)\2|(\S+))/i.exec(text);
   if (c && defines) defines.set(c[1].toUpperCase(), c[3] !== undefined ? c[3] : c[4]);
-  const m = text.match(/SOURCEFORMAT\s*\(?\s*["']?(FREE|FIXED|VARIABLE)/i);
+  const m = text.match(/SOURCEFORMAT\s*(?:\(\s*)?["']?(FREE|FIXED|VARIABLE)/i);
   return m ? m[1].toLowerCase() : null;
 }
 
 function evaluateCondition(text, defines) {
-  const t = text.replace(/\*>.*$/, '').trim();
+  const t = withoutInlineComment(text).trim();
   const known = (n) => (defines && defines.has(n)) || PREDEFINED.has(n);
   const valueOf = (n) => (defines && defines.has(n) ? defines.get(n) : PREDEFINED.get(n));
   let m = t.match(/^([A-Za-z0-9_-]+)\s+(?:IS\s+)?(NOT\s+)?(DEFINED|SET)$/i);
   if (m) { const r = known(m[1].toUpperCase()); return m[2] ? !r : r; }
-  m = t.match(/^([A-Za-z0-9_-]+)\s*(<=|>=|<>|=|<|>)\s*(['"]?)([^'"]*)\3$/);
+  m = t.match(/^([A-Za-z0-9_-]+)\s*(<=|>=|<>|=|<|>)\s*(?!\s)(['"]?)([^'"]*)\3$/);
   if (m) {
     const name = m[1].toUpperCase();
     if (!known(name)) return null;
@@ -446,11 +480,18 @@ export function tokenize(norm, file) {
   return { tokens: out, diags };
 }
 
-// Every rule set walks the tree through here. A directory it may not read is recorded, not treated
-// as empty: only ENOENT means absent. Symlinks are followed while they stay inside the tree, so a
-// symlinked copy library is read; one pointing outside is counted and never followed, because a
-// scan reads the tree it was given and nothing else.
-export function buildFileIndex(root) {
+// A drive that stalls or drops for a moment answers a listing with ENOENT or an I/O error. Such a
+// listing is tried again after a pause; the pauses add up to about 1.3 seconds.
+const LISTING_RETRIED = new Set(['ENOENT', 'EIO', 'ETIMEDOUT', 'ENXIO', 'EBUSY', 'EAGAIN', 'EINTR', 'ESTALE', 'ENOTCONN']);
+const LISTING_PAUSES_MS = [50, 250, 1000];
+const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+
+// Every rule set walks the tree through here. A directory that cannot be listed is recorded, not
+// treated as empty: a scan over it is incomplete, not a scan of a smaller tree. Only the root's own
+// ENOENT means absent; a directory the walk found in its parent was there. Symlinks are followed
+// while they stay inside the tree, so a symlinked copy library is read; one pointing outside is
+// counted and never followed, because a scan reads the tree it was given and nothing else.
+export function buildFileIndex(root, { readdir = readdirSync, wait = pause } = {}) {
   const index = new Map();
   const dirs = new Set();
   const unreadableDirs = [];
@@ -460,16 +501,20 @@ export function buildFileIndex(root) {
   const inside = (p) => p === top || p.startsWith(top + sep);
   const visited = new Set();
   const addFile = (p, name, d) => { index.set(p.toLowerCase(), p); if (/\.(cpy|copy|inc|cbl|cob)$/i.test(name)) dirs.add(d); };
+  const list = (d) => {
+    for (let i = 0; ; i++) {
+      try { return readdir(d, { withFileTypes: true }); } catch (e) {
+        if (i >= LISTING_PAUSES_MS.length || !LISTING_RETRIED.has(e.code)) throw e;
+        wait(LISTING_PAUSES_MS[i]);
+      }
+    }
+  };
   // Each real directory is walked once, however many links lead to it, or its programs count twice.
   const walk = (d, real) => {
     if (visited.has(real)) return;
     visited.add(real);
     let es;
-    try { es = readdirSync(d, { withFileTypes: true }); } catch (e) {
-      if (e.code === 'ENOENT') return;
-      if (e.code === 'EACCES' || e.code === 'EPERM') { unreadableDirs.push(d); return; }
-      throw e;
-    }
+    try { es = list(d); } catch { unreadableDirs.push(d); return; }
     for (const e of es.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
       if (e.name === '.git' || e.name.startsWith('._')) continue;
       const p = join(d, e.name);
@@ -572,7 +617,8 @@ function applyReplacing(tokens, pairs, maxGrowth = Infinity, byCopy = false) {
         if (p.mode || p.partial || p.from.toks.length !== 1 || p.to.toks.length > 1) continue;
         const pat = p.from.toks[0].v.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
         const rep = p.to.toks[0] ? p.to.toks[0].v : '';
-        const v = tk.v.replace(new RegExp(`\\(${pat}\\)`, 'gi'), `(${rep})`);
+        // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- pat is escaped
+        const v = tk.v.replace(new RegExp(`\\(${pat}\\)`, 'gi'), () => `(${rep})`);
         if (v !== tk.v) tk = { ...tk, v, u: v };
       }
     }
@@ -779,15 +825,31 @@ function expand(tokens, ctx, stack, format) {
   return out;
 }
 
+// The symbolic map BMS would generate for a COBOL mapset the tree holds only as BMS source, each
+// line placed at the BMS statement it comes from.
+function symbolicMapFor(name, ctx) {
+  const want = name.toUpperCase();
+  for (const p of filesByName(ctx.fileIndex).get(`${name}.bms`.toLowerCase()) || []) {
+    let mapsets;
+    try { ({ mapsets } = parseBms(ctx.readText(p))); } catch { continue; }
+    const mapset = mapsets.find((m) => m.name?.toUpperCase() === want && String(m.options?.lang || 'COBOL').toUpperCase() === 'COBOL');
+    const copybook = mapset && symbolicMapCopybook(mapset);
+    if (copybook) return { path: p, ...copybook };
+  }
+  return null;
+}
+
 function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
-  const path = resolveCopy(name, lib, ctx);
   const refused = copyRefusal(name, ctx);
-  const record = { name, lib, via, file: at.file, line: at.line, status: path ? 'resolved' : refused || (SYSTEM_COPY.test(name) ? 'system' : 'missing'), path };
+  let path = resolveCopy(name, lib, ctx);
+  const generated = !path && !refused && !lib && via === 'copy' && ctx.fileIndex ? symbolicMapFor(name, ctx) : null;
+  if (generated) path = generated.path;
+  const record = { name, lib, via, file: at.file, line: at.line, status: path ? 'resolved' : refused || (SYSTEM_COPY.test(name) ? 'system' : 'missing'), path, ...(generated ? { generatedFrom: 'bms' } : {}) };
   ctx.copies.push(record);
   if (!path) return [];
   if (stack.includes(path) || stack.length > 40) { record.status = 'recursive'; return []; }
   if (++ctx.inclusions > MAX_INCLUSIONS || ctx.copyTokens > MAX_COPY_TOKENS) { record.status = 'expansion-limit'; return []; }
-  const src = ctx.readText(path);
+  const src = generated ? generated.text : ctx.readText(path);
   // A copybook is read in the format in force where the COPY statement sits, which a >>SOURCE
   // directive earlier in the including file may have changed from the file's starting format.
   const fmt = detectFormat(src) === 'terminal' && ctx.copyFormat === 'auto' ? 'terminal' : (at.fmt || (ctx.copyFormat !== 'auto' ? ctx.copyFormat : (inheritedFormat || ctx.mainFormat)));
@@ -801,10 +863,13 @@ function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
     const f = p.from.toks;
     if (p.mode || !f.length) continue;
     const pat = f.map(t => t.v).join('');
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- escaped
     if (p.from.pseudo && /^[:(][A-Za-z0-9_-]*[:)]$/.test(pat)) { partial.push([p, new RegExp(escape(pat), 'gi')]); continue; }
     if (p.from.pseudo || f.length !== 1 || f[0].t !== 'lit') continue;
     const lit = `(['"])${escape(f[0].v)}\\1`;
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- lit is escaped
     const touching = new RegExp(`${lit}(?=[A-Za-z0-9-])|(?<=[A-Za-z0-9-])${lit}`);
+    // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- lit is escaped
     if (norm.entries.some(e => touching.test(e.text))) partial.push([p, new RegExp(lit, 'g')]);
   }
   if (partial.length) {
@@ -821,6 +886,7 @@ function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
     for (const [p] of partial) p.partial = true;
   }
   const { tokens, diags } = tokenize(norm, path);
+  if (generated) for (const t of tokens) t.line = generated.lines[t.line - 1] ?? t.line;
   ctx.copyTokens += tokens.length;
   ctx.diags.push(...diags, ...norm.diags.map(d => ({ ...d, file: path })));
   const expanded = expand(tokens, ctx, [...stack, path], norm.finalFormat);
@@ -1513,6 +1579,9 @@ function indexTokens(seg) {
       const lone = (from, to) => (to - from === 1 && (seg[from].t === 'word' || seg[from].t === 'num') && /^\d+$/.test(seg[from].v) ? Number(seg[from].v) : null);
       const refLength = colonAt < 0 ? null : lone(colonAt + 1, close);
       const refStart = colonAt < 0 ? null : lone(k + 1, colonAt);
+      // A length is judged against where its reference starts: a name, a name moved by a constant, or neither.
+      const named = (from, to) => (seg[from].t === 'word' && !/^\d+$/.test(seg[from].v) && (to - from === 1 || (to - from === 3 && offsetOf(seg, from, k, close, colonAt))) ? { tok: seg[from], offset: offsetOf(seg, from, k, close, colonAt) } : null);
+      const refFrom = colonAt < 0 || refStart != null ? null : named(k + 1, colonAt) || { tok: null, offset: 0 };
       for (let j = k + 1; j < close; j++) {
         const t = seg[j];
         // A number is tokenized as a word, and no name is all digits.
@@ -1521,7 +1590,7 @@ function indexTokens(seg) {
         if (prev && prev.t === 'word' && (prev.u === 'OF' || prev.u === 'IN' || prev.u === 'FUNCTION')) continue;
         const kind = colonAt < 0 ? 'subscript' : j < colonAt ? 'refmod-offset' : 'refmod-length';
         const other = kind === 'refmod-offset' ? refLength : kind === 'refmod-length' ? refStart : null;
-        out.push({ host, tok: t, kind, offset: offsetOf(seg, j, k, close, colonAt), ...(other ? { span: other } : {}) });
+        out.push({ host, tok: t, kind, offset: offsetOf(seg, j, k, close, colonAt), ...(other ? { span: other } : {}), ...(kind === 'refmod-length' && refFrom ? { from: refFrom } : {}) });
       }
     }
     lastHost = host;
@@ -1841,7 +1910,7 @@ function collectCopybookDefines(src, format, ctx, depth) {
     const sw = /(?:^|\s)>>\s*SOURCE\s+(?:FORMAT\s+)?(?:IS\s+)?(FREE|FIXED|VARIABLE|TERMINAL)/i.exec(l);
     if (sw) { current = sw[1].toLowerCase(); continue; }
     if (current !== 'free' && current !== 'terminal' && l.length > 6 && '*/'.includes(l[6])) continue;
-    const code = l.replace(/\*>.*$/, '');
+    const code = withoutInlineComment(l);
     const m = /(?:^|[\s.])COPY\s+("[^"]+"|'[^']+'|[A-Za-z0-9_-]+)/i.exec(code);
     if (!m) continue;
     const name = m[1].replace(/^["']|["']$/g, '');

package/lib/precompile-cics.mjs

@@ -122,12 +122,14 @@ export const constantsCopybook = (names) => `${names.map((n) => `       01 ${n}
 // prefix when TIOAPFX=YES, then for each named field its length, flag or attribute byte, extended
 // attribute bytes and data, the output record redefining the input. Grouped (GRPNAME) and OCCURS
 // fields are laid out differently, and a mapset holding either is not written. Returns the copybook
-// text and the names it declares, or null.
+// text, the names it declares and, for each of its lines, the BMS line it comes from; or null.
 const ATTRIBUTE_LETTER = { COLOR: 'C', PS: 'P', HILIGHT: 'H', VALIDN: 'V', OUTLINE: 'U', SOSI: 'M', TRANSP: 'T' };
 export function symbolicMapCopybook(mapset) {
   const out = [];
   const names = [];
-  const line = (s) => out.push(`       ${s}`);
+  const lines = [];
+  let from = mapset.line;
+  const line = (s) => { out.push(`       ${s}`); lines.push(from); };
   const declare = (n, s) => { names.push(n); line(s); };
   // The assembler writes FILLER; each gets a name here, the layout unchanged, so the grade can tell
   // the stand-in's items from the program's.
@@ -138,6 +140,7 @@ export function symbolicMapCopybook(mapset) {
     if (fields.some((f) => f.grpname || f.occurs > 1)) return null;
     const { input, output, attributes } = map.symbolic;
     if (!input && !output) continue;
+    from = map.line;
     const prefix = String(map.operands?.get?.('TIOAPFX') ?? mapset.options.tioapfx ?? '').toUpperCase() === 'YES';
     const pic = (f, which) => f[which] || `X(${f.length})`;
     if (input) {
@@ -145,6 +148,7 @@ export function symbolicMapCopybook(mapset) {
       if (prefix) line(`    02  ${filler(input)} PIC X(12).`);
       for (const f of fields) {
         const n = f.name.toUpperCase();
+        from = f.line;
         declare(`${n}L`, `    02  ${n}L COMP PIC S9(4).`);
         declare(`${n}F`, `    02  ${n}F PIC X.`);
         line(`    02  ${filler(input)} REDEFINES ${n}F.`);
@@ -154,10 +158,12 @@ export function symbolicMapCopybook(mapset) {
       }
     }
     if (output) {
+      from = map.line;
       declare(output, input ? `01  ${output} REDEFINES ${input}.` : `01  ${output}.`);
       if (prefix) line(`    02  ${filler(output)} PIC X(12).`);
       for (const f of fields) {
         const n = f.name.toUpperCase();
+        from = f.line;
         if (input) line(`    02  ${filler(output)} PIC X(3).`);
         else { line(`    02  ${filler(output)} PIC X(2).`); declare(`${n}A`, `    02  ${n}A PIC X.`); }
         for (const a of attributes) declare(`${n}${ATTRIBUTE_LETTER[a]}`, `    02  ${n}${ATTRIBUTE_LETTER[a]} PIC X.`);
@@ -165,5 +171,5 @@ export function symbolicMapCopybook(mapset) {
       }
     }
   }
-  return out.length ? { text: `${out.join('\n')}\n`, names } : null;
+  return out.length ? { text: `${out.join('\n')}\n`, names, lines } : null;
 }

package/lib/precompile.mjs

@@ -188,7 +188,7 @@ function place(lines, block, format, text) {
     while (w < words.length && writeFrom + out.length + (out ? 1 : 0) + words[w].length <= to) out += (out ? ' ' : '') + words[w++];
     const line = lines[li].padEnd(to);
     lines[li] = line.slice(0, blankFrom) + ' '.repeat(Math.max(0, writeFrom - blankFrom)) + out.padEnd(Math.max(0, to - Math.max(writeFrom, blankFrom))) + line.slice(to);
-    if (li !== block.endLine) lines[li] = lines[li].replace(/\s+$/, '');
+    if (li !== block.endLine) lines[li] = lines[li].trimEnd();
   }
   return words.slice(w);
 }
@@ -208,6 +208,7 @@ function spill(words, format) {
 }
 
 // A division or section header, or a word, standing alone: not part of a longer name.
+// nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- callers pass literal patterns
 const word = (re) => new RegExp(`(?<![A-Z0-9-])${re}(?![A-Z0-9-])`, 'i');
 const HEADERS = [['procedure', word('PROCEDURE\\s+DIVISION')], ['linkage', word('LINKAGE\\s+SECTION')], ['data', word('DATA\\s+DIVISION')], ['after', word('(?:REPORT|SCREEN)\\s+SECTION')]];
 const PROGRAM_ID = word('PROGRAM-ID');
@@ -373,7 +374,7 @@ function translateText(t, shared, ctx, before = new Map(), after = new Map()) {
     }
     const rest = place(lines, b, format, text);
     if (periodOnly) {
-      lines[b.endLine] = lines[b.endLine].replace(/\s+$/, '');
+      lines[b.endLine] = lines[b.endLine].trimEnd();
       if (rest.length) { add(after, b.endLine, spill(rest, format)); state.spilled++; }
     } else if (rest.length) {
       // What followed END-EXEC - a period, or the rest of a sentence - follows the whole translation.

package/lib/revision.json

@@ -1 +1 @@
-{"commit":"f5995391b095f62da69d501ea7f6bd11a75bdc6e","tag":"v0.2.150"}
+{"commit":"371041fbffe625df6b0a872249cf071268fcb6a6"}

package/lib/sarif.mjs

@@ -32,8 +32,9 @@ const LEVEL = { crit: 'error', high: 'error', med: 'warning', low: 'note', info:
 // Locations are URI references: each segment is encoded, lone surrogates from Windows names replaced first.
 const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
 const uriOf = (p) => String(p).split('/').map((seg) => encodeURIComponent(seg.replace(LONE_SURROGATE, '\uFFFD'))).join('/');
-// Square brackets in a SARIF message are link syntax, and a finding's text can quote the tree.
-const textOf = (s) => (typeof s === 'string' ? s.replace(/[[\]]/g, '\\$&') : s);
+// Square brackets in a SARIF message are link syntax, and a finding's text can quote the tree. A
+// backslash escapes the character after it, so one before a bracket or a backslash is escaped too.
+const textOf = (s) => (typeof s === 'string' ? s.replace(/\\(?=[\\[\]])|[[\]]/g, '\\$&') : s);
 // SARIF's three levels fold critical into high and info into low, so the severity also travels as
 // the number GitHub code scanning ranks by. Info is coverage and context, which rank as nothing.
 const SECURITY_SEVERITY = { crit: '9.5', high: '8.0', med: '5.5', low: '3.0' };

package/lib/scan.mjs

@@ -74,7 +74,8 @@ function scanEach(root, opts) {
   const reports = opts.repos.map(repo => ({ repo, r: scanAll(join(root, repo), { ...opts, repos: null, repoName: repo, feedRoot: opts.feedRoot || root }) }));
   const findings = [];
   const checked = [];
-  const fixedAt = (repo, x) => (x?.fixAt ? { exploitability: { ...x, fixAt: { ...x.fixAt, path: underRepo(repo, x.fixAt.path) } } } : {});
+  const fixedAt = (repo, x) => (x?.fixAt ? { exploitability: { ...x, fixAt: { ...x.fixAt, path: underRepo(repo, x.fixAt.path),
+    ...(x.fixAt.builtInto ? { builtInto: { ...x.fixAt.builtInto, path: underRepo(repo, x.fixAt.builtInto.path) } } : {}) } } } : {});
   const prefixed = (repo, f) => ({ ...f, path: underRepo(repo, f.path), ...(f.related ? { related: f.related.map(x => ({ ...x, path: underRepo(repo, x.path) })) } : {}), ...fixedAt(repo, f.exploitability) });
   for (const { repo, r } of reports) {
     for (const f of r.findings) findings.push(prefixed(repo, f));