@portll/cobolwork 0.0.1 → 0.2.76

package/lib/sets/compile.mjs

@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Whether a program in the tree is one a compiler would accept: every name it uses is declared by
+// the program, by a copybook it includes, or by the compiler or a translator. Enterprise COBOL stops
+// a program that uses anything else with IGYPS2121-S. Source that cannot compile is not the source
+// of anything that runs, and a change carrying it was merged without a build, which is the mark of
+// generated "modernisation" code nobody compiled.
+//
+// A name is only undefined when everything the program copies was found. With a copybook missing
+// the name is most likely declared in it, so the program is counted as undecided rather than
+// reported: the missing member is the thing to act on, and the inventory already names it. The
+// same holds wherever the parse is not the whole program, and the count says why for each.
+import { basename, extname } from 'node:path';
+import { inScope, isProgram, relPath, readSource } from '../sources.mjs';
+import { report } from '../kernel/ruleset.mjs';
+import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
+import { eachWithinMemory } from '../kernel/memory.mjs';
+
+// construct: no input has to reach it. med: certain once every copybook is found, and it means the
+// tree does not hold what runs, but it is not exploitable in itself. CWE-1127: MITRE discourages
+// mapping to CWE-710, a pillar, and of its children this is the build that lets errors through, of
+// which source nobody compiled is the limit. CWE-1164 is code that runs to no effect; this cannot run.
+export const COMPILE_RULES = {
+  'compile-undefined-name': {
+    sev: 'med', evidence: 'construct', cwe: 'CWE-1127',
+    text: 'A program uses names that nothing it declares or copies defines, so it cannot compile',
+    impact: 'The program uses a name nothing it declares or copies defines, so Enterprise COBOL stops it with IGYPS2121-S: the source in the tree is not the source of anything that runs',
+    remedy: 'Define the missing name, or add the copybook that declares it, so the program compiles; if it is generated code, compile it before relying on it',
+  },
+};
+
+// These declare only DFH names or the SQLCA's own, which the parser already takes as supplied.
+const COVERED_SYSTEM = /^(DFHAID|DFHBMSCA|DFHEIBLK|SQLCA)$/i;
+const PARSE_TROUBLE = new Set(['unterminated-exec', 'unterminated-literal', 'copy-without-period']);
+// An Enterprise COBOL listing kept under a program's extension, whose page headers read as code.
+const LISTING = /^[01 -]?PP \d{4}-[A-Z0-9]{3} IBM /m;
+const SHOWN = 5;
+
+const nameOf = (p) => basename(p, extname(p)).toUpperCase();
+
+function whyUndecided(r, src) {
+  if (r.copies.some((c) => c.status === 'expansion-limit')) return 'its COPY or REPLACE statements expand past what one parse reads';
+  if (r.copies.some((c) => c.status !== 'resolved' && !(c.status === 'system' && COVERED_SYSTEM.test(c.name)))) return 'a copybook it includes is not in the tree';
+  // Panvalet's ++INCLUDE and Librarian's -INC, which a library manager expands before the compiler
+  // runs, and an INCLUDE or COPY somewhere the parser does not read one as a statement.
+  if (/^.{0,6}(?:-INC|\+\+INCLUDE)\s/im.test(src) || r.programs.some((p) => p.refs.some((x) => x.tok.u === 'INCLUDE' || x.tok.u === 'COPY'))) return 'it includes source the parser does not expand';
+  if (r.diags.some((d) => PARSE_TROUBLE.has(d.kind))) return 'the parse did not read it cleanly';
+  // A communication description declares its queue and status names in clauses the parser skips.
+  if (r.programs.some((p) => p.diags.some((d) => d.kind === 'unrecognised-data-sentence') || (p.fds || []).some((fd) => fd.kind === 'CD'))) return 'the parse did not recognise all of its data division';
+  return null;
+}
+
+// Where a text declares a name: after a level number, FD, SD, RD, CD, SELECT, INDEXED BY or a
+// directive's CONSTANT, or as a paragraph or section header in Area A. And the words of a COPY
+// REPLACING or a REPLACE, which is where a name comes from when the parser did not replace.
+const DECLARATION = /(?:^|[\s.])(?:0?[1-9]|[1-4][0-9]|66|77|78|88|FD|SD|RD|CD|SELECT(?:\s+OPTIONAL)?|INDEXED\s+BY|CONSTANT)\s+([A-Za-z0-9$#@_][A-Za-z0-9$#@_-]*)/gim;
+const HEADER = /^(?:.{0,6}[ \t]{1,4}|[ \t]{0,4})([A-Za-z0-9][A-Za-z0-9-]*)(?:[ \t]+SECTION(?:[ \t]+\d+)?)?[ \t]*\.[ \t]*\r?$/gm;
+const COPY_STATEMENT = /\bCOPY\b([\s\S]*?)(?:\.\s|\.$)/gim;
+const REPLACE_STATEMENT = /\bREPLACE\b([\s\S]*?)(?:\.\s|\.$)/gim;
+
+// A header declares a paragraph only where the sentence before it has ended: after a line of code
+// ending in a period, or with no code before it. Without the period the name is read as part of the
+// statement before, and the compiler reports it undefined.
+function sentenceEndsBefore(text, at) {
+  const lines = text.slice(0, at).split(/\r?\n/);
+  lines.pop();
+  for (let i = lines.length - 1; i >= 0; i--) {
+    const l = lines[i];
+    if (/^.{6}[*/]/.test(l) || /^\s*\*>/.test(l)) continue;
+    const code = l.slice(0, 72).replace(/\*>.*$/, '').replace(/^[0-9 ]{6}(?=[ \-D])/, '').trim();
+    if (code) return code.endsWith('.');
+  }
+  return true;
+}
+
+function declaredIn(text, names, into) {
+  const note = (w) => { const u = w.toUpperCase(); if (names.has(u)) into.add(u); };
+  for (const re of [DECLARATION, HEADER]) {
+    re.lastIndex = 0;
+    let m;
+    while ((m = re.exec(text))) if (re !== HEADER || sentenceEndsBefore(text, m.index)) note(m[1]);
+  }
+  for (const re of [COPY_STATEMENT, REPLACE_STATEMENT]) {
+    re.lastIndex = 0;
+    let m;
+    while ((m = re.exec(text))) {
+      if (re === COPY_STATEMENT && !/\bREPLACING\b/i.test(m[1])) continue;
+      for (const w of m[1].match(/[A-Za-z0-9][A-Za-z0-9-]*/g) || []) note(w);
+    }
+  }
+  return into;
+}
+
+// A reference cut at column 72 names a word the source does not hold.
+function truncated(ref, lines) {
+  const line = (lines.get(ref.file) || [])[ref.line - 1] || '';
+  const re = new RegExp(`(?<![A-Za-z0-9-])${ref.name.replace(/[$#@]/g, '\\$&')}`, 'gi');
+  let whole = false;
+  let any = false;
+  let m;
+  while ((m = re.exec(line))) { any = true; if (!/[A-Za-z0-9-]/.test(line[m.index + ref.name.length] || '')) whole = true; }
+  return any && !whole;
+}
+
+export function scanCompile(root, opts = {}) {
+  const tree = treeFor(root, opts);
+  const files = tree.list().filter(isProgram).filter(inScope(opts));
+  const findings = [];
+  const stats = { filesScanned: 0, filesUnreadable: 0, programsChecked: 0, programsUndecided: 0 };
+  const because = {};
+  const undecided = [];
+  const undecide = (f, id, why, refs) => {
+    stats.programsUndecided++;
+    because[why] = (because[why] || 0) + 1;
+    undecided.push(`${tree.rel(f)} (${id}): ${why}: ${[...new Set(refs.map((x) => x.name))].slice(0, SHOWN).join(', ')}`);
+  };
+  const textOf = (p) => (tree.contains(p) ? tree.text(p) : readSource(p)).text;
+  const candidates = [];
+
+  const run = eachWithinMemory(files, (f) => {
+    let src;
+    try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
+    if (LISTING.test(src)) { stats.filesScanned++; stats.listingsSkipped = (stats.listingsSkipped || 0) + 1; return src.length; }
+    let r;
+    try { r = tree.parse(f, src); } catch (e) { noteUnparsed(stats, tree, f, e); return src.length; }
+    stats.filesScanned++;
+    const blocked = whyUndecided(r, src);
+    const copied = [...new Set(r.copies.filter((c) => c.path).map((c) => c.path))];
+    for (const p of r.programs) {
+      // A class or a method has no PROGRAM-ID, and the data its methods use belongs to no program here.
+      if (!p.id || p.diags.some((d) => d.kind === 'no-procedure-division')) continue;
+      stats.programsChecked++;
+      if (!p.unresolvedRefs.length) continue;
+      if (blocked) { undecide(f, p.id, blocked, p.unresolvedRefs); continue; }
+      const names = new Set(p.unresolvedRefs.map((x) => x.name));
+      const texts = new Map([[f, src]]);
+      for (const q of copied) { try { texts.set(q, textOf(q)); } catch { /* the parse read it once; a second failure changes nothing */ } }
+      const seen = new Set();
+      for (const text of texts.values()) declaredIn(text, names, seen);
+      const lines = new Map([...texts].map(([q, t]) => [q, t.split(/\r?\n/)]));
+      const left = p.unresolvedRefs.filter((x) => !seen.has(x.name) && !truncated(x, lines));
+      if (!left.length) { undecide(f, p.id, 'the parse missed a declaration the text holds', p.unresolvedRefs); continue; }
+      candidates.push({ file: f, id: p.id, line: p.line, refs: left, copied: new Set(r.copies.map((c) => nameOf(c.name))) });
+    }
+    r = null;
+    return src.length;
+  }, { label: 'compile', maxBytes: opts.maxSourceBytes ?? Infinity });
+
+  // Another file answering to a name the program copies may be the one its build finds.
+  const wanted = new Set(candidates.flatMap((c) => [...c.copied]));
+  const answering = new Map();
+  if (wanted.size) {
+    for (const p of tree.list()) {
+      const n = nameOf(p);
+      if (!wanted.has(n)) continue;
+      if (!answering.has(n)) answering.set(n, []);
+      answering.get(n).push(p);
+    }
+  }
+  for (const c of candidates) {
+    const names = new Set(c.refs.map((x) => x.name));
+    const seen = new Set();
+    for (const n of c.copied) for (const p of answering.get(n) || []) { try { declaredIn(tree.text(p).text, names, seen); } catch { /* unread declares nothing */ } }
+    const left = c.refs.filter((x) => !seen.has(x.name));
+    if (!left.length) { undecide(c.file, c.id, 'another copybook of a name it copies declares them', c.refs); continue; }
+    const firsts = new Map();
+    for (const x of left) if (!firsts.has(x.name)) firsts.set(x.name, x);
+    const own = left.find((x) => x.file === c.file);
+    const at = (x) => (x.file === c.file ? `line ${x.line}` : `${relPath(root, x.file)}:${x.line}`);
+    const shown = [...firsts.values()].slice(0, SHOWN).map((x) => `${x.name} (${at(x)})`).join(', ');
+    const more = firsts.size > SHOWN ? `, and ${firsts.size - SHOWN} more` : '';
+    findings.push({
+      rule: 'compile-undefined-name', path: relPath(root, c.file), line: own ? own.line : c.line || 1, program: c.id,
+      names: [...firsts.keys()],
+      detail: `${c.id} uses ${firsts.size} name(s) that nothing it declares or copies defines, so it cannot compile: ${shown}${more}`,
+      related: [...firsts.values()].map((x) => ({ path: relPath(root, x.file), line: x.line, detail: `${x.name} is used here and declared nowhere the program can reach` })),
+    });
+  }
+
+  if (stats.programsUndecided) {
+    stats.undecidedBecause = because;
+    stats.undecided = undecided;
+    stats.coverageIncomplete = true;
+    stats.readInPart = `${stats.programsUndecided} program(s) use names no declaration the parse read defines, and the names may be declared where it did not reach: ${Object.entries(because).map(([k, v]) => `${v} because ${k}`).join('; ')}`;
+  }
+  if (stats.filesUnparsed) stats.coverageIncomplete = true;
+  return report('compile', { rules: COMPILE_RULES, findings, stats, run });
+}

package/lib/sets/copybook.mjs

@@ -0,0 +1,174 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { readFileSync } from 'node:fs';
+import { basename, dirname, extname } from 'node:path';
+import { detectFormat, parseSource } from '../parser.mjs';
+import { inScope, isCopybook, isProgram, relPath } from '../sources.mjs';
+import { finish } from '../kernel/findings.mjs';
+import { report } from '../kernel/ruleset.mjs';
+import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
+import { eachWithinMemory } from '../kernel/memory.mjs';
+
+// A copybook is resolved by name along a search path: the program's own directory first, then the
+// include directories, then the system library. Two files answering to one name mean the record
+// layout a program compiles against depends on where the program sits, and a copybook named like a
+// system one is found before the system's own. Both are how a single added file changes the layout
+// of every program that COPYs the name without touching any of them.
+
+export const COPYBOOK_RULES = {
+  'copybook-shadows-system': {
+    sev: 'med', evidence: 'tampering', cwe: 'CWE-427',
+    text: 'A copybook in the repository has the name of a system copybook, and is found before it',
+    impact: "A repository copybook answers to the name of a system copybook (SQLCA, DFHAID, ...), so the search path finds it before the system's own and a program compiles against the wrong layout",
+    remedy: 'Rename or remove the repository copy; let the system copybook resolve from its own library',
+  },
+  'copybook-shadowed': {
+    sev: 'med', evidence: 'tampering', cwe: 'CWE-427',
+    text: 'Two copybooks with different contents answer to the same name, and programs resolve it to different files',
+    impact: 'Two copybooks answer to one name with different layouts, so which one a program gets depends on its search path, not its source',
+    remedy: "Give the two copybooks distinct names, or make every program's search path resolve the intended one unambiguously",
+  },
+};
+
+const SYSTEM_NAME = /^(DFHAID|DFHBMSCA|DFHEIBLK|DFHCOMMAREA|DFHRESP|SQLCA|SQLDA|DSNHLI|CEEIGZCT|CMQV|CMQODV|CMQMDV|CMQPMOV|CMQGMOV|EZACICSO)$/i;
+
+// Layout text, not bytes. In fixed format the sequence and identification areas are not part of the
+// layout, and two copies differing only in change tags are the same copybook. In free format they
+// are ordinary text, and cutting at column 72 hid a PIC clause that sat past it.
+function layout(src) {
+  const fixed = detectFormat(src) === 'fixed';
+  return src.split(/\r?\n/)
+    .map(l => (fixed ? l.slice(6, 72) : l).trimEnd())
+    .filter(l => l.trim() && !/^[*/]/.test(l.trimStart()))
+    .join('\n')
+    .toUpperCase();
+}
+
+const nameOf = (p) => basename(p, extname(p)).toUpperCase();
+
+const SYSTEM_LAYOUTS = JSON.parse(readFileSync(new URL('../../rules/system-layouts.json', import.meta.url), 'utf8'));
+
+// Where a repository copy of a system copybook first departs from the system's layout, compared
+// field by elementary field; null when every field sits where the system puts it. Undefined when no
+// layout is held for the name or the copy does not parse.
+export function systemLayoutDeparture(name, text) {
+  const want = SYSTEM_LAYOUTS[String(name).toUpperCase()];
+  if (!want || !want.fields) return undefined;
+  let items;
+  try {
+    const wrapped = ['       IDENTIFICATION DIVISION.', '       PROGRAM-ID. LAYOUT.', '       DATA DIVISION.', '       WORKING-STORAGE SECTION.', text, '       PROCEDURE DIVISION.', '           GOBACK.'].join('\n');
+    items = parseSource(wrapped, `${name}.cpy`, { format: 'auto' }).programs[0]?.items;
+  } catch { return undefined; }
+  if (!items || !items.length) return undefined;
+  const byName = new Map(items.map((it) => [String(it.name).toUpperCase(), it]));
+  for (const [field, offset, size, occurs] of want.fields) {
+    const it = byName.get(field);
+    if (!it) return `${field} is missing`;
+    if (it.offset !== offset || it.size !== size || (it.occurs || 1) !== occurs) return `${field} is ${it.size} byte(s) at offset ${it.offset}${(it.occurs || 1) > 1 ? ` times ${it.occurs}` : ''}, where the system's is ${size} at ${offset}${occurs > 1 ? ` times ${occurs}` : ''}`;
+  }
+  const top = items.find((it) => it.level === 1);
+  if (top && top.size !== want.length) return `the record is ${top.size} bytes, where the system's is ${want.length}`;
+  return null;
+}
+
+export function scanCopybooks(root, opts = {}) {
+  const tree = treeFor(root, opts);
+  const all = tree.list().filter(inScope(opts));
+  const findings = [];
+  const stats = { filesScanned: 0, copybookFiles: 0, namesShadowed: 0, filesUnreadable: 0 };
+
+  const copies = new Map();
+  for (const p of all) {
+    if (!isCopybook(p)) continue;
+    stats.copybookFiles++;
+    stats.filesScanned++;
+    const n = nameOf(p);
+    if (!copies.has(n)) copies.set(n, []);
+    copies.get(n).push(p);
+  }
+
+  // Reading the COPY statements out of the text found only what a program names directly, counted a
+  // commented-out line as a use, and never saw a copybook reached through another copybook. The
+  // parser resolves all of that, so the parse is the answer — paid for once, and only when this
+  // tree holds something worth asking about.
+  const candidates = new Set([...copies].filter(([n, f]) => f.length > 1 || SYSTEM_NAME.test(n)).map(([n]) => n));
+  const referrers = new Map();
+  const resolved = new Map();
+  // This is the expensive half: it parses every program in the tree, and a parse tree is the kind
+  // of structure a byte budget does not bound. Only what each parse yields is kept - a name, a
+  // referrer, a resolved path - and the tree is released, so the working set stays flat. The guard
+  // is what stops the reading before the heap does.
+  let run = { skipped: [], stoppedBy: null, peakHeapBytes: 0, note: null };
+  if (candidates.size) {
+    run = eachWithinMemory(all.filter(isProgram), (p) => {
+      let src;
+      try { src = tree.text(p).text; } catch (e) { noteUnread(stats, tree, p, e); return 0; }
+      let r;
+      try {
+        r = tree.parse(p, src);
+      } catch (e) { noteUnparsed(stats, tree, p, e); return src.length; }
+      stats.filesScanned++;
+      for (const c of r.copies) {
+        const n = nameOf(c.name);
+        if (!candidates.has(n)) continue;
+        if (!referrers.has(n)) referrers.set(n, new Set());
+        referrers.get(n).add(p);
+        if (c.path) {
+          if (!resolved.has(n)) resolved.set(n, new Map());
+          resolved.get(n).set(p, c.path);
+        }
+      }
+      r = null;                     // the tree is not needed past this point
+      return src.length;
+    }, { label: 'copybook', maxBytes: opts.maxSourceBytes ?? Infinity });
+  }
+
+  for (const [n, files] of copies) {
+    if (SYSTEM_NAME.test(n)) {
+      for (const f of files) {
+        const users = [...(referrers.get(n) || [])].map(p => relPath(root, p));
+        let departure;
+        try { departure = systemLayoutDeparture(n, tree.text(f).text); } catch (e) { noteUnread(stats, tree, f, e); }
+        const layout = departure === null ? `; its layout is the system's, field for field`
+          : departure ? `; its layout departs from the system's: ${departure}` : '';
+        findings.push({ rule: 'copybook-shadows-system', path: relPath(root, f), line: 1, name: n, users: users.length,
+          ...(departure === null ? { layoutMatchesSystem: true } : {}),
+          detail: `${n} is a system copybook name; this file is found before the system library by ${users.length} program(s)${users.length ? `: ${users.slice(0, 5).join(', ')}${users.length > 5 ? ', …' : ''}` : ''}${layout}` });
+      }
+    }
+    if (files.length < 2) continue;
+    const layouts = new Map();
+    for (const f of files) {
+      let text;
+      try { text = layout(tree.text(f).text); } catch (e) { noteUnread(stats, tree, f, e); continue; }
+      if (!layouts.has(text)) layouts.set(text, []);
+      layouts.get(text).push(f);
+    }
+    if (layouts.size < 2) continue;
+
+    const resolvedBy = resolved.get(n) || new Map();
+    const targets = new Set(resolvedBy.values());
+    stats.namesShadowed++;
+    // Differing copies that every program resolves to one file are a latent trap, not a live one:
+    // reported, but the detail says which. Programs split across files are the live case.
+    const split = targets.size > 1;
+    const perFile = files.map(f => `${relPath(root, f)} (${[...resolvedBy.values()].filter(v => v === f).length} program(s))`);
+    findings.push({ rule: 'copybook-shadowed', path: relPath(root, files[0]), line: 1, name: n, split,
+      detail: `${files.length} files named ${n} with ${layouts.size} different layouts; ${split ? 'programs resolve it to different files' : 'every program that COPYs it resolves to the same file today'}: ${perFile.join(', ')}`,
+      related: files.slice(1).map(f => ({ path: relPath(root, f), line: 1, detail: `another copybook named ${n}` })) });
+  }
+
+  // This set decides one severity for itself, so it sets it before the shared stamping runs, which
+  // fills only what a finding does not already carry.
+  //
+  // Vendoring DFHAID or SQLCA so a program compiles off the mainframe is common and legitimate;
+  // measured on the 200-repository corpus, every system-named copy found was one. The finding
+  // stays, because the vendored copy is what those programs are built against, but it is not high.
+  for (const f of findings) {
+    const latent = (f.rule === 'copybook-shadowed' && !f.split) || (f.rule === 'copybook-shadows-system' && (!f.users || f.layoutMatchesSystem));
+    if (latent) f.sev = 'low';
+  }
+  const { byRule } = finish(COPYBOOK_RULES, findings, 'copybook');
+  // A program the scan never parsed is a program whose COPY statements nobody read, so a name
+  // that looks unshadowed may only look that way.
+  return report('copybook', { rules: COPYBOOK_RULES, findings, stats, run });
+}