@portll/cobolwork 0.5.0 → 0.7.0

package/lib/sets/secrets.mjs

@@ -0,0 +1,112 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Credentials written into COBOL source. The shapes are the ones rules/gitleaks-mainframe.toml gives
+// gitleaks, read from that file, so one list serves both and the build gate does not depend on
+// gitleaks being installed. Go's RE2 syntax is taken only where it means the same in JavaScript; any
+// other construct is refused when the file is read.
+import { readFileSync } from 'node:fs';
+import { inScope, isProgram, isCopybook, relPath } from '../sources.mjs';
+import { report } from '../kernel/ruleset.mjs';
+import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
+
+export const SECRETS_RULES = {
+  'credential-in-source': {
+    sev: 'high', evidence: 'construct', cwe: 'CWE-798', measured: false,
+    text: 'A credential is written into the source',
+    impact: 'Everyone who can read the source, its history or a listing can sign on with it, and changing it means changing and redeploying the program',
+    remedy: 'Read the credential at run time from where the site keeps them (a RACF-protected data set, a PassTicket, a Db2 trusted context or a CICS-supplied identity), and change the one already exposed',
+  },
+};
+
+// The TOML ids of the shapes that occur in COBOL programs and copybooks, and how a finding names each.
+const SHAPES = {
+  'cobol-value-credential': (m) => `${dataName(m) || 'a data item named for a credential'} has a literal VALUE`,
+  'embedded-sql-connect-password': () => 'EXEC SQL CONNECT gives a literal password',
+  'cics-signon-password': (m) => `EXEC CICS ${/\b(SIGNON|VERIFY|CHANGE)\b/i.exec(m)[1].toUpperCase()} gives a literal PASSWORD`,
+  'cics-new-password': (m) => `EXEC CICS ${/\b(SIGNON|CHANGE)\b/i.exec(m)[1].toUpperCase()} gives a literal NEWPASSWORD`,
+};
+
+const dataName = (m) => (/\b(?:0?[1-9]|[1-4][0-9]|77)\s+([A-Z0-9][A-Z0-9-]*)/i.exec(m) || [])[1]?.toUpperCase();
+
+const value = (raw) => {
+  const t = raw.trim();
+  if (t.startsWith("'''")) return t.slice(3, t.lastIndexOf("'''"));
+  if (t.startsWith('"')) return JSON.parse(t);
+  if (t.startsWith('[')) return [...t.matchAll(/'''(.*?)'''|"((?:[^"\\]|\\.)*)"/g)].map((x) => x[1] ?? JSON.parse(`"${x[2]}"`));
+  if (/^\d+$/.test(t)) return Number(t);
+  if (t === 'true' || t === 'false') return t === 'true';
+  throw new Error(`gitleaks-mainframe.toml: value not read: ${t}`);
+};
+
+// The [[rules]] tables of the file, each with its allowlist folded in.
+export function readToml(text) {
+  const rules = [];
+  let into = null;
+  for (const line of text.split('\n')) {
+    const l = line.trim();
+    if (!l || l.startsWith('#')) continue;
+    if (l === '[[rules]]') { rules.push((into = { allowlist: {} })); continue; }
+    if (l === '[rules.allowlist]') { into = rules.at(-1).allowlist; continue; }
+    if (l.startsWith('[')) { into = null; continue; }
+    const kv = /^([A-Za-z]+)\s*=\s*(.+)$/.exec(l);
+    if (kv && into) into[kv[1]] = value(kv[2]);
+  }
+  return rules;
+}
+
+// A gitleaks regex as a JavaScript one: a leading (?flags) group becomes the flags.
+export function jsRegex(re, extra = '') {
+  const lead = /^\(\?([ims]+)\)/.exec(re);
+  const body = lead ? re.slice(lead[0].length) : re;
+  if (/\(\?[a-zA-Z]|\(\?P<|\\[AzZpPQE]/.test(body)) throw new Error(`gitleaks-mainframe.toml: no JavaScript equivalent for ${re}`);
+  return new RegExp(body, [...new Set([...(lead ? lead[1] : ''), ...extra])].join(''));
+}
+
+const secretOf = (m, group) => (group ? m[group] : m.slice(1).find((g) => g !== undefined)) ?? m[0];
+
+export const SHAPE_RULES = readToml(readFileSync(new URL('../../rules/gitleaks-mainframe.toml', import.meta.url), 'utf8'))
+  .filter((r) => r.id in SHAPES)
+  .map((r) => ({
+    id: r.id,
+    pattern: jsRegex(r.regex, 'g'),
+    secretGroup: r.secretGroup || 0,
+    keywords: (r.keywords || []).map((k) => k.toLowerCase()),
+    allow: (r.allowlist.regexes || []).map((x) => jsRegex(x)),
+    allowTarget: r.allowlist.regexTarget || 'secret',
+  }));
+
+// Every shape in the text, at the line its first non-blank character is on.
+export function credentialsIn(text) {
+  const lower = text.toLowerCase();
+  const out = [];
+  for (const s of SHAPE_RULES) {
+    if (s.keywords.length && !s.keywords.some((k) => lower.includes(k))) continue;
+    for (const m of text.matchAll(s.pattern)) {
+      const target = s.allowTarget === 'match' ? m[0] : secretOf(m, s.secretGroup);
+      if (s.allow.some((a) => a.test(target))) continue;
+      const start = m.index + (m[0].length - m[0].trimStart().length);
+      out.push({ shape: s.id, line: text.slice(0, start).split('\n').length, detail: SHAPES[s.id](m[0]) });
+    }
+  }
+  return out;
+}
+
+export function* scanSecretsSteps(root, opts = {}) {
+  const tree = treeFor(root, opts);
+  const files = tree.list().filter((f) => isProgram(f) || isCopybook(f)).filter(inScope(opts));
+  const findings = [];
+  const stats = { filesScanned: 0, filesUnreadable: 0 };
+
+  const run = yield loopOver(files, (f) => {
+    let src;
+    try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
+    stats.filesScanned++;
+    const path = relPath(root, f);
+    for (const c of credentialsIn(src)) findings.push({ rule: 'credential-in-source', path, line: c.line, detail: `${c.detail} (${c.shape})` });
+    return src.length;
+  }, { label: 'secrets', maxBytes: opts.maxSourceBytes ?? Infinity });
+
+  return report('secrets', { rules: SECRETS_RULES, findings, stats, run });
+}
+
+export const scanSecrets = (root, opts = {}) => drive(scanSecretsSteps(root, opts));

package/lib/sets/semantics.mjs

@@ -9,7 +9,7 @@ import { basename, extname } from 'node:path';
 import { inScope, isProgram, isJcl, relPath, ebcdicByte } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { loadSite } from '../site.mjs';
 import { parseJcl } from '../jcl.mjs';
 import { optionCards, optionTokens, compileStepOptions } from '../options.mjs';
@@ -206,7 +206,7 @@ function thruPairs(toks) {
 }
 const WHEN_WORDS = new Set(['THRU', 'THROUGH', 'ALSO', 'OR', 'NOT', 'ANY', 'OTHER', 'TRUE', 'FALSE']);
 
-export function scanSemantics(root, opts = {}) {
+export function* scanSemanticsSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const all = tree.list().filter(inScope(opts));
   const files = all.filter(isProgram);
@@ -256,6 +256,9 @@ export function scanSemantics(root, opts = {}) {
     const narrowed = [];
 
     for (const st of p.statements) {
+      // Without TRUNC(OPT) only a COMPUTE is judged, and the slice below runs to the end of the
+      // program for a statement whose end the parse did not mark.
+      if (st.verb !== 'COMPUTE' && !truncOpt) continue;
       const seg = toks.slice(st.at + 1, st.end ?? toks.length);
       const sizeError = seg.some((t, k) => t.t === 'word' && t.u === 'SIZE' && seg[k + 1] && seg[k + 1].u === 'ERROR');
 
@@ -359,7 +362,7 @@ export function scanSemantics(root, opts = {}) {
     }
   }
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     let r;
@@ -375,3 +378,5 @@ export function scanSemantics(root, opts = {}) {
 
   return report('semantics', { rules: SEMANTICS_RULES, findings, stats, run });
 }
+
+export const scanSemantics = (root, opts = {}) => drive(scanSemanticsSteps(root, opts));

package/lib/sets/web.mjs

@@ -22,7 +22,7 @@ import { parseCsd } from '../csd.mjs';
 import { classify } from './log.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver, loopsOver } from '../kernel/shared-pass.mjs';
 
 
 export const WEB_RULES = {
@@ -168,7 +168,7 @@ const HTTP_METHOD = /^(GET|POST|PUT|PATCH|DELETE)$/i;
 // the absence and the explicit NO are one finding, and PROTOCOL only changes what is in the clear.
 const encrypted = (svc) => /^(YES|CLIENTAUTH|CLIENTCERT)$/i.test(String(svc.ssl || ''));
 
-export function scanWeb(root, opts = {}) {
+export function* scanWebSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const all = tree.list().filter(inScope(opts));
   const files = all.filter(isProgram);
@@ -302,11 +302,12 @@ export function scanWeb(root, opts = {}) {
   // is named whatever the estate named it, and filtering on an extension is how a BMS count once
   // came out as zero.
   // Client URIMAPs come from the same files: WEB OPEN URIMAP(name) takes its scheme from one.
+  // Read in the pass the program loop shares, so a URIMAP is looked up only once every file is read.
   const urimaps = new Map();
-  for (const f of all) {
+  const definitions = loopOver(all, (f) => {
     let src;
-    try { src = tree.text(f).text; } catch { continue; }
-    if (!/^\s*DEFINE\s+(TCPIPSERVICE|URIMAP)\s*\(/im.test(src)) continue;
+    try { src = tree.text(f).text; } catch { return; }
+    if (!/^\s*DEFINE\s+(TCPIPSERVICE|URIMAP)\s*\(/im.test(src)) return;
     const path = relPath(root, f);
     const csd = parseCsd(src);
     for (const [name, u] of csd.urimaps) if (!urimaps.has(name)) urimaps.set(name, { ...u, path });
@@ -321,9 +322,11 @@ export function scanWeb(root, opts = {}) {
           + ', so everything the session carries crosses the network in the clear',
       });
     }
-  }
+  }, { label: 'web' });
 
-  const run = eachWithinMemory(files, (f) => {
+  // A program's findings in its own order, an open through a URIMAP held as the function that judges it.
+  const fromPrograms = [];
+  const programs = loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     // The whole set is about EXEC CICS WEB and EXEC CICS DOCUMENT. Nothing else needs parsing.
@@ -348,23 +351,25 @@ export function scanWeb(root, opts = {}) {
         const to = o.host ? ` to ${o.host}` : '';
         if (o.https) continue;
         if (o.http) {
-          findings.push({ rule: 'web-client-opens-cleartext', ...o.at, program: v.id,
+          fromPrograms.push({ rule: 'web-client-opens-cleartext', ...o.at, program: v.id,
             detail: `${v.id} opens a connection${to} asking for HTTP rather than HTTPS, so what it sends and reads crosses the network in the clear unless an AT-TLS policy outside the program encrypts it` });
           continue;
         }
-        const u = o.urimap ? urimaps.get(o.urimap.toUpperCase()) : null;
-        if (u && /^HTTP$/i.test(u.scheme || '') && !/^AWARE$/i.test(u.attls || '')) {
-          findings.push({ rule: 'web-client-opens-cleartext', ...o.at, program: v.id,
-            detail: `${v.id} opens a connection through URIMAP ${o.urimap.toUpperCase()}, which ${u.path}:${u.line} defines with SCHEME(HTTP)${u.host ? ` to ${u.host}` : ''} and no ATTLS(AWARE), so what it sends and reads crosses the network in the clear unless an AT-TLS policy encrypts it` });
-          continue;
-        }
-        // A scheme in a variable, or a URIMAP the tree does not define, is decided where this cannot see.
-        if (!u || !u.scheme) stats.opensUndecided++;
+        fromPrograms.push(() => {
+          const u = o.urimap ? urimaps.get(o.urimap.toUpperCase()) : null;
+          if (u && /^HTTP$/i.test(u.scheme || '') && !/^AWARE$/i.test(u.attls || '')) {
+            return { rule: 'web-client-opens-cleartext', ...o.at, program: v.id,
+              detail: `${v.id} opens a connection through URIMAP ${o.urimap.toUpperCase()}, which ${u.path}:${u.line} defines with SCHEME(HTTP)${u.host ? ` to ${u.host}` : ''} and no ATTLS(AWARE), so what it sends and reads crosses the network in the clear unless an AT-TLS policy encrypts it` };
+          }
+          // A scheme in a variable, or a URIMAP the tree does not define, is decided where this cannot see.
+          if (!u || !u.scheme) stats.opensUndecided++;
+          return null;
+        });
       }
 
       for (const r of v.receives) {
         if (r.max.bytes <= r.size) continue;
-        findings.push({ rule: 'web-receive-length-exceeds-area', ...r.at, program: v.id,
+        fromPrograms.push({ rule: 'web-receive-length-exceeds-area', ...r.at, program: v.id,
           detail: `${v.id} has WEB ${r.verb} pass up to MAXLENGTH(${r.max.written}), ${r.max.bytes} bytes, into ${r.area}, which is ${r.size} bytes` });
       }
       stats.headersWritten += v.headers.length;
@@ -378,7 +383,7 @@ export function scanWeb(root, opts = {}) {
         if (!/;\s*httponly\b/i.test(c.value)) missing.push('HttpOnly');
         if (!/;\s*samesite\s*=/i.test(c.value)) missing.push('SameSite');
         if (!missing.length) continue;
-        findings.push({
+        fromPrograms.push({
           rule: 'web-cookie-without-secure-attributes', ...c.at, program: v.id,
           detail: `${v.id} writes Set-Cookie without ${missing.join(', ')}`
             + (missing.includes('Secure') ? ', so the browser will send this cookie over a cleartext hop' : ''),
@@ -386,7 +391,7 @@ export function scanWeb(root, opts = {}) {
       }
 
       for (const h of v.headers.filter((x) => TELLS.test(x.name))) {
-        findings.push({
+        fromPrograms.push({
           rule: 'web-response-tells-the-caller-what-it-runs', ...h.at, program: v.id,
           detail: `${v.id} writes the header ${h.name}${h.value ? ` with the value '` + h.value + `'` : ''}, which names the software answering rather than anything the client needs`,
         });
@@ -397,14 +402,14 @@ export function scanWeb(root, opts = {}) {
         ? { line: v.built[0].line, why: 'STRINGs ' + v.built[0].name + ' into a value it sends, and the name says it holds a credential' }
         : (v.uris.length ? { line: 1, why: 'builds a URI whose query names a credential: ' + v.uris[0] } : null);
       if (credInUri) {
-        findings.push({
+        fromPrograms.push({
           rule: 'web-uri-carries-a-credential', path, line: credInUri.line, program: v.id,
           detail: v.id + ' ' + credInUri.why,
         });
       }
       if (v.webRequest && v.changes.length && !v.comparesToken) {
         const what = [...new Set(v.changes.map((c) => c.what))];
-        findings.push({
+        fromPrograms.push({
           rule: 'web-request-changes-state-without-a-token', ...where(v.changes[0].e, path), program: v.id,
           detail: `${v.id} changes state (${what.slice(0, 3).join(', ')}${what.length > 3 ? ', …' : ''}) on a request it received from the web and compares no token with another field`
             + (v.testsMethod ? '; it tests the HTTP method, which a forged request sets too' : '')
@@ -420,7 +425,7 @@ export function scanWeb(root, opts = {}) {
       // A program that wrote none of the three has said nothing about any of this; one that wrote
       // two of three made a choice about the third. Both are reported, and the detail says which.
       if (absent.length) {
-        findings.push({
+        fromPrograms.push({
           rule: 'web-response-without-protective-headers', ...at, program: v.id,
           detail: `${v.id} sends HTML and writes no ${absent.map((h) => h.header.source.replace(/[^a-z-]/gi, '')).join(', no ')} header`
             + (v.computedHeaders ? `. ${v.computedHeaders} header name or value in this program is computed, so what it sets was not read` : ''),
@@ -431,7 +436,7 @@ export function scanWeb(root, opts = {}) {
       // the document is built from.
       for (const m of v.docText.matchAll(/<a\b[^>]*target\s*=\s*["']?_blank[^>]*>/gi)) {
         if (/\brel\s*=\s*["'][^"']*noopener/i.test(m[0])) continue;
-        findings.push({
+        fromPrograms.push({
           rule: 'web-link-opens-without-noopener', ...at, program: v.id,
           detail: `${v.id} writes a link with target="_blank" and no rel="noopener", so the page it opens can navigate this one`,
         });
@@ -442,5 +447,16 @@ export function scanWeb(root, opts = {}) {
     return src.length;
   }, { label: 'web', maxBytes: opts.maxSourceBytes ?? Infinity });
 
+  // A pass the memory guard stopped after the last program but before a later definition is no
+  // more complete than one stopped among the programs.
+  const [defined, read] = yield loopsOver(definitions, programs);
+  const run = read.complete && !defined.complete ? defined : read;
+  for (const x of fromPrograms) {
+    const finding = typeof x === 'function' ? x() : x;
+    if (finding) findings.push(finding);
+  }
+
   return report('web', { rules: WEB_RULES, findings, stats, run });
 }
+
+export const scanWeb = (root, opts = {}) => drive(scanWebSteps(root, opts));

package/lib/site.mjs

@@ -12,8 +12,10 @@ import { readFileSync, existsSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { resolvesInside } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
+import { versionProblem, unknownKeys, ignoredKey } from './kernel/version-key.mjs';
 
 export const SITE_FILE = 'cobolwork.site.json';
+export const SITE_VERSION = 1;
 
 const EMPTY = {
   present: false,
@@ -38,8 +40,11 @@ const EMPTY = {
   privilegedTransactions: [],
   privilegedJobs: [],
   problems: [],
+  warnings: [],
 };
 
+const KNOWN_KEYS = new Set(['$schema', 'version', ...Object.keys(EMPTY).filter((k) => !['present', 'problems', 'warnings'].includes(k))]);
+
 // A configuration that is absent is not a configuration that says "nothing is production". The
 // difference is the whole point: without it the recon rules have not looked, and a scan must say
 // it has not looked rather than report a clean result.
@@ -65,6 +70,10 @@ export function loadSite(root, explicit = null, tree = null) {
   }
   if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return { ...EMPTY, path, problems: [`${SITE_FILE} is not a JSON object`] };
 
+  const unreadable = versionProblem(raw, SITE_VERSION);
+  if (unreadable) return { ...EMPTY, path, problems: [`${SITE_FILE} ${unreadable}, so it was not read`] };
+  const warnings = unknownKeys(raw, KNOWN_KEYS).map((k) => ignoredKey(SITE_FILE, k));
+
   const problems = [];
   const list = (name) => {
     const v = raw[name];
@@ -123,6 +132,7 @@ export function loadSite(root, explicit = null, tree = null) {
     privilegedTransactions: list('privilegedTransactions').map((s) => s.toUpperCase()),
     privilegedJobs: list('privilegedJobs').map((s) => s.toUpperCase()),
     problems,
+    warnings,
   };
 
   if (site.present && !site.productionQualifiers.length && !site.systemNames.length) {

package/lib/sources.mjs

@@ -17,6 +17,9 @@ export const BMS_EXT = ['.bms'];
 // Assembler source. A BMS map or an IMS DBD or PSB is also assembler macros, and is often named .asm;
 // lib/hlasm.mjs tells them apart by the macros they hold.
 export const HLASM_EXT = ['.asm', '.mac', '.mlc', '.hlasm', '.assemble'];
+export const IMS_EXT = ['.dbd', '.psb'];
+export const DB2_EXT = ['.sql', '.ddl', '.db2'];
+export const PLI_EXT = ['.pli', '.pl1', '.plx'];
 
 const has = (list) => (p) => list.includes(extname(p).toLowerCase());
 
@@ -26,19 +29,22 @@ const has = (list) => (p) => list.includes(extname(p).toLowerCase());
 // of it and reporting a clean zero, which is the failure this project exists to refuse.
 //
 // So a file with no extension is classified by what is in it. Only the first few kilobytes are
-// read and only files without an extension are sniffed. The tree a path belongs to classifies it
-// and keeps the answer for as long as the tree lives: this runs over every file in the tree and
-// must not cost a parse, and a cache that outlives the scan grows by one entry per file for the
-// life of the process.
+// read, and more only when they read like assembler cards that have not yet said so; only files
+// without an extension are sniffed. The tree a path belongs to classifies it and keeps the answer
+// for as long as the tree lives: this runs over every file in the tree and must not cost a parse,
+// and a cache that outlives the scan grows by one entry per file for the life of the process.
 const SNIFF_BYTES = 4096;
+const WIDE_BYTES = 65536;
 
 // Names that are certainly not source, so the commonest extensionless files cost nothing.
 const NOT_SOURCE = /^(LICEN[CS]E|README|MAKEFILE|DOCKERFILE|CHANGELOG|NOTICE|AUTHORS|CONTRIBUTING|CODEOWNERS|VERSION|COPYING|Jenkinsfile|Procfile|\..*)$/i;
 const notSource = (path) => NOT_SOURCE.test(path.split(/[\\/]/).pop());
 
+const textOf = (buf) => (looksEbcdic(buf) ? decodeEbcdic(buf) : buf.toString('latin1'));
+
 function kindOfHead(head) {
   // An EBCDIC member is decoded before it is read, or every test below sees mojibake.
-  const text = looksEbcdic(head) ? decodeEbcdic(head) : head.toString('latin1');
+  const text = textOf(head);
   // A NUL byte means binary. Tested on the bytes rather than with a literal, because a literal
   // NUL in source is exactly the character an editor or a patch is most likely to eat.
   if (head.includes(0)) return null;
@@ -49,25 +55,62 @@ function kindOfHead(head) {
   // A BMS map opens its mapset with DFHMSD in the operation field. Tested last, so nothing that
   // classified before classifies differently.
   if (/^[^*\s]*[ \t]+DFHMSD(?=\s|$)/im.test(text)) return 'bms';
+  // An IMS DBD or PSB is assembler macros too: a DBD statement and a SEGM, or a PCB and a PSBGEN.
+  if (/^[^*\s]*[ \t]+(?:DBD|PCB)(?=\s|$)/m.test(text) && /^[^*\s]*[ \t]+(?:SEGM|DBDGEN|SENSEG|PSBGEN)(?=\s|$)/m.test(text)) return 'ims';
   // Assembler source defines a section or a macro, and declares storage or a base register, in the
   // operation field; either alone is a word a text file can hold. Tested after BMS, which is
   // assembler macros too.
   if (/^[^*\s]*[ \t]+(?:CSECT|RSECT|START|MACRO)(?=\s|$)/m.test(text)
     && /^[^*\s]*[ \t]+(?:USING|DC|DS|BALR|BASR|STM|STMG|MEND)(?=\s|$)/m.test(text)) return 'hlasm';
+  // Db2 DDL: a statement that creates an object or grants a privilege, at the start of a line and
+  // ended by a semicolon. Tested last, after the program and JCL that also carry SQL.
+  if (/^\s*(?:CREATE\s+(?:TABLE|TABLESPACE|DATABASE|STOGROUP|(?:UNIQUE\s+)?INDEX|VIEW|PROCEDURE|FUNCTION|ALIAS|SYNONYM)|GRANT\s)/im.test(text) && /;\s*$/m.test(text)) return 'db2';
   return null;
 }
 
+// A head of 4 KB misses assembler whose prologue runs past it, a macro whose first DS or MEND
+// comes later, and a program whose section a macro generates. When the head is mostly comment
+// cards and statements in the card's fields, up to 64 KB is read and the member is assembler if
+// it holds a section-level and a statement-level operation from the wider lists, or five distinct
+// common machine instructions with two operands.
+const CARD = /^(?:\*|\.\*|(?:[A-Z$#@&.][A-Z0-9$#@_&.]{0,62})?[ \t]+[A-Z$#@][A-Z0-9$#@]{0,7}(?:[ \t]|$))/i;
+const inOperationField = (ops) => new RegExp(String.raw`^[^*\s]*[ \t]+(?:${ops})(?=\s|$)`, 'm');
+const WIDE_SECTION = inOperationField('CSECT|RSECT|START|MACRO|DSECT|TITLE|ENTRY|AMODE|RMODE');
+const WIDE_STATEMENT = inOperationField('USING|DC|DS|BALR|BASR|STM|STMG|MEND|AIF|AGO|SETA|SETB|SETC|GBLA|GBLB|GBLC|LCLA|LCLB|LCLC|ANOP|MNOTE|EQU|LTORG|EJECT|PRINT');
+const INSTRUCTION = /^[^*\s]*[ \t]+(?:L|LA|LR|LH|ST|STH|STM|LM|MVC|MVI|CLC|CLI|CL|CR|CH|C|LTR|TM|IC|STC|SR|AR|A|S|N|O|X|NI|OI|XC|NC|OC|ZAP|AP|SP|CVD|CVB|PACK|UNPK|ED|EX|BAL|BALR|BAS|BASR|BCT|BXLE|BXH|SLL|SRL|SLA|SRA|SLDL|SRDL|ICM|STCM|CLM|TR|TRT|MVCL|CLCL)[ \t]+(?:R?\d{1,2}|[A-Z$#@][A-Z0-9$#@_]*\(?)[^ \t,]*,\S/gim;
+
+function cardShaped(text) {
+  const lines = text.split(/\r?\n/).filter((l) => l.trim());
+  return lines.length >= 10 && lines.filter((l) => CARD.test(l)).length / lines.length >= 0.8;
+}
+
+function assemblerFurtherIn(text) {
+  if (WIDE_SECTION.test(text) && WIDE_STATEMENT.test(text)) return true;
+  return new Set(text.match(INSTRUCTION) || []).size >= 5;
+}
+
+// The head decides; a card-shaped head it leaves undecided is read on through `wider`.
+function kindOfMember(head, wider) {
+  const kind = kindOfHead(head);
+  if (kind || head.includes(0) || !cardShaped(textOf(head))) return kind;
+  return assemblerFurtherIn(textOf(wider())) ? 'hlasm' : null;
+}
+
 // What a member's bytes say it is, for a tree that holds them.
-export const kindOfBytes = (buf) => kindOfHead(buf.subarray(0, SNIFF_BYTES));
+export const kindOfBytes = (buf) => kindOfMember(buf.subarray(0, SNIFF_BYTES), () => buf.subarray(0, WIDE_BYTES));
+
+function readHead(path, bytes) {
+  const fd = openSync(path, 'r');
+  const buf = Buffer.alloc(bytes);
+  let n;
+  try { n = readSync(fd, buf, 0, bytes, 0); } finally { closeSync(fd); }
+  return buf.subarray(0, n);
+}
 
 function sniffDisk(path) {
   if (notSource(path)) return null;
   try {
-    const fd = openSync(path, 'r');
-    const buf = Buffer.alloc(SNIFF_BYTES);
-    let n;
-    try { n = readSync(fd, buf, 0, SNIFF_BYTES, 0); } finally { closeSync(fd); }
-    return kindOfHead(buf.subarray(0, n));
+    return kindOfMember(readHead(path, SNIFF_BYTES), () => readHead(path, WIDE_BYTES));
   } catch { return null; }
 }
 
@@ -125,11 +168,15 @@ export const isCopybook = byExtOrContent(COPY_EXT, 'copybook');
 export const isJcl = byExtOrContent(JCL_EXT, 'jcl');
 export const isBms = byExtOrContent(BMS_EXT, 'bms');
 export const isAssembler = byExtOrContent(HLASM_EXT, 'hlasm');
-// Not BMS or assembler, by either route: the sets that read every source file have not chosen to
-// read maps or assembler.
+export const isIms = byExtOrContent(IMS_EXT, 'ims');
+export const isDb2 = byExtOrContent(DB2_EXT, 'db2');
+// PL/I is known by its extension only: no content sniff reads an extensionless member as PL/I.
+export const isPli = (p) => PLI_EXT.includes(extname(p).toLowerCase());
+// Not BMS, assembler, an IMS definition or Db2 DDL, by either route: the sets that read every source
+// file have not chosen to read those.
 export const isSource = (p) => (extname(p)
   ? [...PROGRAM_EXT, ...COPY_EXT, ...JCL_EXT].includes(extname(p).toLowerCase())
-  : ![null, 'bms', 'hlasm'].includes(sniffKind(p)));
+  : ![null, 'bms', 'hlasm', 'ims', 'db2'].includes(sniffKind(p)));
 
 // `opts.only` is about rule sets; this is about files. A rule set given an allow list reads the
 // files on it and no others.
@@ -199,14 +246,19 @@ export function looksEbcdic(buf) {
 }
 
 export function decodeEbcdic(buf) {
-  let hasNewline = false;
-  for (let i = 0; i < buf.length; i++) if (buf[i] === 0x15 || buf[i] === 0x25) { hasNewline = true; break; }
+  let newlines = 0;
+  for (let i = 0; i < buf.length; i++) if (buf[i] === 0x15 || buf[i] === 0x25) newlines++;
   const chars = new Array(buf.length);
   for (let i = 0; i < buf.length; i++) chars[i] = E2A[buf[i]];
-  const text = chars.join('');
-  // A member copied off the mainframe in binary keeps its fixed 80-byte records and no line ends.
-  if (!hasNewline && buf.length % 80 === 0) return text.match(/[\s\S]{80}/g).join('\n');
-  return text;
+  // A member copied off the mainframe in binary keeps its fixed 80-byte records and no line ends. A
+  // constant can hold the newline bytes X'15' and X'25' as data, so a few of them, fewer than one
+  // record in two, do not make the member text with line ends.
+  if (buf.length % 80 === 0 && newlines < buf.length / 160) {
+    const records = [];
+    for (let i = 0; i < chars.length; i += 80) records.push(chars.slice(i, i + 80).join('').replace(/[\n\r]/g, ' '));
+    return records.join('\n');
+  }
+  return chars.join('');
 }
 
 // Every rule set reads source through here, so an EBCDIC member is decoded once, the same way,
@@ -214,7 +266,15 @@ export function decodeEbcdic(buf) {
 export const readSource = (path) => decodeSource(readFileSync(path));
 
 // The same decoding for bytes that did not come from a file: a blob read out of a git revision.
+const UTF8 = new TextDecoder('utf-8', { fatal: true });
+
+// A character beyond ASCII takes two to four bytes in UTF-8, and read a byte at a time it takes as
+// many columns, which moves everything after it on a fixed-format line: column 72, the sequence
+// area, the end of a card. A file that is valid UTF-8 and holds such characters is read as UTF-8.
 export function decodeSource(buf) {
   if (looksEbcdic(buf)) return { text: decodeEbcdic(buf), encoding: 'ebcdic' };
+  if (buf.some((b) => b >= 0x80)) {
+    try { return { text: UTF8.decode(buf), encoding: 'utf-8' }; } catch { /* not UTF-8: one byte, one character */ }
+  }
   return { text: buf.toString('latin1'), encoding: 'latin1' };
 }

package/lib/statement-cursor.mjs

@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// A cursor over one statement's tokens, shared by the PL/I and Db2 statement parsers so they read
+// tokens the same way; each language fails with its own syntax error.
+
+const isOpen = (t) => t && t.t === 'op' && t.v === '(';
+const isClose = (t) => t && t.t === 'op' && t.v === ')';
+
+export function tokenCursor(toks, Syntax) {
+  let i = 0;
+  const c = {
+    get pos() { return i; },
+    set pos(v) { i = v; },
+    peek: (k = 0) => toks[i + k] || null,
+    next: () => toks[i++] || null,
+    done: () => i >= toks.length,
+    rest: () => toks.slice(i),
+    drain: () => { const r = toks.slice(i); i = toks.length; return r; },
+    slice: (from, to = i) => toks.slice(from, to),
+    isWord: (u, k = 0) => { const t = toks[i + k]; return !!t && t.t === 'word' && (u === undefined || (Array.isArray(u) ? u.includes(t.u) : t.u === u)); },
+    isOp: (v, k = 0) => { const t = toks[i + k]; return !!t && t.t === 'op' && (Array.isArray(v) ? v.includes(t.v) : t.v === v); },
+    word(u) { return c.isWord(u) ? toks[i++] : null; },
+    op(v) { return c.isOp(v) ? toks[i++] : null; },
+    expectWord(u) { const t = c.word(u); if (!t) c.fail(u === undefined ? 'a name' : Array.isArray(u) ? u.join(' or ') : u); return t; },
+    expectOp(v) { const t = c.op(v); if (!t) c.fail(`'${v}'`); return t; },
+    fail(what) { const t = toks[i]; throw new Syntax(`expected ${what}${t ? ` at '${t.v ?? t.t}'` : ' at end of statement'}`, t || toks[toks.length - 1]); },
+    // The tokens between a '(' at the cursor and its closing ')', the cursor left after the ')'.
+    group() {
+      if (!isOpen(toks[i])) c.fail("'('");
+      const start = i;
+      let depth = 0;
+      for (; i < toks.length; i++) {
+        if (isOpen(toks[i])) depth++;
+        else if (isClose(toks[i]) && --depth === 0) { i++; return toks.slice(start + 1, i - 1); }
+      }
+      i = start;
+      c.fail("')'");
+    },
+    // A parenthesised list as its comma-separated members, each a token array.
+    items() { return split(c.group(), ','); },
+    // Tokens up to, not including, the first at depth 0 that `stop` accepts.
+    until(stop) {
+      const start = i;
+      let depth = 0;
+      for (; i < toks.length; i++) {
+        const t = toks[i];
+        if (depth === 0 && stop(t)) break;
+        if (isOpen(t)) depth++;
+        else if (isClose(t)) depth--;
+      }
+      return toks.slice(start, i);
+    },
+  };
+  return c;
+}
+
+// Token arrays split at depth-0 tokens of op `sep`.
+export function split(toks, sep) {
+  const parts = [[]];
+  let depth = 0;
+  for (const t of toks) {
+    if (isOpen(t)) depth++;
+    else if (isClose(t)) depth--;
+    if (depth === 0 && t.t === 'op' && t.v === sep) { parts.push([]); continue; }
+    parts[parts.length - 1].push(t);
+  }
+  return parts.length === 1 && !parts[0].length ? [] : parts;
+}

package/lib/verify.mjs

@@ -29,7 +29,7 @@ const SINKS = {
   'record-update': (x) => ({ value: 'the key of a test record set up for a different test user', reads: `${x.item} holding that key`, run: 'the other user\'s test record is changed or deleted: set the record up for this test alone' }),
   'dynamic-program-load': (x) => ({ value: ABSENT, reads: `${MARKER} as the program the CALL loads, from ${x.item}`, run: `the CALL fails to load ${MARKER} (an S806 abend in batch)` }),
   'cics-dynamic-transfer': (x) => ({ value: ABSENT, reads: `${MARKER} as the program or transaction the command names, from ${x.item}`, run: `the command fails with PGMIDERR, or TRANSIDERR for a START` }),
-  'dynamic-file-path': (x) => ({ value: ABSENT, reads: `${MARKER} as the file the program assigns, from ${x.item}`, run: 'the OPEN fails with file status 35' }),
+  'dynamic-file-path': (x) => ({ value: ABSENT, reads: `${MARKER} as the file the program ${x.cics ? 'names' : 'assigns'}, from ${x.item}`, run: x.cics ? 'the command fails with FILENOTFOUND' : 'the OPEN fails with file status 35' }),
   'cics-sysid': (x) => ({ value: ABSENT, reads: `${MARKER} as the SYSID the command is shipped to, from ${x.item}`, run: 'the command fails with SYSIDERR' }),
   'queue-name': (x) => marker(x, 'as the queue the command acts on', 'the command would act on a queue the input chose'),
   'web-response': (x) => ({ value: `${MARKER}<>`, reads: `${MARKER}<> in ${x.item}`, run: `the response body carries ${MARKER}<> as sent, not ${MARKER}&lt;&gt;` }),
@@ -73,6 +73,7 @@ function enterAt(source, f, src, loc) {
     case 'jcl-parm': return `${src?.detail || 'the PARM'}, in a copy of the job`;
     case 'jcl-instream': return `${src?.detail || 'the in-stream data'}, in a copy of the job`;
     case 'file-record': return `a test record in a test copy of the file read at ${loc}`;
+    case 'cics-queue': return `a test item in the queue read at ${loc}, written by a test task`;
     case 'database': return `a test row in a test copy of the table read at ${loc}`;
     default: return null;
   }
@@ -95,7 +96,7 @@ export function verificationPlan(f) {
   const loc = src ? `${src.path}:${src.line}` : 'the source';
   const last = f.trace?.length ? f.trace[f.trace.length - 1] : null;
   const table = Number(/a table of (\d+)$/.exec(f.detail || '')?.[1]) || null;
-  const x = { item: last?.item || 'the value', table, ssrange: !!f.ssrange };
+  const x = { item: last?.item || 'the value', table, ssrange: !!f.ssrange, cics: / reaches EXEC CICS /.test(f.detail || '') };
   const sink = `${f.path}:${f.line}`;
   const tool = toolFor(kinds.source, entries, f.program || 'the program');
   const start = entries.length

package/lib/version.mjs

@@ -6,6 +6,12 @@ import { readFileSync } from 'node:fs';
 // 3: findings carry `evidence`, and setsIncomplete entries a `kind`.
 export const SCHEMA_VERSION = 3;
 
+// The smaller documents each command writes, versioned apart from the report.
+export const EXPLAIN_SCHEMA_VERSION = 1;
+export const PARSE_SCHEMA_VERSION = 1;
+export const BASELINE_RESULT_SCHEMA_VERSION = 1;
+export const EVIDENCE_RESULT_SCHEMA_VERSION = 1;
+
 // Names how taint moves through groups, so a finding that appears or disappears between two reports
 // can be attributed to the model rather than to the code scanned. 'byte-range': a tainted field
 // taints those bytes of its group, and the bytes are followed through MOVE, REDEFINES and CALL.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@portll/cobolwork",
-  "version": "0.5.0",
+  "version": "0.7.0",
   "description": "COBOL, JCL and CICS security analysis with zero runtime dependencies: reference-format parser, cross-program data flow, mainframe credential rules. Plugs into commitwork.",
   "license": "AGPL-3.0-or-later",
   "author": "Portll <john@portll.net>",
@@ -33,7 +33,8 @@
     "NOTICE",
     "LICENSING.md",
     "THIRD-PARTY-NOTICES.md",
-    "README.md"
+    "README.md",
+    "STABILITY.md"
   ],
   "scripts": {
     "test": "node --test",