@portll/cobolwork 0.2.117 → 0.2.150

package/lib/equivalence.mjs

@@ -50,12 +50,18 @@ function closure(root, file, idx, systemDirs) {
   return out;
 }
 
-// Programs the change edits or adds, or whose copybooks it edits: [{ path, base, head, via }] with
-// each side's source digest and the copybooks that changed under an unchanged program.
+// Programs the change edits, adds or deletes, or whose copybooks it edits: [{ path, base, head, via }]
+// with each side's source digest (head null for a deleted program) and the copybooks that changed
+// under an unchanged program.
 export function changedPrograms(baseDir, headDir, allow = null, systemDirs = []) {
   const out = [];
   const headTree = directoryTree(headDir);
-  const baseIdx = directoryTree(baseDir).index;
+  const baseTree = directoryTree(baseDir);
+  const baseIdx = baseTree.index;
+  for (const base of baseTree.list().filter(isProgram).sort()) {
+    const path = relPath(baseDir, base);
+    if (!existsSync(join(headDir, path))) out.push({ path, base: sha256(readFileSync(base)), head: null, deleted: true });
+  }
   for (const head of headTree.list().filter((p) => isProgram(p) && (!allow || allow.has(p))).sort()) {
     const path = relPath(headDir, head);
     const h = sha256(readFileSync(head));
@@ -74,12 +80,17 @@ export function changedPrograms(baseDir, headDir, allow = null, systemDirs = [])
 // Whether a commit in base..head reads as authored by a tool, from its identities and trailers.
 const TOOL = /\[bot\]|(^|[^a-z])bot@|copilot|claude|anthropic|openai|codex|devin|cursor|gemini|aider|sweep-ai|dependabot|renovate/i;
 export function machineAuthored(repo, base, head, env = process.env) {
-  const r = spawnSync('git', ['-C', repo, 'rev-list', '--format=%x1e%an%x00%ae%x00%cn%x00%ce%x00%B', `${base}..${head}`], { encoding: 'utf8', timeout: 30000, maxBuffer: 16 << 20, env: cleanGitEnv(env) });
+  // NUL ends every field and every commit (-z), and a commit message cannot hold one.
+  const r = spawnSync('git', ['-C', repo, 'log', '-z', '--format=%an%x00%ae%x00%cn%x00%ce%x00%B', `${base}..${head}`], { encoding: 'utf8', timeout: 30000, maxBuffer: 16 << 20, env: cleanGitEnv(env) });
   if (r.status !== 0) return { known: false, commits: [] };
+  const fields = r.stdout.split('\x00');
+  if (fields.at(-1) === '') fields.pop();
+  if (fields.length % 5 !== 0) return { known: false, commits: [] };
   const commits = [];
-  for (const chunk of r.stdout.split('\x1e').slice(1)) {
-    const [an, ae, cn, ce, body = ''] = chunk.split('\x00');
-    const trailers = body.split('\n').filter((l) => /^(co-authored-by|generated-by|assisted-by):/i.test(l.trim()));
+  for (let i = 0; i < fields.length; i += 5) {
+    const [an, ae, cn, ce, body = ''] = fields.slice(i, i + 5);
+    // Control characters are dropped first, so one placed before a trailer does not hide it.
+    const trailers = body.split('\n').map((l) => l.replace(/[\x00-\x1f\x7f]/g, '').trim()).filter((l) => /^(co-authored-by|generated-by|assisted-by):/i.test(l));
     const signals = [an, ae, cn, ce, ...trailers, ...(/generated with/i.test(body) ? ['generated with'] : [])].filter((x) => x && TOOL.test(x));
     if (signals.length) commits.push({ signals: [...new Set(signals.map((s) => s.trim().slice(0, 80)))] });
   }
@@ -101,13 +112,20 @@ export function checkEquivalence({ baseDir, headDir, allow, files = [], allowed
       problems.push(`${s.file} names head ${h ? h.slice(0, 12) : 'nothing'} and base ${b ? b.slice(0, 12) : 'nothing'}, which match no program this change edits`);
     }
   }
+  // Every statement that names a program is judged, and the program passes only if each does.
   const programs = changed.map((c) => {
-    const s = statements.find((x) => x.statement && subjectDigest(x.statement, 'head') === c.head && subjectDigest(x.statement, 'base') === c.base);
+    if (c.deleted) return { path: c.path, deleted: true, statement: null, ok: false, because: 'the change deletes this program, and no equivalence statement can show what its callers do now' };
+    const matching = statements.filter((x) => x.statement && subjectDigest(x.statement, 'head') === c.head && subjectDigest(x.statement, 'base') === c.base);
+    const via = c.via ? { via: c.via } : {};
+    if (!matching.length) return { path: c.path, ...via, statement: null, ok: false, because: c.via ? `no equivalence statement names this program, whose copybook${c.via.length > 1 ? 's' : ''} ${c.via.join(', ')} the change edits` : 'no equivalence statement names this change' };
+    const judged = matching.map((s) => judge(c, s));
+    return judged.find((j) => !j.ok) || judged[0];
+  });
+  function judge(c, s) {
     const via = c.via ? { via: c.via } : {};
-    if (!s) return { path: c.path, ...via, statement: null, ok: false, because: c.via ? `no equivalence statement names this program, whose copybook${c.via.length > 1 ? 's' : ''} ${c.via.join(', ')} the change edits` : 'no equivalence statement names this change' };
     const p = s.statement.predicate || {};
     const verdict = p.verdict || null;
-    const covered = p.coverage !== null && p.coverage !== undefined;
+    const covered = !!p.coverage && typeof p.coverage === 'object' && Array.isArray(p.coverage.unreached) && p.coverage.unreached.every((u) => typeof u === 'string');
     const ranOn = new Set(((p.closure && p.closure.head) || []).map((x) => x.sha256));
     const missing = (c.viaDigests || []).filter((d) => !ranOn.has(d));
     const because = !PASSING.has(verdict) ? `the statement's verdict is ${verdict}${Array.isArray(p.inconclusive) && p.inconclusive.length ? ` (${p.inconclusive.join('; ')})` : ''}`
@@ -115,9 +133,9 @@ export function checkEquivalence({ baseDir, headDir, allow, files = [], allowed
       : !covered ? 'the statement measured no coverage of the changed paragraphs, so it is inconclusive'
         : allowed && !s.signed ? `the statement is ${s.reason}`
           : null;
-    const unreached = covered && Array.isArray(p.coverage.unreached) ? p.coverage.unreached : [];
+    const unreached = covered ? p.coverage.unreached : [];
     return { path: c.path, ...via, statement: s.file, verdict, signed: s.signed, signer: s.signer, coverage: covered, ...(unreached.length ? { unreached } : {}), ok: because === null && !unreached.length, ...(because || unreached.length ? { because: because || `the inputs never reached ${unreached.join(', ')}` } : {}) };
-  });
+  }
   const authorship = mode === 'machineAuthored' && repo && base ? machineAuthored(repo, base, head || 'HEAD', env) : null;
   const required = mode === 'always' ? true : mode === 'machineAuthored' ? (authorship && authorship.known ? authorship.commits.length > 0 : null) : false;
   return { mode, required, ...(authorship ? { machineAuthored: authorship } : {}), programs, problems };

package/lib/evidence/journal.mjs

@@ -4,7 +4,7 @@
 import { randomBytes } from 'node:crypto';
 import { closeSync } from 'node:fs';
 import { join } from 'node:path';
-import { makeRecord, newChain } from './record.mjs';
+import { makeRecord, newChain, recordHash } from './record.mjs';
 import { EvidenceRefusal, createExclusive, lastLine, openAppend, prepareDir, syncClose, takeLock, writeAll } from './store.mjs';
 
 export const LEDGER = 'ledger.jsonl';
@@ -13,12 +13,21 @@ export const LOCK = 'ledger.lock';
 const isoNow = () => new Date().toISOString();
 const runStamp = (iso) => iso.replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
 
+// The record the next ledger line extends: one whose chain, seq and hash hold.
+function ledgerTip(line, path) {
+  let r = null;
+  try { r = JSON.parse(line); } catch { /* refused below */ }
+  const ok = r && typeof r === 'object' && /^[0-9a-f]{32}$/.test(r.chain) && Number.isInteger(r.seq) && r.seq >= 0 && typeof r.hash === 'string' && recordHash(r) === r.hash;
+  if (!ok) throw new EvidenceRefusal(`${path} ends in a line that is no ledger record; verify it before anything is appended`);
+  return r;
+}
+
 // Appends to the ledger; the caller holds the lock. A torn ledger is refused rather than extended.
 function appendLedger(dir, entries, now) {
   const path = join(dir, LEDGER);
   const { line, torn } = lastLine(path);
   if (torn) throw new EvidenceRefusal(`${path} ends in a partial line; verify it before anything is appended`);
-  let prev = line ? JSON.parse(line) : null;
+  let prev = line ? ledgerTip(line, path) : null;
   const fd = openAppend(path);
   try {
     const out = [];

package/lib/evidence/record.mjs

@@ -36,12 +36,14 @@ export const KINDS = {
   dd: { fields: { dd: str, event: (x) => ['open', 'close', 'end'].includes(x), mode: str, sha256: hex64, bytes: int }, required: ['dd', 'event'] },
   call: { fields: { program: str, from: str, sha256: hex64 }, required: ['program'] },
   abend: { fields: { code: str, file: str, line: int }, required: ['code'] },
+  step: { fields: { step: str, pgm: str, outcome: str }, required: ['step', 'pgm', 'outcome'] },
+  sink: { fields: { sink: str, file: str, line: int, marker: str, reached: (x) => typeof x === 'boolean' }, required: ['sink', 'file', 'line', 'marker', 'reached'] },
   genesis: { fields: { createdAt: str, rotatedFrom: any }, required: ['createdAt'] },
   run: { fields: { run: str, runChain: hex32, runLength: int, runTip: hex64 }, required: ['run', 'runChain', 'runLength', 'runTip'] },
   'lock-broken': { fields: { holderPid: int, ageMs: int }, required: ['holderPid', 'ageMs'] },
 };
 
-export const JOURNAL_KINDS = new Set(['open', 'input', 'finding', 'suppressed', 'baseline-write', 'witness', 'verdict', 'output', 'close', 'dd', 'call', 'abend']);
+export const JOURNAL_KINDS = new Set(['open', 'input', 'finding', 'suppressed', 'baseline-write', 'witness', 'verdict', 'output', 'close', 'dd', 'call', 'abend', 'step', 'sink']);
 export const LEDGER_KINDS = new Set(['genesis', 'run', 'lock-broken']);
 
 export const newChain = () => randomBytes(16).toString('hex');

package/lib/evidence/slsa.mjs

@@ -20,9 +20,24 @@ function named(path, root) {
   return abs.startsWith(r + sep) ? relative(r, abs).split(sep).join('/') : basename(abs);
 }
 
+// The compiler's arguments with every value replaced: an option keeps its name, a path or other value
+// becomes <value>. The digest of the arguments as run lets a holder of them check the statement.
+function redacted(argv) {
+  return argv.map((a) => {
+    const s = String(a);
+    if (!s.startsWith('-')) return '<value>';
+    const cut = s.search(/[=/\\]/);
+    return cut < 0 ? s : `${s.slice(0, cut)}${s[cut] === '=' ? '=' : ''}<value>`;
+  });
+}
+
 export function slsaStatement({ provenance, docBytes, root, artifacts = [], runId = null, runTip = null, builderId = null, startedOn = null, finishedOn = null }) {
   const subject = [{ name: 'build.json', digest: { sha256: sha256(docBytes) } },
-    ...artifacts.map((p) => ({ name: named(p, root), digest: { sha256: sha256(readFileSync(p)) } }))];
+    ...artifacts.map((p) => {
+      let bytes;
+      try { bytes = readFileSync(p); } catch (e) { throw new Error(`the artifact ${basename(p)} could not be read (${e.code || e.message})`); }
+      return { name: named(p, root), digest: { sha256: sha256(bytes) } };
+    })];
 
   const resolvedDependencies = (provenance.sources || []).map((s) => ({ uri: `file:${s.path}`, name: s.path, digest: { sha256: s.sha256 } }));
   if (provenance.compiler && provenance.compiler.sha256) {
@@ -36,7 +51,7 @@ export function slsaStatement({ provenance, docBytes, root, artifacts = [], runI
     revisions: provenance.revisions,
     policy: { sha256: provenance.policy.sha256, setBy: provenance.policy.setBy },
     ...(provenance.copylibs && provenance.copylibs.length ? { copylibs: provenance.copylibs.map((d) => basename(d)) } : {}),
-    ...(provenance.compiler && provenance.compiler.argv ? { compilerArguments: provenance.compiler.argv } : {}),
+    ...(provenance.compiler && provenance.compiler.argv ? { compilerArguments: redacted(provenance.compiler.argv), compilerArgumentsSha256: sha256(provenance.compiler.argv.map(String).join('\0')) } : {}),
   };
   const internalParameters = {
     tool: 'cobolwork', toolVersion: provenance.toolVersion, flowModel: provenance.flowModel,

package/lib/execution.mjs

@@ -0,0 +1,77 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Execution: whether the estate's own runs entered the paragraph a finding is in, from the
+// reports `ironwork run --coverage` writes. A finding in a paragraph a run entered is code the tests
+// exercise; one in a paragraph no run entered is code they never reach. See docs/spec/evidence.md §13.4 (execution coverage).
+import { readFileSync } from 'node:fs';
+import { basename, delimiter, join } from 'node:path';
+
+export function executionFeedPaths(opts = {}) {
+  if (opts.executionFeeds) return opts.executionFeeds;
+  return (process.env.COBOLWORK_EXECUTION || '').split(delimiter).filter(Boolean);
+}
+
+// One report: per PROGRAM-ID, each paragraph's source line, name and how often a run entered it.
+// Shape (ironwork's): { programs: [{ program, detail: [{ name, line, entered }] }] }.
+export function loadExecutionFeed(path) {
+  const feed = { file: basename(path), programs: new Map(), problem: null };
+  let doc;
+  try { doc = JSON.parse(readFileSync(path, 'utf8')); } catch (e) { feed.problem = e.code ? `could not be read (${e.code})` : `is not JSON (${e.message})`; return feed; }
+  if (!doc || !Array.isArray(doc.programs)) { feed.problem = 'holds no programs'; return feed; }
+  for (const p of doc.programs) {
+    const detail = Array.isArray(p?.detail) ? p.detail : null;
+    if (typeof p?.program !== 'string' || !detail || !detail.every((d) => typeof d?.name === 'string' && Number.isInteger(d.line) && Number.isInteger(d.entered) && d.entered >= 0)) {
+      feed.problem = `a program's paragraphs are not { name, line, entered }`;
+      return feed;
+    }
+    feed.programs.set(p.program.toUpperCase(), detail.map((d) => ({ name: d.name, line: d.line, entered: d.entered })));
+  }
+  return feed;
+}
+
+const PROGRAM_ID = /^.{0,6}[ \d]?\s*PROGRAM-ID\.?\s+['"]?([A-Za-z0-9#$@-]+)/;
+
+// The programs a source holds, by PROGRAM-ID and the line it starts on.
+function programsOf(text) {
+  const out = [];
+  text.split(/\r?\n/).forEach((line, i) => {
+    const m = PROGRAM_ID.exec(line);
+    if (m) out.push({ id: m[1].toUpperCase(), line: i + 1 });
+  });
+  return out;
+}
+
+// Each finding inside a paragraph of a covered program gets `executed: { paragraph, entered }`,
+// entered summed over the feeds. A finding before the program's first paragraph, or in a program
+// no feed covers, is left as it is.
+export function applyExecution(findings, root, feeds) {
+  const good = feeds.filter((f) => !f.problem);
+  const byExecution = { entered: 0, 'never-entered': 0 };
+  if (!good.length) return { byExecution: null };
+  const sources = new Map();
+  const programs = (path) => {
+    if (!sources.has(path)) {
+      let text = null;
+      try { text = readFileSync(join(root, path), 'utf8'); } catch { /* not a file the scan can read again */ }
+      sources.set(path, text === null ? [] : programsOf(text));
+    }
+    return sources.get(path);
+  };
+  for (const f of findings) {
+    if (!f.path || !Number.isInteger(f.line)) continue;
+    const owner = programs(f.path).filter((p) => p.line <= f.line && good.some((g) => g.programs.has(p.id))).at(-1);
+    if (!owner) continue;
+    let paragraph = null;
+    let entered = 0;
+    for (const g of good) {
+      const detail = g.programs.get(owner.id);
+      const at = detail?.filter((d) => d.line >= owner.line && d.line <= f.line).at(-1);
+      if (!at) continue;
+      paragraph = at.name;
+      entered += at.entered;
+    }
+    if (paragraph === null) continue;
+    f.executed = { paragraph, entered };
+    byExecution[entered > 0 ? 'entered' : 'never-entered']++;
+  }
+  return { byExecution };
+}

package/lib/explain.mjs

@@ -7,15 +7,16 @@ import { EVIDENCE } from './kernel/findings.mjs';
 import { REGISTRY } from './kernel/registry.mjs';
 import { WHO_ACTS } from './tui/model.mjs';
 import { verificationPlan } from './verify.mjs';
+import { cobolCard, statementCard } from './cards.mjs';
 
 const HIDDEN_RULES = new Set(Object.keys(REGISTRY.find((s) => s.name === 'hidden').rules));
 
 // The columns quoted: COBOL's indicator and program text, JCL's statement field, or the whole line.
 function codeArea(line, kind, format) {
-  if (kind === 'cobol' && format === 'fixed') return { text: line.slice(6, 72), seq: line.slice(0, 6), rest: line.slice(72) };
-  if (kind === 'cobol' && format === 'variable') return { text: line.slice(6, 250), seq: line.slice(0, 6), rest: line.slice(250) };
-  if (kind === 'jcl') return { text: line.slice(0, 72), seq: '', rest: line.slice(72) };
-  return { text: line, seq: '', rest: '' };
+  if (kind === 'jcl') { const c = statementCard(line); return { text: c.text, seq: '', rest: c.sequence }; }
+  if (kind !== 'cobol') return { text: line, seq: '', rest: '' };
+  const c = cobolCard(line, format);
+  return { text: c.indicator + c.text, seq: c.sequence, rest: c.ignored };
 }
 
 const NOTE = 'This packet contains source text from the files it names. A cobolwork report never does: '

package/lib/ftp.mjs

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { statementCard } from './cards.mjs';
+
+// IBM's batch FTP client takes the host and its options on PARM, or the host as the first line of
+// its input, reads subcommands from INPUT, and logs on from NETRC or from its input. CardDemo's job
+// gives the same lines on SYSIN, which is read here when there is no INPUT, though the page below
+// does not say the client falls back to it.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halu001/ftpreq.htm
+// A session is in cleartext unless something asks for TLS: -a TLS or -r TLS on PARM, or
+// SECURE_MECHANISM TLS in an in-stream SYSFTPD. A SYSFTPD naming a data set is configuration this
+// reader cannot see, so that step is not called cleartext either way.
+//   https://www.ibm.com/docs/en/zos/2.1.0?topic=commands-ftp-command
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halz002/ftp_cust_client_for_tls.htm
+// PUT, MPUT and APPEND send local files, and IBM's capitals - PUt, MPut, APpend, LCd - are the
+// shortest spellings. A local name in quotes is a whole data set name, //DD: names a DD of the
+// step, and anything else hangs off the local working directory, which only a quoted LCD fixes.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/ftpsubcmd.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/put.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/mput.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/append.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/lcd.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/dd_name_support.htm
+// Get and MGet copy remote files in, each to the local name given or, with none, to the foreign
+// file's own name, placed as a sent name is. SIte FILETYPE=JES makes the remote host submit what
+// is sent as a job and return job output to a Get; FILETYPE=SQL makes it run a query. Either holds
+// for the transfers after it, until another FILETYPE.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/get.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/mget.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/site.htm
+const FTP_SENDS = [[/^PUT?$/i, 'PUT'], [/^MP(?:UT?)?$/i, 'MPUT'], [/^AP(?:P(?:E(?:ND?)?)?)?$/i, 'APPEND']];
+
+// Every subcommand on the page above, in IBM's spelling.
+const FTP_SUBCOMMANDS = ['?', '!', 'ACCount', 'APpend', 'AUth', 'AScii', 'BIG5', 'BINary', 'BLock', 'CCc', 'CD',
+  'CDUp', 'CLEar', 'CLose', 'COMpress', 'CProtect', 'CWd', 'DEBug', 'DELEte', 'DELImit', 'DIr', 'DUMP', 'EBcdic',
+  'EUckanji', 'FEature', 'FIle', 'Get', 'GLob', 'HAngeul', 'HElp', 'Ibmkanji', 'JIS78kj', 'JIS83kj', 'Ksc5601',
+  'LANGuage', 'LCd', 'LOCSIte', 'LOCSTat', 'LMkdir', 'LPwd', 'LS', 'MDelete', 'MGet', 'MKdir', 'MKFifo', 'MOde',
+  'MPut', 'MVSGet', 'MVSPut', 'NOop', 'Open', 'PAss', 'PRIvate', 'PROMpt', 'PROTect', 'PROXy', 'PUt', 'PWd', 'QUIt',
+  'QUOte', 'RECord', 'REName', 'REStart', 'RMdir', 'SAfe', 'SChinese', 'SENDPort', 'SENDSite', 'SIte', 'SJiskanji',
+  'SRestart', 'STAtus', 'STREam', 'STRucture', 'SUnique', 'SYstem', 'TChinese', 'TSO', 'TYpe', 'UCs2', 'User', 'Verbose']
+  .map((name) => ({ full: name.toUpperCase(), min: name.match(/^[^a-z]*/)[0].length }));
+const subcommandOf = (word) => {
+  const w = word.toUpperCase();
+  const hit = FTP_SUBCOMMANDS.find((c) => w.length >= c.min && c.full.startsWith(w));
+  return hit ? hit.full : null;
+};
+// A value that stands for a password rather than being one: a symbolic, a mask, or the word itself.
+const PLACEHOLDER = /^(&.*|\*+|X+|\?+|<.*>|\{.*\}|PASSWORD|PASSWD|PWD|PASS|SECRET)$/i;
+
+// The session one FTP step runs, from its DDs grouped by name (ddGroups in lib/utilities.mjs).
+export function ftpSession(step, dds) {
+  let host = null, tls = null;
+  const words = (step.parm || '').split('(')[0].trim().split(/\s+/).filter(Boolean);
+  for (let i = 0; i < words.length; i++) {
+    if (/^-[ar]$/i.test(words[i])) {
+      const mechanism = (words[++i] || '').toUpperCase();
+      if (mechanism === 'TLS') tls = 'asked';
+      else if (mechanism === 'NEVER' && !tls) tls = 'never';
+    } else if (!words[i].startsWith('-') && !host) host = words[i];
+  }
+  let configUnread = null;
+  for (const dd of dds.get('SYSFTPD') || []) {
+    if (dd.inStream) { if (tls !== 'never' && dd.inStream.some((l) => /^\s*SECURE_MECHANISM\s+TLS\b/i.test(l.text))) tls = 'asked'; }
+    else if (dd.dsn && !configUnread) configUnread = dd.dsn;
+  }
+
+  const input = dds.get('INPUT') || dds.get('SYSIN') || [];
+  const hostLine = !host && input[0]?.inStream;
+  let inputUnread = null, lcd = null, unixDir = false;
+  const sends = [], gets = [];
+  let filetype = null;
+  // With no NETRC to answer for it, the client asks for a user ID and a password and in batch reads
+  // them from its input after the host: "user password" on one line, or each on its own. A first
+  // line that is a subcommand means something else answered, a NETRC the client found for itself.
+  // USER and PASS give the same answers later in a session.
+  let expect = dds.has('NETRC') ? null : 'logon', user = null;
+  const passwords = [];
+  const password = (dd, line, value) => { if (!PLACEHOLDER.test(value)) passwords.push({ dd, line, user }); };
+  const local = (name) => {
+    const dd = /^\/\/DD:([A-Z$#@][A-Z0-9$#@]{0,7})$/i.exec(name);
+    if (dd) return { dd: dd[1].toUpperCase(), dsns: (dds.get(dd[1].toUpperCase()) || []).map((d) => d.dsn).filter(Boolean) };
+    const quoted = /^'(.+)'$/.exec(name);
+    if (quoted) return { dsns: [quoted[1].toUpperCase()] };
+    if (name.startsWith('/') || unixDir) return { dsns: [] };
+    return lcd ? { dsns: [`${lcd}.${name.toUpperCase()}`] } : { dsns: [], relative: name };
+  };
+  for (const dd of input) {
+    if (!dd.inStream) { if (dd.dsn && !inputUnread) inputUnread = dd.dsn; continue; }
+    for (const { line, text } of dd.inStream) {
+      const [verb, ...args] = statementCard(text).text.trim().split(/\s+/);
+      if (!verb) continue;
+      if (hostLine && !host) { host = verb; continue; }
+      const sub = subcommandOf(verb);
+      if (expect && !sub) {
+        if (expect === 'logon') user = verb;
+        if (expect === 'password') password(dd, line, verb);
+        else if (args[0]) password(dd, line, args[0]);
+        expect = expect === 'logon' && !args[0] ? 'password' : null;
+        continue;
+      }
+      expect = null;
+      if (sub === 'USER' && args[0]) {
+        user = args[0];
+        if (args[1]) password(dd, line, args[1]); else expect = 'password';
+        continue;
+      }
+      if (sub === 'PASS' && args[0]) { password(dd, line, args[0]); continue; }
+      if (/^LCD?$/i.test(verb) && args[0]) {
+        const quoted = /^'(.+)'$/.exec(args[0]);
+        unixDir = args[0].startsWith('/');
+        lcd = quoted ? quoted[1].toUpperCase() : lcd && !unixDir ? `${lcd}.${args[0].toUpperCase()}` : null;
+        continue;
+      }
+      const site = sub === 'SITE' ? args : sub === 'QUOTE' && subcommandOf(args[0] || '') === 'SITE' ? args.slice(1) : null;
+      if (site) {
+        for (const a of site) { const t = /^FILE(?:T(?:Y(?:P(?:E)?)?)?)?=(\w+)$/i.exec(a); if (t) filetype = t[1].toUpperCase() === 'SEQ' ? null : t[1].toUpperCase(); }
+        continue;
+      }
+      const typed = filetype ? { filetype } : {};
+      if (sub === 'GET' && args[0]) {
+        const to = args[1] && !args[1].startsWith('(') ? args[1] : args[0];
+        gets.push({ line, verb: 'GET', name: args[0], ...typed, ...local(to) });
+        continue;
+      }
+      if (sub === 'MGET') {
+        for (const name of args.filter((a) => !a.startsWith('(') && !/[*?]/.test(a))) gets.push({ line, verb: 'MGET', name, ...typed, ...local(name) });
+        continue;
+      }
+      const sent = FTP_SENDS.find(([re]) => re.test(verb));
+      if (!sent) continue;
+      const foreign = sent[1] !== 'MPUT' && args[1] && !args[1].startsWith('(') ? args[1] : null;
+      for (const name of sent[1] === 'MPUT' ? args : args.slice(0, 1)) sends.push({ line, verb: sent[1], name, foreign: foreign || name, ...typed, ...local(name) });
+    }
+  }
+  const cleartext = tls === 'never' || (!tls && !configUnread);
+  return { host, cleartext, configUnread, inputUnread, logon: dds.has('NETRC') ? 'NETRC' : null, sends, gets, passwords, input };
+}

package/lib/inventory.mjs

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { dirname } from 'node:path';
-import { parseSource, buildFileIndex, detectFormat } from './parser.mjs';
-import { inScope, isCopybook, isJcl, isProgram, readSource, relPath } from './sources.mjs';
+import { detectFormat } from './parser.mjs';
+import { inScope, isCopybook, isJcl, isProgram, relPath } from './sources.mjs';
 import { SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 import { treeFor } from './kernel/source-tree.mjs';
 
@@ -36,19 +35,19 @@ export function inventory(root, opts = {}) {
     out.summary.programFiles++;
     let src;
     try {
-      const s = readSource(f);
+      const s = tree.text(f);
       src = s.text;
-      if (s.encoding === 'ebcdic') { out.summary.filesEbcdic++; out.ebcdic.push(relPath(root, f)); }
-    } catch (e) { out.summary.filesUnreadable++; out.unreadable.push(`${relPath(root, f)}: ${e.code || e.name}`); continue; }
+      if (s.encoding === 'ebcdic') { out.summary.filesEbcdic++; out.ebcdic.push(tree.rel(f)); }
+    } catch (e) { out.summary.filesUnreadable++; out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`); continue; }
     // A file with a program extension and no program in it is usually a copybook named .cbl; it is
     // counted so the difference between program files and programs read is accounted for.
     if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) { out.summary.notPrograms++; continue; }
     let res;
     try {
-      res = parseSource(src, f, { format: 'auto', includeDirs: idx.copyDirs, fileIndex: idx.index, mainDir: dirname(f), copyFormat: 'auto', systemDirs: opts.systemDirs || [] });
+      res = tree.parse(f, src);
     } catch (e) {
       out.summary.filesUnreadable++;
-      out.unreadable.push(`${relPath(root, f)}: ${e.code || e.name}`);
+      out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`);
       continue;
     }
     out.summary.filesScanned++;
@@ -60,7 +59,7 @@ export function inventory(root, opts = {}) {
     if (/EXEC\s+DLI/i.test(src)) out.summary.withDli++;
     for (const c of res.copies) {
       if (c.status === 'resolved') out.summary.copiesResolved++;
-      else if (String(c.status).startsWith('refused')) { out.summary.copiesRefused++; out.refusedCopies.push({ name: c.name, from: relPath(root, c.file), line: c.line, why: c.status }); }
+      else if (String(c.status).startsWith('refused')) { out.summary.copiesRefused++; out.refusedCopies.push({ name: c.name, from: tree.rel(c.file), line: c.line, why: c.status }); }
       // Before the system names, which a copybook in the tree is free to take.
       else if (c.status === 'expansion-limit') out.summary.copiesOverLimit = (out.summary.copiesOverLimit || 0) + 1;
       else if (c.status === 'system' || SYSTEM_COPY.test(c.name)) out.summary.copiesSystem++;

package/lib/ironwork.mjs

@@ -7,7 +7,7 @@ import { tmpdir } from 'node:os';
 import { resolve } from 'node:path';
 import { directoryTree } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
-import { buildFileIndex, VERBS } from './parser.mjs';
+import { buildFileIndex } from './parser.mjs';
 import { isProgram, relPath } from './sources.mjs';
 import { EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS } from './words.mjs';
 
@@ -20,15 +20,13 @@ const NOT_AN_ERROR = /^(?:informational:|.*?: warning: )/;
 const PER_PROGRAM_MS = 60000;
 const MAX_LISTED = 10;
 
-// ironwork refuses by name what it does not model yet; that says nothing about the program. Where its
-// parser stops at a statement it does not read (ENTRY, ALTER), or at a name the CICS, DL/I or SQL
-// translator declares, it says so without the name, so both are read as its gap, not the program's.
+// ironwork refuses by name what it does not model yet; that says nothing about the program. A name
+// the CICS, DL/I or SQL translator declares it reports as undefined, without saying so, and that is
+// read as its gap, not the program's.
 const NOT_SUPPORTED = /\bnot supported\b/i;
 const TRANSLATOR_NAMES = [EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS];
 function notModelled(message) {
   if (NOT_SUPPORTED.test(message)) return true;
-  const verb = /^a statement, found ([A-Z][A-Z-]*)$/.exec(message);
-  if (verb) return VERBS.has(verb[1]);
   const name = /^([A-Z][A-Z0-9-]*) is not defined$/.exec(message);
   return !!name && TRANSLATOR_NAMES.some((s) => s.has(name[1]));
 }

package/lib/jcl.mjs

@@ -12,17 +12,16 @@
 // output as `unreadable`, never dropped, because a step nobody parsed is not a step that is not
 // there.
 import { readSource } from './sources.mjs';
-import { copiesOf } from './utilities.mjs';
+import { copiesOf, tsoCommands } from './utilities.mjs';
+import { statementCard, parseOperands } from './cards.mjs';
+
+export { splitOperands, parseOperands } from './cards.mjs';
 
 // The operations a statement may carry. Anything else in the operation field is a statement this
 // reader does not know, which is a diagnostic rather than a silent skip.
 export const OPERATIONS = new Set(['JOB', 'EXEC', 'DD', 'PROC', 'PEND', 'SET', 'IF', 'THEN',
   'ELSE', 'ENDIF', 'INCLUDE', 'JCLLIB', 'OUTPUT', 'CNTL', 'ENDCNTL', 'XMIT', 'COMMAND', 'NOTIFY', 'EXPORT']);
 
-// Columns 73 to 80 are the sequence area. They are not the statement, and a payload hidden there
-// is the hidden-content rules' business, not this reader's.
-const STATEMENT_COLUMNS = 72;
-
 // A continued operand resumes in these columns. Outside them it is not a continuation, whatever
 // the previous line ended with.
 const CONTINUE_FROM = 4;
@@ -41,53 +40,6 @@ const SYSTEM_SYMBOLS = new Set(`SYSUID SYSALVL SYSCLONE SYSNAME SYSOSLVL SYSPLEX
   YYMMDD LYYMMDD HHMMSS LHHMMSS DAY HR MIN SEC JDAY MON YR2 YR4 WDAY
   LDATE LDAY LHR LMIN LSEC LJDAY LMON LYR2 LYR4 LWDAY LTIME JOBNAME DS SEQ DATE TIME`.split(/\s+/));
 
-// Splits an operand field on commas that are not inside parentheses or quotes. JCL nests both, and
-// a naive split on comma turns DISP=(NEW,CATLG,DELETE) into three operands.
-export function splitOperands(s) {
-  const out = [];
-  let depth = 0, quoted = false, start = 0;
-  for (let i = 0; i < s.length; i++) {
-    const c = s[i];
-    if (quoted) { if (c === "'") { if (s[i + 1] === "'") i++; else quoted = false; } continue; }
-    if (c === "'") quoted = true;
-    else if (c === '(') depth++;
-    else if (c === ')') depth = Math.max(0, depth - 1);
-    else if (c === ',' && depth === 0) { out.push(s.slice(start, i)); start = i + 1; }
-  }
-  out.push(s.slice(start));
-  return out.filter((x, i, a) => x.length || i < a.length - 1);
-}
-
-// Operands are positional until the first KEY=VALUE, and keyword after it. Both forms matter:
-// EXEC takes its procedure name positionally, and everything interesting on DD is a keyword.
-export function parseOperands(field) {
-  const positional = [];
-  const keywords = new Map();
-  for (const raw of splitOperands(field)) {
-    const part = raw.trim();
-    if (!part) continue;
-    const eq = keywordSplit(part);
-    if (eq < 0) positional.push(part);
-    else keywords.set(part.slice(0, eq).toUpperCase(), part.slice(eq + 1));
-  }
-  return { positional, keywords };
-}
-
-// The first '=' that is not inside parentheses or quotes. DCB=(RECFM=FB,LRECL=80) is one keyword
-// whose value happens to contain more of them.
-function keywordSplit(part) {
-  let depth = 0, quoted = false;
-  for (let i = 0; i < part.length; i++) {
-    const c = part[i];
-    if (quoted) { if (c === "'") { if (part[i + 1] === "'") i++; else quoted = false; } continue; }
-    if (c === "'") quoted = true;
-    else if (c === '(') depth++;
-    else if (c === ')') depth = Math.max(0, depth - 1);
-    else if (c === '=' && depth === 0) return i;
-  }
-  return -1;
-}
-
 // Symbolic substitution. &NAME ends at a non-name character, and a trailing dot is a separator
 // that is consumed rather than kept: &PREFIX..DATA resolves to <prefix>.DATA.
 export function substitute(text, symbols) {
@@ -146,8 +98,10 @@ export function foldStatements(src) {
     const line = i + 1;
     const raw = phys[i];
     if (raw.length > 80) diags.push({ sev: 'warn', line, text: 'line is longer than 80 columns' });
-    const text = raw.slice(0, STATEMENT_COLUMNS).replace(/\s+$/, '');
-    const sequence = raw.slice(STATEMENT_COLUMNS);
+    // The sequence area is not the statement; a payload hidden there is the hidden-content rules' business.
+    const card = statementCard(raw);
+    const text = card.text.replace(/\s+$/, '');
+    const sequence = card.sequence;
 
     if (stream) {
       // A custom DLM ends the stream and nothing else does. The default ends at /* or at the next
@@ -402,6 +356,27 @@ export function parseJcl(src, file, opts = {}) {
     else diags.push({ sev: 'warn', line: 0, text: `symbolic parameter &${s} has no value, so the operands using it were read unsubstituted` });
   }
 
+  // A TSO batch step's ALLOCATE commands define DDs its JCL does not show. Each is kept as a DD of the
+  // step, marked `allocated`, with the datasets of a list concatenated under it as JCL would.
+  for (const s of steps) {
+    const tso = tsoCommands(s);
+    for (const n of tso.notes) diags.push({ sev: 'info', ...n });
+    for (const a of tso.allocations) {
+      const ends = a.datasets.length ? a.datasets : [{ written: null, dsn: null }];
+      ends.forEach((d, i) => {
+        const dd = {
+          name: i === 0 ? a.dd : null, procStep: null, dsn: d.dsn, rawDsn: d.written, referback: null,
+          disp: a.status, access: accessOf(a.status), temporary: false, sysout: a.sysout,
+          inStream: null, dlm: null, concatenated: i > 0, keywords: new Map(), line: a.line, endLine: a.line,
+          step: s.name, allocated: true,
+        };
+        dds.push(dd);
+        s.dds.push(dd);
+        stepOf.set(dd, s);
+      });
+    }
+  }
+
   // What each step's program copies, from lib/utilities.mjs. A program that table does not know
   // copies nothing here, which is not the same as copying nothing.
   const copies = [];