@portll/cobolwork 0.2.117 → 0.2.140

package/lib/dataflow.mjs

@@ -4,6 +4,9 @@ import { parseBms, symbolicNames } from './bms.mjs';
 import { dirname, join } from 'node:path';
 import { parseSource, buildFileIndex } from './parser.mjs';
 import { parseJcl } from './jcl.mjs';
+import { holdsDecimal } from './layout.mjs';
+import { hostVariablesIn } from './embedded-sql.mjs';
+import { tsoCommands } from './utilities.mjs';
 import { parseCsd, ddOfQueue } from './csd.mjs';
 import { loadSite } from './site.mjs';
 import { ssrangeAbends } from './options.mjs';
@@ -68,6 +71,7 @@ STANDARD-DEVIATION RANDOM SQRT FACTORIAL LOG LOG10 EXP EXP10 PI E ACOS ASIN ATAN
 SECONDS-PAST-MIDNIGHT`.split(/\s+/));
 const computes = (st) => ARITHMETIC.has(st.verb) || st.counts === true || (st.fns || []).some((f) => NUMERIC_FUNCTIONS.has(f));
 const computedOnRoute = (state) => { for (let s = state; s; s = s.prev) if (s.why && s.why.computes) return true; return false; };
+const typedOnRoute = (state) => { for (let s = state; s; s = s.prev) if (s.node.typed) return true; return false; };
 
 // Every literal a program holds that could be a program name: its VALUE clauses and the literals its
 // statements and commands use. A menu that XCTLs through a table of names starts every one of them.
@@ -89,15 +93,6 @@ function lastOption(name, list) {
   return null;
 }
 
-// Whether an item's bytes are read as decimal digits: zoned (DISPLAY with a numeric picture) or
-// packed. A group, an edited picture, binary and floating point are not.
-function holdsDecimal(it) {
-  if ((it.children || []).some((c) => c.level !== 88)) return false;
-  const usage = String(it.effectiveUsage || 'DISPLAY').replace('COMPUTATIONAL', 'COMP');
-  if (usage === 'COMP-3' || usage === 'PACKED-DECIMAL' || usage === 'COMP-6') return true;
-  return usage === 'DISPLAY' && !!it.picture && /9/.test(it.picture) && /^[S9VP()0-9]+$/i.test(it.picture);
-}
-
 const RELATION = new Set(['>', '<', '>=', '<=', '=', '<>']);
 const RELATION_WORDS = new Set(['GREATER', 'LESS', 'EQUAL']);
 const FILLER_WORDS = new Set(['IS', 'NOT', 'THAN', 'TO', 'OR', 'GREATER', 'LESS', 'EQUAL', 'OF', 'IN']);
@@ -178,27 +173,6 @@ function execOptions(exec) {
   return { opts, words };
 }
 
-function sqlHostVars(exec) {
-  const out = [];
-  for (let i = 0; i < exec.toks.length; i++) {
-    const t = exec.toks[i];
-    if (t.t === 'op' && t.v === ':' && exec.toks[i + 1] && exec.toks[i + 1].t === 'word') out.push({ tok: exec.toks[i + 1], at: i + 1 });
-  }
-  return out;
-}
-
-// The host variables a SELECT or FETCH fills are the ones in its INTO list, which ends at FROM.
-// Taking every variable at or after the INTO line made the input parameters in a WHERE clause read
-// as database values — and they are whatever the caller sent, under a rule that says data at rest.
-function sqlIntoRange(exec) {
-  const words = exec.toks.map(t => (t.t === 'word' ? t.u : ''));
-  const into = words.indexOf('INTO');
-  if (into < 0) return null;
-  let end = exec.toks.length;
-  for (let i = into + 1; i < words.length; i++) if (words[i] === 'FROM') { end = i; break; }
-  return [into, end];
-}
-
 export function analyze(root, opts = {}) {
   const repos = opts.repos || [''];
   const stats = { files: 0, programs: 0, edges: 0, nodes: 0, threw: 0, overBudget: 0, unreadable: 0, ebcdic: 0,
@@ -457,11 +431,14 @@ export function analyze(root, opts = {}) {
           usesOf.get(at.name).push(rec);
         }
         const offset = x.offset || 0;
+        // A constant length past the start, or a constant start before the length, takes that many bytes off the top.
+        const base = limitOf(host, table);
+        const limit = kind === 'reference-modification' && base != null && x.span ? base - x.span + 1 : base;
         const use = `${key}|${point ?? `s${st.at}`}|${offset}`;
         if (indexed.has(use)) continue;
         indexed.add(use);
         at.sinks.push({
-          kind, onlyFrom: FROM_OUTSIDE, file: st.file, line: st.line, point, group: `${pk}|${key}`, limit: limitOf(host, table), ...(offset ? { offset } : {}), ...(offset > 0 && idx && unsignedWhole(idx) ? { nonNeg: true } : {}), ...(ssrange ? { ssrange } : {}),
+          kind, onlyFrom: FROM_OUTSIDE, file: st.file, line: st.line, point, group: `${pk}|${key}`, limit, ...(offset ? { offset } : {}), ...(offset > 0 && idx && unsignedWhole(idx) ? { nonNeg: true } : {}), ...(ssrange ? { ssrange } : {}),
           detail: kind === 'subscript'
             ? `${at.name} subscripts ${host.name}, a table of ${table.occurs}`
             : `${at.name} sets the ${x.kind === 'refmod-offset' ? 'start' : 'length'} of a reference to ${host.name}, which is ${host.size} bytes`,
@@ -724,14 +701,16 @@ export function analyze(root, opts = {}) {
       walk(record);
       for (const field of map.fields) {
         if (!field.name) continue;
-        const marks = ['PROT', 'ASKIP', 'DRK', 'NUM'].filter((a) => field.effective.has(a));
-        if (!marks.length) continue;
         const iname = symbolicNames(map, field).find((n) => n.suffix === 'I');
         const item = iname && within.get(iname.name);
         if (!item) continue;
+        const marks = ['PROT', 'ASKIP', 'DRK', 'NUM'].filter((a) => field.effective.has(a));
+        const typed = !marks.includes('PROT') && !marks.includes('ASKIP');
+        if (typed) nodeOf(item).typed = true;
+        if (!marks.length) continue;
         const n = nodeOf(item);
         n.screen = { item: item.name, field: field.name, map: map.name, mapset: mapset.name, marks, declared: !!field.attrb };
-        if (!marks.includes('PROT') && !marks.includes('ASKIP')) continue;
+        if (typed) continue;
         stats.protectedFields++;
         const back = field.effective.has('FSET') ? ' with FSET' : '';
         n.sources.push({ kind: 'cics-protected-field', onlyTo: ['record-key', 'record-update'], ...at,
@@ -921,15 +900,14 @@ export function analyze(root, opts = {}) {
         }
       }
       if (e.kind === 'SQL') {
-        // The parser's host variables name the item a qualified :G.F resolves to, where the first
-        // word after the colon would name the group G and so every field in it.
-        const hv = e.hostVariables || sqlHostVars(e);
+        // A host variable names the item a qualified :G.F resolves to; the statement fills the ones
+        // Db2 writes, and those hold what the database held.
+        const hv = e.hostVariables || hostVariablesIn(e.toks);
         const verb = words[0];
-        const range = e.hostVariables ? null : ['SELECT', 'FETCH'].includes(verb) && sqlIntoRange(e);
         for (const h of hv) {
-          if (e.hostVariables ? !h.written : !range || h.at < range[0] || h.at > range[1]) continue;
+          if (!h.written) continue;
           const n = nodeOfToken(h.tok);
-          if (n) n.sources.push({ kind: 'database', ...at, detail: `EXEC SQL ${verb} INTO host variable` });
+          if (n) n.sources.push({ kind: 'database', ...at, detail: `EXEC SQL ${verb} ${verb === 'SELECT' || verb === 'FETCH' ? 'INTO' : 'writes'} host variable` });
         }
         // The location CONNECT TO or SET CONNECTION names, where a host variable holds it; what
         // follows USER and USING is the credential, not the target.
@@ -1123,59 +1101,61 @@ export function analyze(root, opts = {}) {
         for (const dd of step.dds) if (dd.name && dd.name.toUpperCase() === 'SYSIN' && dd.inStream) addCsd(dd.inStream.map((l) => l.text).join('\n'), f, dd.inStream[0].line - 1);
       }
       jobSteps.push({ file: f, job: job.jobs[0]?.name || null, step: step.name, line: step.line, pgm: step.pgm.toUpperCase() });
-      // A TSO batch step starts a program by name in its commands: DSN ... RUN PROGRAM(X).
-      if (/^IKJEFT(01|1A|1B)$/i.test(step.pgm)) {
-        for (const dd of step.dds) for (const l of dd.inStream || []) {
-          const run = /\bRUN\s+PROGRAM\s*\(\s*([A-Z0-9$#@]{1,8})\s*\)/i.exec(l.text);
-          if (run) jobSteps.push({ file: f, job: job.jobs[0]?.name || null, step: step.name, line: l.line, pgm: run[1].toUpperCase() });
-        }
+      // A TSO batch step starts programs by name in its commands, TSO CALL and DSN RUN PROGRAM, and
+      // each is handed the step's DDs and its own parameter as EXEC PGM= would be.
+      const runs = [{ pgm: step.pgm.toUpperCase(), parm: step.parm, line: step.line, how: `PARM= on step ${step.name || '(unnamed)'} of ${rel(f)}` }];
+      for (const r of tsoCommands(step).runs) {
+        jobSteps.push({ file: f, job: job.jobs[0]?.name || null, step: step.name, line: r.line, pgm: r.program });
+        runs.push({ pgm: r.program, parm: r.parm, line: r.line, how: `the parameter ${r.via} passes in step ${step.name || '(unnamed)'} of ${rel(f)}` });
       }
-      const holders = allById.get(step.pgm.toUpperCase()) || [];
-      if (!holders.length) continue;            // a system utility, or a program not in this tree
-      stats.jclStepsResolved++;
-      // A program id held several times has no caller file to choose by, so each holder is a candidate.
-      const amb = holders.length > 1 ? `, any of ${holders.length} programs named ${step.pgm}` : '';
-      for (const callee of holders) {
+      for (const run of runs) {
+        const holders = allById.get(run.pgm) || [];
+        if (!holders.length) continue;            // a system utility, or a program not in this tree
+        stats.jclStepsResolved++;
+        // A program id held several times has no caller file to choose by, so each holder is a candidate.
+        const amb = holders.length > 1 ? `, any of ${holders.length} programs named ${run.pgm}` : '';
+        for (const callee of holders) {
 
-        // PARM arrives in the first PROCEDURE DIVISION USING item: on z/OS a halfword length
-        // followed by the text. A program with no USING cannot receive one, and saying it does
-        // would be a path nobody could follow.
-        if (step.parm !== null && callee.params && callee.params[0] && callee.params[0].node) {
-          callee.params[0].node.sources.push({
-            kind: 'jcl-parm', file: f, line: step.line,
-            detail: `PARM= on step ${step.name || '(unnamed)'} of ${rel(f)}, which runs ${step.pgm}${amb}`,
-          });
-          stats.jclCrossings++;
-        }
+          // PARM arrives in the first PROCEDURE DIVISION USING item: on z/OS a halfword length
+          // followed by the text. A program with no USING cannot receive one, and saying it does
+          // would be a path nobody could follow.
+          if (run.parm !== null && callee.params && callee.params[0] && callee.params[0].node) {
+            callee.params[0].node.sources.push({
+              kind: 'jcl-parm', file: f, line: run.line,
+              detail: `${run.how}, which runs ${run.pgm}${amb}`,
+            });
+            stats.jclCrossings++;
+          }
 
-        // In-stream data reaches whatever record the program reads from that DD. The COBOL says
-        // ASSIGN TO SYSIN and the job says //SYSIN DD *; neither half names the other, and the
-        // join is the whole point of reading both.
-        for (const dd of step.dds) {
-          // The same join in the other direction: what the program writes through SELECT ... ASSIGN
-          // TO a DD the job sends to the internal reader is submitted as a job.
-          if (dd.name && dd.sysout && /\bINTRDR\b/i.test(dd.sysout)) {
+          // In-stream data reaches whatever record the program reads from that DD. The COBOL says
+          // ASSIGN TO SYSIN and the job says //SYSIN DD *; neither half names the other, and the
+          // join is the whole point of reading both.
+          for (const dd of step.dds) {
+            // The same join in the other direction: what the program writes through SELECT ... ASSIGN
+            // TO a DD the job sends to the internal reader is submitted as a job.
+            if (dd.name && dd.sysout && /\bINTRDR\b/i.test(dd.sysout)) {
+              for (const m of callee.ddFiles || []) {
+                if (m.dd !== dd.name.toUpperCase()) continue;
+                for (const n of m.nodes) {
+                  n.sinks.push({ kind: 'internal-reader', file: m.file, line: m.line,
+                    detail: `records written through SELECT ${m.select} to //${dd.name}, which step ${step.name || '(unnamed)'} of ${rel(f)} sends to the internal reader${amb}` });
+                }
+              }
+            }
+            if (!dd.inStream || !dd.inStream.length || !dd.name) continue;
             for (const m of callee.ddFiles || []) {
               if (m.dd !== dd.name.toUpperCase()) continue;
               for (const n of m.nodes) {
-                n.sinks.push({ kind: 'internal-reader', file: m.file, line: m.line,
-                  detail: `records written through SELECT ${m.select} to //${dd.name}, which step ${step.name || '(unnamed)'} of ${rel(f)} sends to the internal reader${amb}` });
+                n.sources.push({
+                  kind: 'jcl-instream', file: f, line: dd.line,
+                  detail: `${dd.inStream.length} lines of in-stream data on //${dd.name} in step ${step.name || '(unnamed)'}, read through SELECT ${m.select}${amb}`,
+                });
+                stats.jclCrossings++;
               }
             }
-          }
-          if (!dd.inStream || !dd.inStream.length || !dd.name) continue;
-          for (const m of callee.ddFiles || []) {
-            if (m.dd !== dd.name.toUpperCase()) continue;
-            for (const n of m.nodes) {
-              n.sources.push({
-                kind: 'jcl-instream', file: f, line: dd.line,
-                detail: `${dd.inStream.length} lines of in-stream data on //${dd.name} in step ${step.name || '(unnamed)'}, read through SELECT ${m.select}${amb}`,
-              });
-              stats.jclCrossings++;
-            }
-          }
 
-      }
+          }
+        }
       }
     }
   }
@@ -1494,6 +1474,9 @@ export function analyze(root, opts = {}) {
           // arithmetic or a numeric function carries no bad bytes to the next one. Only this route is
           // dropped: a later one that copies the bytes still reports.
           if (sink.kind === 'arithmetic' && computedOnRoute(state)) continue;
+          // A protected field's value carried through a field the terminal may type into reaches the
+          // key as typed input, which the user could have entered anyway: the protection was no control.
+          if (src.kind === 'cics-protected-field' && typedOnRoute(state)) continue;
           seen.add(key);
           // The ends of a long path are what a reader uses: where the value came from, and what it
           // reached. Keeping every hop of a thousand-hop chain, for a thousand findings, is the

package/lib/embedded-sql.mjs

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// EXEC SQL as Db2 reads it, whichever language embeds it: a statement's words, host variables and
+// literals, and which host variables it reads and which it writes.
+// The words, host variables and literals of a statement. A host variable is :NAME, qualified as
+// :GROUP.NAME, and may carry an indicator variable, :NAME:IND or :NAME INDICATOR :IND.
+export function sqlTokens(sql) {
+  const toks = [];
+  const re = /'(?:[^']|'')*'|"(?:[^"]|"")*"|:\s*[A-Z0-9_$#@-]+(?:\.[A-Z0-9_$#@-]+)*|[A-Z0-9_$#@-]+|[(),=<>+*/;.]/gi;
+  for (const m of sql.matchAll(re)) {
+    const t = m[0];
+    if (t.startsWith(':')) toks.push({ host: t.slice(1).trim().toUpperCase() });
+    else if (/^['"]/.test(t)) toks.push({ lit: t });
+    else toks.push({ word: t.toUpperCase() });
+  }
+  return toks;
+}
+
+// Clauses that end an INTO list.
+const AFTER_INTO = new Set(['FROM', 'WHERE', 'GROUP', 'HAVING', 'ORDER', 'FETCH', 'FOR', 'WITH', 'OPTIMIZE', 'QUERYNO', 'SKIP', 'UNION', 'USING', 'VALUES']);
+
+// Which host variables a statement reads and which it writes, as Db2's SQL reference gives them. A
+// variable both read and written is in both lists. INTO a host variable writes it (INSERT INTO and
+// MERGE INTO name a table); so do SET's targets, GET DIAGNOSTICS' and ASSOCIATE LOCATORS' list. A
+// procedure's argument that is a lone variable may be IN, OUT or INOUT, which only the server knows.
+export function sqlRoles(toks) {
+  const sending = [];
+  const receiving = [];
+  const read = (h) => { if (!sending.includes(h)) sending.push(h); };
+  const write = (h) => { if (!receiving.includes(h)) receiving.push(h); };
+  const verb = toks.find((t) => t.word)?.word;
+  if (verb === 'CALL') return callRoles(toks, read, write), { sending, receiving };
+  // DESCRIBE and PREPARE read the SQLDA's SQLN and write the rest of it.
+  const intoBoth = verb === 'DESCRIBE' || verb === 'PREPARE';
+  let into = false;
+  let assigning = verb === 'SET';
+  let depth = 0;
+  for (let i = 0; i < toks.length; i++) {
+    const t = toks[i];
+    const next = toks[i + 1];
+    if (t.word === '(') depth++;
+    else if (t.word === ')') depth--;
+    else if (verb === 'SET' && t.word === '=' && depth === 0) assigning = false;
+    else if (verb === 'SET' && t.word === ',' && depth === 0) assigning = true;
+    if (t.word === 'INTO' && next && (next.host || next.word === 'DESCRIPTOR')) { into = true; continue; }
+    if (into && t.word && AFTER_INTO.has(t.word)) into = false;
+    if (t.word === 'DESCRIPTOR' && next?.host) {
+      // FETCH ... INTO DESCRIPTOR and USING DESCRIPTOR are synonyms: the program fills the SQLDA and Db2 writes where it points.
+      if (verb === 'FETCH' || intoBoth) { read(next.host); write(next.host); } else read(next.host);
+      i++;
+      continue;
+    }
+    if (!t.host) continue;
+    const written = into || assigning
+      || (verb === 'GET' && next?.word === '=')
+      || (verb === 'ASSOCIATE' && !toks.slice(0, i).some((x) => x.word === 'WITH'));
+    if (!written || (into && intoBoth)) read(t.host);
+    if (written) write(t.host);
+  }
+  return { sending, receiving };
+}
+
+// CALL :name reads the name. Each argument that is a lone variable, with or without its indicator,
+// is read and may be written; any other argument is an expression, IN only.
+function callRoles(toks, read, write) {
+  let depth = 0;
+  let arg = [];
+  const close = () => {
+    const hosts = arg.filter((t) => t.host);
+    const lone = hosts.length && arg.every((t) => t.host || t.word === 'INDICATOR');
+    for (const h of hosts) { read(h.host); if (lone) write(h.host); }
+    arg = [];
+  };
+  for (let i = 1; i < toks.length; i++) {
+    const t = toks[i];
+    if (t.word === 'DESCRIPTOR' && toks[i + 1]?.host) { read(toks[i + 1].host); write(toks[i + 1].host); i++; continue; }
+    if (t.word === '(') { if (depth++ > 0) arg.push(t); continue; }
+    if (t.word === ')') { if (--depth > 0) arg.push(t); else close(); continue; }
+    if (depth === 0) { if (t.host) read(t.host); continue; }
+    if (depth === 1 && t.word === ',') { close(); continue; }
+    arg.push(t);
+  }
+}
+
+export function hostVariableRoles(sql) {
+  return sqlRoles(sqlTokens(sql));
+}
+
+// The host variables in a parsed EXEC SQL block: a colon, then a name qualified by joined periods as
+// :GROUP.FIELD. Each carries its path, the token naming the item, where it stands, and whether the
+// statement writes it by sqlRoles.
+export function hostVariablesIn(toks) {
+  const found = [];
+  const sql = [];
+  for (let k = 0; k < toks.length; k++) {
+    const t = toks[k];
+    if (t.t === 'op' && t.v === ':' && toks[k + 1] && toks[k + 1].t === 'word') {
+      const path = [toks[k + 1]];
+      let j = k + 2;
+      for (; j + 1 < toks.length && toks[j].t === 'period' && toks[j].joined && toks[j + 1].t === 'word'; j += 2) path.push(toks[j + 1]);
+      const host = path.map((p) => p.u).join('.');
+      found.push({ path, tok: path[path.length - 1], at: k + 1, host });
+      sql.push({ host });
+      k = j - 1;
+      continue;
+    }
+    sql.push(t.t === 'lit' ? { lit: t.v } : { word: String(t.u ?? t.v).toUpperCase() });
+  }
+  const { receiving } = sqlRoles(sql);
+  return found.map((h) => ({ ...h, written: receiving.includes(h.host) }));
+}

package/lib/evidence/journal.mjs

@@ -4,7 +4,7 @@
 import { randomBytes } from 'node:crypto';
 import { closeSync } from 'node:fs';
 import { join } from 'node:path';
-import { makeRecord, newChain } from './record.mjs';
+import { makeRecord, newChain, recordHash } from './record.mjs';
 import { EvidenceRefusal, createExclusive, lastLine, openAppend, prepareDir, syncClose, takeLock, writeAll } from './store.mjs';
 
 export const LEDGER = 'ledger.jsonl';
@@ -13,12 +13,21 @@ export const LOCK = 'ledger.lock';
 const isoNow = () => new Date().toISOString();
 const runStamp = (iso) => iso.replace(/[-:]/g, '').replace(/\.\d+Z$/, 'Z');
 
+// The record the next ledger line extends: one whose chain, seq and hash hold.
+function ledgerTip(line, path) {
+  let r = null;
+  try { r = JSON.parse(line); } catch { /* refused below */ }
+  const ok = r && typeof r === 'object' && /^[0-9a-f]{32}$/.test(r.chain) && Number.isInteger(r.seq) && r.seq >= 0 && typeof r.hash === 'string' && recordHash(r) === r.hash;
+  if (!ok) throw new EvidenceRefusal(`${path} ends in a line that is no ledger record; verify it before anything is appended`);
+  return r;
+}
+
 // Appends to the ledger; the caller holds the lock. A torn ledger is refused rather than extended.
 function appendLedger(dir, entries, now) {
   const path = join(dir, LEDGER);
   const { line, torn } = lastLine(path);
   if (torn) throw new EvidenceRefusal(`${path} ends in a partial line; verify it before anything is appended`);
-  let prev = line ? JSON.parse(line) : null;
+  let prev = line ? ledgerTip(line, path) : null;
   const fd = openAppend(path);
   try {
     const out = [];

package/lib/explain.mjs

@@ -7,15 +7,16 @@ import { EVIDENCE } from './kernel/findings.mjs';
 import { REGISTRY } from './kernel/registry.mjs';
 import { WHO_ACTS } from './tui/model.mjs';
 import { verificationPlan } from './verify.mjs';
+import { cobolCard, statementCard } from './card.mjs';
 
 const HIDDEN_RULES = new Set(Object.keys(REGISTRY.find((s) => s.name === 'hidden').rules));
 
 // The columns quoted: COBOL's indicator and program text, JCL's statement field, or the whole line.
 function codeArea(line, kind, format) {
-  if (kind === 'cobol' && format === 'fixed') return { text: line.slice(6, 72), seq: line.slice(0, 6), rest: line.slice(72) };
-  if (kind === 'cobol' && format === 'variable') return { text: line.slice(6, 250), seq: line.slice(0, 6), rest: line.slice(250) };
-  if (kind === 'jcl') return { text: line.slice(0, 72), seq: '', rest: line.slice(72) };
-  return { text: line, seq: '', rest: '' };
+  if (kind === 'jcl') { const c = statementCard(line); return { text: c.text, seq: '', rest: c.sequence }; }
+  if (kind !== 'cobol') return { text: line, seq: '', rest: '' };
+  const c = cobolCard(line, format);
+  return { text: c.indicator + c.text, seq: c.sequence, rest: c.ignored };
 }
 
 const NOTE = 'This packet contains source text from the files it names. A cobolwork report never does: '

package/lib/ftp.mjs

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { statementCard } from './card.mjs';
+
+// IBM's batch FTP client takes the host and its options on PARM, or the host as the first line of
+// its input, reads subcommands from INPUT, and logs on from NETRC or from its input. CardDemo's job
+// gives the same lines on SYSIN, which is read here when there is no INPUT, though the page below
+// does not say the client falls back to it.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halu001/ftpreq.htm
+// A session is in cleartext unless something asks for TLS: -a TLS or -r TLS on PARM, or
+// SECURE_MECHANISM TLS in an in-stream SYSFTPD. A SYSFTPD naming a data set is configuration this
+// reader cannot see, so that step is not called cleartext either way.
+//   https://www.ibm.com/docs/en/zos/2.1.0?topic=commands-ftp-command
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halz002/ftp_cust_client_for_tls.htm
+// PUT, MPUT and APPEND send local files, and IBM's capitals - PUt, MPut, APpend, LCd - are the
+// shortest spellings. A local name in quotes is a whole data set name, //DD: names a DD of the
+// step, and anything else hangs off the local working directory, which only a quoted LCD fixes.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/ftpsubcmd.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/put.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/mput.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/append.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/lcd.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/dd_name_support.htm
+// Get and MGet copy remote files in, each to the local name given or, with none, to the foreign
+// file's own name, placed as a sent name is. SIte FILETYPE=JES makes the remote host submit what
+// is sent as a job and return job output to a Get; FILETYPE=SQL makes it run a query. Either holds
+// for the transfers after it, until another FILETYPE.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/get.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/mget.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/site.htm
+const FTP_SENDS = [[/^PUT?$/i, 'PUT'], [/^MP(?:UT?)?$/i, 'MPUT'], [/^AP(?:P(?:E(?:ND?)?)?)?$/i, 'APPEND']];
+
+// Every subcommand on the page above, in IBM's spelling.
+const FTP_SUBCOMMANDS = ['?', '!', 'ACCount', 'APpend', 'AUth', 'AScii', 'BIG5', 'BINary', 'BLock', 'CCc', 'CD',
+  'CDUp', 'CLEar', 'CLose', 'COMpress', 'CProtect', 'CWd', 'DEBug', 'DELEte', 'DELImit', 'DIr', 'DUMP', 'EBcdic',
+  'EUckanji', 'FEature', 'FIle', 'Get', 'GLob', 'HAngeul', 'HElp', 'Ibmkanji', 'JIS78kj', 'JIS83kj', 'Ksc5601',
+  'LANGuage', 'LCd', 'LOCSIte', 'LOCSTat', 'LMkdir', 'LPwd', 'LS', 'MDelete', 'MGet', 'MKdir', 'MKFifo', 'MOde',
+  'MPut', 'MVSGet', 'MVSPut', 'NOop', 'Open', 'PAss', 'PRIvate', 'PROMpt', 'PROTect', 'PROXy', 'PUt', 'PWd', 'QUIt',
+  'QUOte', 'RECord', 'REName', 'REStart', 'RMdir', 'SAfe', 'SChinese', 'SENDPort', 'SENDSite', 'SIte', 'SJiskanji',
+  'SRestart', 'STAtus', 'STREam', 'STRucture', 'SUnique', 'SYstem', 'TChinese', 'TSO', 'TYpe', 'UCs2', 'User', 'Verbose']
+  .map((name) => ({ full: name.toUpperCase(), min: name.match(/^[^a-z]*/)[0].length }));
+const subcommandOf = (word) => {
+  const w = word.toUpperCase();
+  const hit = FTP_SUBCOMMANDS.find((c) => w.length >= c.min && c.full.startsWith(w));
+  return hit ? hit.full : null;
+};
+// A value that stands for a password rather than being one: a symbolic, a mask, or the word itself.
+const PLACEHOLDER = /^(&.*|\*+|X+|\?+|<.*>|\{.*\}|PASSWORD|PASSWD|PWD|PASS|SECRET)$/i;
+
+// The session one FTP step runs, from its DDs grouped by name (ddGroups in lib/utilities.mjs).
+export function ftpSession(step, dds) {
+  let host = null, tls = null;
+  const words = (step.parm || '').split('(')[0].trim().split(/\s+/).filter(Boolean);
+  for (let i = 0; i < words.length; i++) {
+    if (/^-[ar]$/i.test(words[i])) {
+      const mechanism = (words[++i] || '').toUpperCase();
+      if (mechanism === 'TLS') tls = 'asked';
+      else if (mechanism === 'NEVER' && !tls) tls = 'never';
+    } else if (!words[i].startsWith('-') && !host) host = words[i];
+  }
+  let configUnread = null;
+  for (const dd of dds.get('SYSFTPD') || []) {
+    if (dd.inStream) { if (tls !== 'never' && dd.inStream.some((l) => /^\s*SECURE_MECHANISM\s+TLS\b/i.test(l.text))) tls = 'asked'; }
+    else if (dd.dsn && !configUnread) configUnread = dd.dsn;
+  }
+
+  const input = dds.get('INPUT') || dds.get('SYSIN') || [];
+  const hostLine = !host && input[0]?.inStream;
+  let inputUnread = null, lcd = null, unixDir = false;
+  const sends = [], gets = [];
+  let filetype = null;
+  // With no NETRC to answer for it, the client asks for a user ID and a password and in batch reads
+  // them from its input after the host: "user password" on one line, or each on its own. A first
+  // line that is a subcommand means something else answered, a NETRC the client found for itself.
+  // USER and PASS give the same answers later in a session.
+  let expect = dds.has('NETRC') ? null : 'logon', user = null;
+  const passwords = [];
+  const password = (dd, line, value) => { if (!PLACEHOLDER.test(value)) passwords.push({ dd, line, user }); };
+  const local = (name) => {
+    const dd = /^\/\/DD:([A-Z$#@][A-Z0-9$#@]{0,7})$/i.exec(name);
+    if (dd) return { dd: dd[1].toUpperCase(), dsns: (dds.get(dd[1].toUpperCase()) || []).map((d) => d.dsn).filter(Boolean) };
+    const quoted = /^'(.+)'$/.exec(name);
+    if (quoted) return { dsns: [quoted[1].toUpperCase()] };
+    if (name.startsWith('/') || unixDir) return { dsns: [] };
+    return lcd ? { dsns: [`${lcd}.${name.toUpperCase()}`] } : { dsns: [], relative: name };
+  };
+  for (const dd of input) {
+    if (!dd.inStream) { if (dd.dsn && !inputUnread) inputUnread = dd.dsn; continue; }
+    for (const { line, text } of dd.inStream) {
+      const [verb, ...args] = statementCard(text).text.trim().split(/\s+/);
+      if (!verb) continue;
+      if (hostLine && !host) { host = verb; continue; }
+      const sub = subcommandOf(verb);
+      if (expect && !sub) {
+        if (expect === 'logon') user = verb;
+        if (expect === 'password') password(dd, line, verb);
+        else if (args[0]) password(dd, line, args[0]);
+        expect = expect === 'logon' && !args[0] ? 'password' : null;
+        continue;
+      }
+      expect = null;
+      if (sub === 'USER' && args[0]) {
+        user = args[0];
+        if (args[1]) password(dd, line, args[1]); else expect = 'password';
+        continue;
+      }
+      if (sub === 'PASS' && args[0]) { password(dd, line, args[0]); continue; }
+      if (/^LCD?$/i.test(verb) && args[0]) {
+        const quoted = /^'(.+)'$/.exec(args[0]);
+        unixDir = args[0].startsWith('/');
+        lcd = quoted ? quoted[1].toUpperCase() : lcd && !unixDir ? `${lcd}.${args[0].toUpperCase()}` : null;
+        continue;
+      }
+      const site = sub === 'SITE' ? args : sub === 'QUOTE' && subcommandOf(args[0] || '') === 'SITE' ? args.slice(1) : null;
+      if (site) {
+        for (const a of site) { const t = /^FILE(?:T(?:Y(?:P(?:E)?)?)?)?=(\w+)$/i.exec(a); if (t) filetype = t[1].toUpperCase() === 'SEQ' ? null : t[1].toUpperCase(); }
+        continue;
+      }
+      const typed = filetype ? { filetype } : {};
+      if (sub === 'GET' && args[0]) {
+        const to = args[1] && !args[1].startsWith('(') ? args[1] : args[0];
+        gets.push({ line, verb: 'GET', name: args[0], ...typed, ...local(to) });
+        continue;
+      }
+      if (sub === 'MGET') {
+        for (const name of args.filter((a) => !a.startsWith('(') && !/[*?]/.test(a))) gets.push({ line, verb: 'MGET', name, ...typed, ...local(name) });
+        continue;
+      }
+      const sent = FTP_SENDS.find(([re]) => re.test(verb));
+      if (!sent) continue;
+      const foreign = sent[1] !== 'MPUT' && args[1] && !args[1].startsWith('(') ? args[1] : null;
+      for (const name of sent[1] === 'MPUT' ? args : args.slice(0, 1)) sends.push({ line, verb: sent[1], name, foreign: foreign || name, ...typed, ...local(name) });
+    }
+  }
+  const cleartext = tls === 'never' || (!tls && !configUnread);
+  return { host, cleartext, configUnread, inputUnread, logon: dds.has('NETRC') ? 'NETRC' : null, sends, gets, passwords, input };
+}

package/lib/ironwork.mjs

@@ -7,7 +7,7 @@ import { tmpdir } from 'node:os';
 import { resolve } from 'node:path';
 import { directoryTree } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
-import { buildFileIndex, VERBS } from './parser.mjs';
+import { buildFileIndex } from './parser.mjs';
 import { isProgram, relPath } from './sources.mjs';
 import { EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS } from './words.mjs';
 
@@ -20,15 +20,13 @@ const NOT_AN_ERROR = /^(?:informational:|.*?: warning: )/;
 const PER_PROGRAM_MS = 60000;
 const MAX_LISTED = 10;
 
-// ironwork refuses by name what it does not model yet; that says nothing about the program. Where its
-// parser stops at a statement it does not read (ENTRY, ALTER), or at a name the CICS, DL/I or SQL
-// translator declares, it says so without the name, so both are read as its gap, not the program's.
+// ironwork refuses by name what it does not model yet; that says nothing about the program. A name
+// the CICS, DL/I or SQL translator declares it reports as undefined, without saying so, and that is
+// read as its gap, not the program's.
 const NOT_SUPPORTED = /\bnot supported\b/i;
 const TRANSLATOR_NAMES = [EIB_FIELDS, DIB_FIELDS, SQLCA_FIELDS];
 function notModelled(message) {
   if (NOT_SUPPORTED.test(message)) return true;
-  const verb = /^a statement, found ([A-Z][A-Z-]*)$/.exec(message);
-  if (verb) return VERBS.has(verb[1]);
   const name = /^([A-Z][A-Z0-9-]*) is not defined$/.exec(message);
   return !!name && TRANSLATOR_NAMES.some((s) => s.has(name[1]));
 }

package/lib/jcl.mjs

@@ -12,17 +12,14 @@
 // output as `unreadable`, never dropped, because a step nobody parsed is not a step that is not
 // there.
 import { readSource } from './sources.mjs';
-import { copiesOf } from './utilities.mjs';
+import { copiesOf, tsoCommands } from './utilities.mjs';
+import { statementCard } from './card.mjs';
 
 // The operations a statement may carry. Anything else in the operation field is a statement this
 // reader does not know, which is a diagnostic rather than a silent skip.
 export const OPERATIONS = new Set(['JOB', 'EXEC', 'DD', 'PROC', 'PEND', 'SET', 'IF', 'THEN',
   'ELSE', 'ENDIF', 'INCLUDE', 'JCLLIB', 'OUTPUT', 'CNTL', 'ENDCNTL', 'XMIT', 'COMMAND', 'NOTIFY', 'EXPORT']);
 
-// Columns 73 to 80 are the sequence area. They are not the statement, and a payload hidden there
-// is the hidden-content rules' business, not this reader's.
-const STATEMENT_COLUMNS = 72;
-
 // A continued operand resumes in these columns. Outside them it is not a continuation, whatever
 // the previous line ended with.
 const CONTINUE_FROM = 4;
@@ -146,8 +143,10 @@ export function foldStatements(src) {
     const line = i + 1;
     const raw = phys[i];
     if (raw.length > 80) diags.push({ sev: 'warn', line, text: 'line is longer than 80 columns' });
-    const text = raw.slice(0, STATEMENT_COLUMNS).replace(/\s+$/, '');
-    const sequence = raw.slice(STATEMENT_COLUMNS);
+    // The sequence area is not the statement; a payload hidden there is the hidden-content rules' business.
+    const card = statementCard(raw);
+    const text = card.text.replace(/\s+$/, '');
+    const sequence = card.sequence;
 
     if (stream) {
       // A custom DLM ends the stream and nothing else does. The default ends at /* or at the next
@@ -402,6 +401,27 @@ export function parseJcl(src, file, opts = {}) {
     else diags.push({ sev: 'warn', line: 0, text: `symbolic parameter &${s} has no value, so the operands using it were read unsubstituted` });
   }
 
+  // A TSO batch step's ALLOCATE commands define DDs its JCL does not show. Each is kept as a DD of the
+  // step, marked `allocated`, with the datasets of a list concatenated under it as JCL would.
+  for (const s of steps) {
+    const tso = tsoCommands(s);
+    for (const n of tso.notes) diags.push({ sev: 'info', ...n });
+    for (const a of tso.allocations) {
+      const ends = a.datasets.length ? a.datasets : [{ written: null, dsn: null }];
+      ends.forEach((d, i) => {
+        const dd = {
+          name: i === 0 ? a.dd : null, procStep: null, dsn: d.dsn, rawDsn: d.written, referback: null,
+          disp: a.status, access: accessOf(a.status), temporary: false, sysout: a.sysout,
+          inStream: null, dlm: null, concatenated: i > 0, keywords: new Map(), line: a.line, endLine: a.line,
+          step: s.name, allocated: true,
+        };
+        dds.push(dd);
+        s.dds.push(dd);
+        stepOf.set(dd, s);
+      });
+    }
+  }
+
   // What each step's program copies, from lib/utilities.mjs. A program that table does not know
   // copies nothing here, which is not the same as copying nothing.
   const copies = [];