@portll/cobolwork 0.2.76 → 0.2.140

package/lib/sets/cics.mjs

@@ -13,6 +13,24 @@ import { parseCsd } from '../csd.mjs';
 // must act on from one that only wants a look; `remedy` is the standard fix, the same for every
 // instance. Both ride with every finding of the rule (as ruleImpact/ruleRemedy).
 export const CICS_RULES = {
+  'program-checks-stored-password': {
+    sev: 'med', evidence: 'construct', cwe: 'CWE-256',
+    text: 'A program compares a password it reads back from its own file or table',
+    impact: 'The password is kept where the application reads it, so whoever can read that file or table - a developer, a backup, an unload - holds every user\'s password, and the program, not the security manager, decides who signs on',
+    remedy: 'Verify the password with EXEC CICS VERIFY PASSWORD or SIGNON, so RACF checks it and the application never stores it',
+  },
+  'password-case-folded-before-compare': {
+    sev: 'low', evidence: 'construct', cwe: 'CWE-178',
+    text: 'A sign-on folds a password\'s case before comparing it',
+    impact: 'A password that matches in either case has fewer values to guess, and the program accepts a password its user never typed',
+    remedy: 'Compare the password as typed, or leave the comparison to the security manager with EXEC CICS VERIFY PASSWORD',
+  },
+  'cics-condition-ignored': {
+    sev: 'low', evidence: 'construct', cwe: 'CWE-252',
+    text: 'A program issues CICS commands whose failure it cannot see: NOHANDLE with no RESP, or IGNORE CONDITION',
+    impact: 'A command that fails - a record not found, a queue full, storage not available - returns to the next statement as if it had worked, so the program acts on an area the command never filled',
+    remedy: 'Give each command RESP and test it, or let the condition reach a HANDLE CONDITION that deals with it; keep NOHANDLE for commands whose failure is harmless',
+  },
   'cics-commarea-without-length-check': {
     sev: 'high', evidence: 'construct', cwe: 'CWE-1284',
     text: 'A program reads its communication area without checking EIBCALEN',
@@ -118,6 +136,38 @@ function execOptions(exec) {
 
 const subtreeRead = (x) => x.directRefs > 0 || x.children.some(subtreeRead);
 
+// Commands whose failure a program ordinarily lets pass: removing a temporary-storage queue that may
+// not exist, a line written to a transient-data log, the clock, a screen sent to a terminal that may
+// have gone, an ASSIGN, and the RETURN that ends the task anyway.
+const QUIET_FAILURES = (words) => (words[0] === 'DELETEQ' && words[1] === 'TS') || (words[0] === 'WRITEQ' && words[1] === 'TD')
+  || ['ASKTIME', 'FORMATTIME', 'SEND', 'ASSIGN', 'RETURN'].includes(words[0]);
+const SHOWN_IGNORED = 5;
+
+// Commands whose outcome the program cannot see: NOHANDLE with no RESP and no EIBRESP test before the
+// next command, and IGNORE CONDITION. One finding per program, at the first, since the count is what
+// a reviewer weighs and the fix is the same at each.
+function conditionsIgnored(p, path) {
+  const execs = p.execs.filter((e) => e.kind === 'CICS').sort((a, b) => a.line - b.line);
+  const hits = [];
+  execs.forEach((e, k) => {
+    const { opts: o, words } = execOptions(e);
+    if (words[0] === 'IGNORE' && words[1] === 'CONDITION') {
+      hits.push({ e, what: `IGNORE CONDITION ${words.slice(2).join(' ')}`.trim() });
+      return;
+    }
+    if (!words.includes('NOHANDLE') || o.has('RESP') || QUIET_FAILURES(words)) return;
+    const until = execs[k + 1] ? execs[k + 1].line : Infinity;
+    const tested = p.statements.some((st) => st.line > e.line && st.line < until && (st.sources || []).some((t) => t.t === 'word' && t.u === 'EIBRESP'));
+    if (!tested) hits.push({ e, what: `${words[0]}${words[1] && words[1] !== 'NOHANDLE' && !o.has(words[1]) ? ` ${words[1]}` : ''} NOHANDLE` });
+  });
+  if (!hits.length) return null;
+  const shown = hits.slice(0, SHOWN_IGNORED).map((h) => `${h.what} at line ${h.e.line}`).join(', ');
+  return {
+    rule: 'cics-condition-ignored', path, line: hits[0].e.line, program: p.id,
+    detail: `${p.id} leaves ${hits.length} place${hits.length === 1 ? '' : 's'} where a CICS command's failure goes unseen: ${shown}${hits.length > SHOWN_IGNORED ? `, and ${hits.length - SHOWN_IGNORED} more` : ''}`,
+  };
+}
+
 // Sizes come from the parser's data-division layout, which is graded against the compiler, so a
 // length compared here is the length the compiler would compute.
 //
@@ -171,12 +221,15 @@ export function scanCics(root, opts = {}) {
         declared,
       });
     }
-    // A program that points DFHCOMMAREA at storage of its own is not reading a caller's area.
+    // A program that points DFHCOMMAREA at storage of its own is not reading a caller's area. Only
+    // the receiving side of SET, before TO, repoints it: SET P TO ADDRESS OF DFHCOMMAREA reads the
+    // caller's area's address.
     const tokens = (p.proc && p.proc.tokens) || [];
     const pointsItself = (p.statements || []).some((st) => {
       if (st.verb !== 'SET') return false;
       const w = tokens.slice(st.at + 1, st.end).filter((t) => t.t === 'word').map((t) => t.u);
-      return w.some((x, k) => x === 'ADDRESS' && w[k + 1] === 'OF' && w[k + 2] === 'DFHCOMMAREA');
+      const receiving = w.includes('TO') ? w.slice(0, w.indexOf('TO')) : w;
+      return receiving.some((x, k) => x === 'ADDRESS' && receiving[k + 1] === 'OF' && receiving[k + 2] === 'DFHCOMMAREA');
     });
     return {
       id: p.id, path,
@@ -194,6 +247,67 @@ export function scanCics(root, opts = {}) {
   }
 
   // Where a program sends control after its sign-on succeeds, and every other route there.
+  // A sign-on that compares a password the program read back from its own file or table, rather than
+  // asking the security manager, and one that folds a password's case first. Passwords are recognised
+  // by field name, as the sign-on rules above recognise them, and each finding says so.
+  function passwordChecks(p, path) {
+    if (p.execs.some((e) => e.kind === 'CICS' && /^(VERIFY|SIGNON)$/.test(execOptions(e).words[0] || ''))) return [];
+    const byName = new Map();
+    for (const it of p.items) if (!byName.has(it.name)) byName.set(it.name, it);
+    const resolve = (tok) => (tok && tok.t === 'word' ? p.resolved.get(tok) || byName.get(tok.u) || null : null);
+    const tests = [];
+    try { buildControl(p, resolve, { extra: (t) => { const r = signOnTest(t); if (r) tests.push(t); return r; } }); } catch { return []; }
+    if (!tests.length) return [];
+    // What the program reads back: an FD record, the INTO of a CICS READ, a host variable a SELECT or
+    // FETCH fills, and everything under any of them. A copy made by MOVE first is not followed.
+    const read = new Set();
+    for (const e of p.execs) {
+      const { opts: o, words } = execOptions(e);
+      if (e.kind === 'CICS' && /^READ(NEXT|PREV)?$/.test(words[0] || '')) for (const t of o.get('INTO') || []) if (t.t === 'word') read.add(t.u);
+      if (e.kind === 'SQL' && (e.hostVariables || []).length) for (const h of e.hostVariables) if (h.written && h.tok) read.add(h.tok.u);
+    }
+    const stored = (name) => {
+      for (let it = byName.get(name); it; it = it.parent) if (it.section === 'FILE' || read.has(it.name)) return true;
+      return false;
+    };
+    const out = [];
+    const FOLD = new Set(['UPPER-CASE', 'LOWER-CASE']);
+    for (const t of tests) {
+      const cond = t.cond || (t.evaluate && t.evaluate.conds[0]) || [];
+      const names = cond.filter((x, k) => x.t === 'word' && byName.has(x.u) && !(cond[k - 1] && /^(OF|IN)$/.test(cond[k - 1].u))).map((x) => x.u);
+      const kept = names.find(stored);
+      const line = t.st ? t.st.line : 1;
+      if (kept) {
+        out.push({
+          rule: 'program-checks-stored-password', path, line, program: p.id,
+          detail: `${p.id} compares ${names.join(' with ')}, and ${kept} is read back from the program's own file or table, so the password is stored where the application can read it; the password is recognised by its field name`,
+        });
+      }
+      if (cond.some((x) => x.t === 'word' && FOLD.has(x.u))) {
+        out.push({
+          rule: 'password-case-folded-before-compare', path, line, program: p.id,
+          detail: `${p.id} folds the case of ${names.join(' or ')} inside its sign-on comparison, so a password matches in either case; the password is recognised by its field name`,
+        });
+      }
+    }
+    // Case folded before the comparison: a MOVE of FUNCTION UPPER-CASE or LOWER-CASE of a password
+    // field, or an INSPECT that converts one.
+    const toks = (p.proc && p.proc.tokens) || [];
+    for (const st of p.statements || []) {
+      const seg = toks.slice(st.at + 1, st.end ?? st.at + 1);
+      const folds = st.verb === 'MOVE' ? seg.some((x) => x.t === 'word' && FOLD.has(x.u))
+        : st.verb === 'INSPECT' && seg.some((x) => x.t === 'word' && x.u === 'CONVERTING');
+      if (!folds) continue;
+      const pw = seg.find((x) => x.t === 'word' && PASSWORD.test(x.u) && byName.has(x.u));
+      if (!pw || out.some((o) => o.rule === 'password-case-folded-before-compare' && o.line === st.line)) continue;
+      out.push({
+        rule: 'password-case-folded-before-compare', path, line: st.line, program: p.id,
+        detail: `${p.id} folds the case of ${pw.u} with ${st.verb === 'MOVE' ? 'FUNCTION UPPER-CASE or LOWER-CASE' : 'INSPECT CONVERTING'} in a program that compares passwords, so a password matches in either case; the password is recognised by its field name`,
+      });
+    }
+    return out;
+  }
+
   function signOnBypasses(p, path) {
     let ctl;
     const byName = new Map();
@@ -238,6 +352,9 @@ export function scanCics(root, opts = {}) {
     stats.filesScanned++;
     const path = relPath(root, f);
     for (const p of r.programs) {
+      const ignored = conditionsIgnored(p, path);
+      if (ignored) findings.push(ignored);
+      if (p.refs.some((x) => PASSWORD.test(x.tok.u))) findings.push(...passwordChecks(p, path));
       const sum = summarise(p, path);
       summaries.push(sum);
       if (p.id && !commareaSize.has(p.id)) commareaSize.set(p.id, sum.commarea ? sum.commarea.size : null);

package/lib/sets/copybook.mjs

@@ -7,6 +7,7 @@ import { finish } from '../kernel/findings.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
+import { cobolCard } from '../card.mjs';
 
 // A copybook is resolved by name along a search path: the program's own directory first, then the
 // include directories, then the system library. Two files answering to one name mean the record
@@ -37,7 +38,7 @@ const SYSTEM_NAME = /^(DFHAID|DFHBMSCA|DFHEIBLK|DFHCOMMAREA|DFHRESP|SQLCA|SQLDA|
 function layout(src) {
   const fixed = detectFormat(src) === 'fixed';
   return src.split(/\r?\n/)
-    .map(l => (fixed ? l.slice(6, 72) : l).trimEnd())
+    .map(l => { const c = cobolCard(l, fixed ? 'fixed' : 'free'); return (c.indicator + c.text).trimEnd(); })
     .filter(l => l.trim() && !/^[*/]/.test(l.trimStart()))
     .join('\n')
     .toUpperCase();

package/lib/sets/flow.mjs

@@ -3,6 +3,9 @@ import { analyze } from '../dataflow.mjs';
 import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from '../version.mjs';
 import { finish, withMeta, sortFindings } from '../kernel/findings.mjs';
 
+// Other uses of the same index listed on a finding; the count of the rest is alsoUses.
+const ALSO_LISTED = 25;
+
 export const RULES = {
   'argv-or-env-to-os-command': { sev: 'crit', evidence: 'path', cwe: 'CWE-78', text: 'Command-line or environment input reaches an operating-system command routine' },
   'cics-terminal-to-os-command': { sev: 'crit', evidence: 'path', cwe: 'CWE-78', text: 'Terminal input reaches an operating-system command routine' },
@@ -108,10 +111,42 @@ export const RULES = {
   // misleads whoever reads the log rather than changing what the program does.
   'cics-terminal-to-log': { sev: 'low', evidence: 'path', cwe: 'CWE-117', text: 'Terminal input is written to a log unchecked, where control characters in it can start a line the program did not write' },
   'cics-web-to-log': { sev: 'low', evidence: 'path', cwe: 'CWE-117', text: 'Web request data is written to a log unchecked, where control characters in it can start a line the program did not write' },
+  // Storage acquired with a length the caller chose: in a CICS region the storage is shared by every
+  // task, where a job's own PARM or command line exhausts only its own address space.
+  'cics-terminal-to-storage-length': { sev: 'med', evidence: 'path', cwe: 'CWE-770', text: 'Terminal input decides how much storage the program acquires' },
+  'cics-web-to-storage-length': { sev: 'med', evidence: 'path', cwe: 'CWE-770', text: 'Web input decides how much storage the program acquires' },
+  'argv-or-env-to-storage-length': { sev: 'low', evidence: 'path', cwe: 'CWE-770', text: 'Command-line or environment input decides how much storage the program acquires' },
+  'jcl-parm-to-storage-length': { sev: 'low', evidence: 'path', cwe: 'CWE-770', text: 'A JCL PARM decides how much storage the program acquires' },
+  'jcl-instream-to-storage-length': { sev: 'low', evidence: 'path', cwe: 'CWE-770', text: 'In-stream job data decides how much storage the program acquires' },
+  // The database or queue manager a program connects to, chosen by its caller: CVE-2024-52899 is the
+  // JDBC shape of this in an IBM product. A job's own PARM reaches only what its submitter could.
+  'cics-terminal-to-connection-target': { sev: 'med', evidence: 'path', cwe: 'CWE-99', text: 'Terminal input names the database or queue manager the program connects to' },
+  'cics-web-to-connection-target': { sev: 'med', evidence: 'path', cwe: 'CWE-99', text: 'Web input names the database or queue manager the program connects to' },
+  'argv-or-env-to-connection-target': { sev: 'low', evidence: 'path', cwe: 'CWE-99', text: 'Command-line or environment input names the database or queue manager the program connects to' },
+  'jcl-parm-to-connection-target': { sev: 'low', evidence: 'path', cwe: 'CWE-99', text: 'A JCL PARM names the database or queue manager the program connects to' },
+  'jcl-instream-to-connection-target': { sev: 'low', evidence: 'path', cwe: 'CWE-99', text: 'In-stream job data names the database or queue manager the program connects to' },
+  // Routing by input. A SYSID ships the request to another region, which applies its own link
+  // security; an EXEC CICS SET changes a region's resources with system-programming authority.
+  'cics-terminal-to-cics-sysid': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'Terminal input decides which region a CICS command is shipped to' },
+  'cics-web-to-cics-sysid': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'Web input decides which region a CICS command is shipped to' },
+  'file-record-to-cics-sysid': { sev: 'low', evidence: 'path', cwe: 'CWE-15', text: 'A file record decides which region a CICS command is shipped to' },
+  'database-to-cics-sysid': { sev: 'low', evidence: 'path', cwe: 'CWE-15', text: 'A database value decides which region a CICS command is shipped to' },
+  'cics-terminal-to-cics-system-resource': { sev: 'high', evidence: 'path', cwe: 'CWE-15', text: 'Terminal input names the resource an EXEC CICS SET changes' },
+  'cics-web-to-cics-system-resource': { sev: 'high', evidence: 'path', cwe: 'CWE-15', text: 'Web input names the resource an EXEC CICS SET changes' },
+  'file-record-to-cics-system-resource': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'A file record names the resource an EXEC CICS SET changes' },
+  'database-to-cics-system-resource': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'A database value names the resource an EXEC CICS SET changes' },
+  // A document from outside parsed as XML. Whether a DTD in it is honoured - an external entity
+  // fetched, an internal one expanded - is the parser's, chosen by the XMLPARSE compiler option, and
+  // no public repository parses XML from outside, so it is low until graded against the compiler.
+  'cics-web-to-xml-document': { sev: 'low', evidence: 'path', cwe: 'CWE-611', text: 'Web input is parsed as an XML document' },
+  'cics-terminal-to-xml-document': { sev: 'low', evidence: 'path', cwe: 'CWE-611', text: 'Terminal input is parsed as an XML document' },
   // A response code in a reply tells the client which failure it caused and something of the system
   // behind the program, which is what an attacker probing it wants.
   'system-response-to-web-response': { sev: 'low', evidence: 'path', cwe: 'CWE-209', text: 'A response code or SQL error the system set is sent in a web response' },
   'system-response-to-http-header': { sev: 'low', evidence: 'path', cwe: 'CWE-209', text: 'A response code or SQL error the system set is sent in a response header' },
+  // Db2's message text shown on a terminal names the tables, columns and constraints behind the
+  // program. A RESP or SQLCODE on an error line is ordinary and is not followed there.
+  'system-response-to-screen': { sev: 'low', evidence: 'path', cwe: 'CWE-209', text: 'Db2\'s error message text is shown on the terminal' },
   // A field the program filled and the map protects, read back and used to choose a record: the 3270
   // hidden form field. The terminal enforces PROT and ASKIP, CICS does not (NetSPI 'Conquering CICS',
   // ways 3 and 4). No check lowers it: a key that is well formed is still someone else's key.
@@ -150,7 +185,7 @@ export const RULES = {
 // from one to note; `remedy` is the standard fix, one per sink. Both ride with every finding as
 // ruleImpact/ruleRemedy. Composed from the source's actor and the sink's guidance rather than
 // written out for each path rule.
-const WHO = {
+export const WHO = {
   'argv-or-env': "Whoever sets the program's command line or environment",
   'cics-terminal': 'A terminal user',
   'cics-web': 'A web caller',
@@ -247,6 +282,26 @@ const SINK = {
     clause: 'updates or deletes a record chosen by a value a modified 3270 client controls',
     remedy: 'Re-derive the key from server state and authorize the change; do not trust a protected or hidden screen field as a key',
   },
+  'cics-sysid': {
+    clause: 'chooses the region a CICS command is shipped to',
+    remedy: 'Ship to a region chosen from a fixed table, or name the SYSID as a literal; never take it from input or a record',
+  },
+  'cics-system-resource': {
+    clause: 'chooses the program, transaction, file or queue a system-programming command enables, disables or changes',
+    remedy: 'Name the resource an EXEC CICS SET changes from a fixed table, and keep SP commands out of programs input reaches',
+  },
+  'connection-target': {
+    clause: 'chooses the database or queue manager the program connects to, with the program\'s own credentials',
+    remedy: 'Connect only to a location or queue manager chosen from a fixed table, never one named by input',
+  },
+  'xml-document': {
+    clause: 'supplies the document XML PARSE reads, where a DTD it carries can name entities for the parser to fetch or expand, if the parser the program is compiled with honours them',
+    remedy: 'Refuse a document that carries a DTD (<!DOCTYPE) before XML PARSE, and bound its length; check what the XMLPARSE option the program is compiled with does with entities',
+  },
+  'storage-length': {
+    clause: 'chooses how much storage the program acquires, so a large enough value fails the request or uses up storage it shares',
+    remedy: 'Test the length against the most the program means to acquire before GETMAIN or CEEGTST, and refuse anything larger',
+  },
 };
 // Exfiltration over HTTP reads the same as the host case.
 SINK['outbound-http'] = SINK['outbound-host'];
@@ -294,7 +349,6 @@ const DEFAULT = { sev: 'med', evidence: 'path', cwe: null, text: 'Untrusted inpu
 // is safe for the sink - one of a list of literals, digits where digits are needed, a bound where an
 // index is - is not a finding at all, and is listed under `checked` with the check that stops it.
 const LOWER = { crit: 'high', high: 'med', med: 'low', low: 'info', info: 'info' };
-
 // One graph per repository, for the reason scanAll gives: names are only unique within one.
 function analyzeEach(root, opts) {
   if (!opts.repos || opts.repos.length < 2) return analyze(root, opts);
@@ -349,6 +403,7 @@ export function scan(root, opts = {}) {
       // how many hops are missing cannot tell a truncated trace from one that mentions hops.
       trace: f.path.map(p => ({ program: p.program, item: p.item, file: p.file, via: p.via, ...(p.dir ? { dir: p.dir } : {}), ...(p.elided ? { elided: p.elided } : {}) })),
       related: [{ path: f.source.file, line: f.source.line, detail: f.source.detail }],
+      ...(f.also ? { also: f.also } : {}),
     });
   }
   // One sink reached by many sources is one finding, not many: a report where a single statement
@@ -361,8 +416,33 @@ export function scan(root, opts = {}) {
     held.sources++;
     // A sink is credited only as far as the least-credited source that reaches it.
     const level = Math.min(held.level, f.level);
+    const also = [...(held.also || []), ...(f.also || [])];
     if (f.hops < held.hops) Object.assign(held, f, { sources: held.sources });
     held.level = level;
+    if (also.length) held.also = also;
+  }
+  // A use one source's route reaches unchecked, and the report leaves out for another use of the same
+  // index, still limits how far a finding at that use is credited.
+  for (const f of findings) {
+    for (const a of f.also || []) {
+      const held = a.unreached ? null : perSink.get(`${f.rule}|${a.file}:${a.line}|${a.program || ''}`);
+      if (held && a.level < held.level) held.level = a.level;
+    }
+  }
+  for (const held of perSink.values()) {
+    if (!held.also) continue;
+    const seenAt = new Set();
+    const left = held.also.filter((a) => {
+      const at = `${a.file}:${a.line}`;
+      if (seenAt.has(at) || perSink.has(`${held.rule}|${at}|${a.program || ''}`)) return false;
+      seenAt.add(at);
+      return true;
+    }).sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : a.line - b.line));
+    delete held.also;
+    if (!left.length) continue;
+    const how = (a) => (a.unreached ? 'no route from the entries reaches it' : a.level === 2 ? `stopped by the check at line ${a.guardLine}` : a.level === 1 ? `checked at line ${a.guardLine}, not on every route` : 'not checked');
+    held.related = [...held.related, ...left.slice(0, ALSO_LISTED).map((a) => ({ path: a.file, line: a.line, detail: `also ${a.detail}: ${how(a)}` }))];
+    held.alsoUses = left.length;
   }
   findings.length = 0;
   const checked = [];

package/lib/sets/hidden.mjs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 import { detectFormat, expandTabs } from '../parser.mjs';
+import { cobolCard } from '../card.mjs';
 import { decodeEbcdic, inScope, isJcl, isSource, looksEbcdic, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
@@ -69,14 +70,9 @@ const BANNER = /^[\s*\-=_+#/\\.]*$/;
 
 function areas(line, format) {
   const l = expandTabs(line);
-  if (format === 'free' || format === 'terminal') return { sequence: '', indicator: '', text: l, identification: '', ignored: '' };
-  return {
-    sequence: l.slice(0, 6),
-    indicator: l[6] || ' ',
-    text: l.slice(7, format === 'variable' ? 250 : 72),
-    identification: format === 'variable' ? '' : l.slice(72, 80),
-    ignored: format === 'variable' ? '' : l.slice(72),
-  };
+  const card = cobolCard(l, format);
+  const fixed = format === 'fixed';
+  return { ...card, identification: fixed ? card.ignored.slice(0, 8) : '', ignored: fixed ? card.ignored : '' };
 }
 
 // A floating comment, from the *> that starts it outside a literal to the end of the line.

package/lib/sets/jcl.mjs

@@ -7,7 +7,8 @@ import { inScope, isJcl, isProgram, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { parseJcl, dispositionOf } from '../jcl.mjs';
-import { ddGroups } from '../utilities.mjs';
+import { ddGroups, tsoCommands } from '../utilities.mjs';
+import { ftpSession } from '../ftp.mjs';
 import { loadSite, productionQualifierOf, SITE_FILE } from '../site.mjs';
 import { loadPacks, programsOf } from '../packs.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
@@ -59,116 +60,77 @@ export const JCL_RULES = {
     impact: 'A dataset the estate calls production leaves the system to the FTP partner',
     remedy: 'Confirm the transfer and its partner are authorized for production data, and send over a TLS session',
   },
+  'control-cards-from-dataset': {
+    sev: 'low', evidence: 'construct', cwe: 'CWE-829',
+    text: 'A utility step reads the commands it runs from a cataloged data set, so what it runs is not in the reviewed JCL',
+    impact: 'Whoever can update that data set decides what the step runs the next time the job does, with the job\'s authority',
+    remedy: 'Keep the commands in-stream where they are reviewed with the job, or put the data set under the same change control and write access as the job',
+  },
+  'sort-exit-named': {
+    sev: 'med', evidence: 'construct', cwe: 'CWE-829',
+    text: 'A DFSORT or ICETOOL step names an exit routine on a MODS statement, a load module the sort loads and runs',
+    impact: 'The exit runs inside the sort with the step\'s authority and sees every record; whoever can replace that load module runs code in the job',
+    remedy: 'Confirm the exit\'s load library is under change control with restricted write access, or remove the exit',
+  },
+  'tso-batch-runs-program': {
+    sev: 'low', evidence: 'construct', cwe: 'CWE-78',
+    text: 'A TSO batch step runs another program, CLIST, REXX exec or job from its in-stream commands',
+    impact: 'What the step runs is decided by the commands, not the PGM= the job names, and runs with the job\'s TSO authority',
+    remedy: 'Review what the commands run; name a program with PGM= where the step exists to run one',
+  },
 };
 
-// IBM's batch FTP client takes the host and its options on PARM, or the host as the first line of
-// its input, reads subcommands from INPUT, and logs on from NETRC or from its input. CardDemo's job
-// gives the same lines on SYSIN, which is read here when there is no INPUT, though the page below
-// does not say the client falls back to it.
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halu001/ftpreq.htm
-// A session is in cleartext unless something asks for TLS: -a TLS or -r TLS on PARM, or
-// SECURE_MECHANISM TLS in an in-stream SYSFTPD. A SYSFTPD naming a data set is configuration this
-// reader cannot see, so that step is not called cleartext either way.
-//   https://www.ibm.com/docs/en/zos/2.1.0?topic=commands-ftp-command
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.halz002/ftp_cust_client_for_tls.htm
-// PUT, MPUT and APPEND send local files, and IBM's capitals - PUt, MPut, APpend, LCd - are the
-// shortest spellings. A local name in quotes is a whole data set name, //DD: names a DD of the
-// step, and anything else hangs off the local working directory, which only a quoted LCD fixes.
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/ftpsubcmd.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/put.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/mput.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/append.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/lcd.htm
-//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.halu001/dd_name_support.htm
-const FTP_SENDS = [[/^PUT?$/i, 'PUT'], [/^MP(?:UT?)?$/i, 'MPUT'], [/^AP(?:P(?:E(?:ND?)?)?)?$/i, 'APPEND']];
-
-// Every subcommand on the page above, in IBM's spelling.
-const FTP_SUBCOMMANDS = ['?', '!', 'ACCount', 'APpend', 'AUth', 'AScii', 'BIG5', 'BINary', 'BLock', 'CCc', 'CD',
-  'CDUp', 'CLEar', 'CLose', 'COMpress', 'CProtect', 'CWd', 'DEBug', 'DELEte', 'DELImit', 'DIr', 'DUMP', 'EBcdic',
-  'EUckanji', 'FEature', 'FIle', 'Get', 'GLob', 'HAngeul', 'HElp', 'Ibmkanji', 'JIS78kj', 'JIS83kj', 'Ksc5601',
-  'LANGuage', 'LCd', 'LOCSIte', 'LOCSTat', 'LMkdir', 'LPwd', 'LS', 'MDelete', 'MGet', 'MKdir', 'MKFifo', 'MOde',
-  'MPut', 'MVSGet', 'MVSPut', 'NOop', 'Open', 'PAss', 'PRIvate', 'PROMpt', 'PROTect', 'PROXy', 'PUt', 'PWd', 'QUIt',
-  'QUOte', 'RECord', 'REName', 'REStart', 'RMdir', 'SAfe', 'SChinese', 'SENDPort', 'SENDSite', 'SIte', 'SJiskanji',
-  'SRestart', 'STAtus', 'STREam', 'STRucture', 'SUnique', 'SYstem', 'TChinese', 'TSO', 'TYpe', 'UCs2', 'User', 'Verbose']
-  .map((name) => ({ full: name.toUpperCase(), min: name.match(/^[^a-z]*/)[0].length }));
-const subcommandOf = (word) => {
-  const w = word.toUpperCase();
-  const hit = FTP_SUBCOMMANDS.find((c) => w.length >= c.min && c.full.startsWith(w));
-  return hit ? hit.full : null;
+const TSO = new Set(['IKJEFT01', 'IKJEFT1A', 'IKJEFT1B']);
+const SORTS = new Set(['SORT', 'ICEMAN', 'SYNCSORT']);
+const CONTROL_DDS = [
+  [TSO, ['SYSTSIN']], [new Set(['BPXBATCH']), ['STDPARM']], [new Set(['IDCAMS']), ['SYSIN']],
+  [SORTS, ['SYSIN', 'DFSPARM']], [new Set(['ICETOOL']), ['TOOLIN', 'DFSPARM']],
+];
+const TSO_RUNS = /^\s*(CALL|EXEC|EX|SUBMIT|ISPSTART|RUN)\b/i;
+// Db2's sample programs read SQL statements from SYSIN, run as the step's program or under TSO by
+// DSN RUN PROGRAM. Those statements are the program's input, not commands to TSO or a security product.
+const SQL_PROGRAMS = new Set(['DSNTEP2', 'DSNTEP4', 'DSNTIAD']);
+const sqlInput = (step, groups) => {
+  const pgm = step.pgm ? step.pgm.toUpperCase() : null;
+  const runsSql = SQL_PROGRAMS.has(pgm) || tsoCommands(step).runs.some((x) => SQL_PROGRAMS.has(x.program));
+  return runsSql ? groups.get('SYSIN') || [] : [];
 };
-// A value that stands for a password rather than being one: a symbolic, a mask, or the word itself.
-const PLACEHOLDER = /^(&.*|\*+|X+|\?+|<.*>|\{.*\}|PASSWORD|PASSWD|PWD|PASS|SECRET)$/i;
 
-function ftpSession(step) {
-  const dds = ddGroups(step);
-  let host = null, tls = null;
-  const words = (step.parm || '').split('(')[0].trim().split(/\s+/).filter(Boolean);
-  for (let i = 0; i < words.length; i++) {
-    if (/^-[ar]$/i.test(words[i])) {
-      const mechanism = (words[++i] || '').toUpperCase();
-      if (mechanism === 'TLS') tls = 'asked';
-      else if (mechanism === 'NEVER' && !tls) tls = 'never';
-    } else if (!words[i].startsWith('-') && !host) host = words[i];
-  }
-  let configUnread = null;
-  for (const dd of dds.get('SYSFTPD') || []) {
-    if (dd.inStream) { if (tls !== 'never' && dd.inStream.some((l) => /^\s*SECURE_MECHANISM\s+TLS\b/i.test(l.text))) tls = 'asked'; }
-    else if (dd.dsn && !configUnread) configUnread = dd.dsn;
-  }
-
-  const input = dds.get('INPUT') || dds.get('SYSIN') || [];
-  const hostLine = !host && input[0]?.inStream;
-  let inputUnread = null, lcd = null, unixDir = false;
-  const sends = [];
-  // With no NETRC to answer for it, the client asks for a user ID and a password and in batch reads
-  // them from its input after the host: "user password" on one line, or each on its own. A first
-  // line that is a subcommand means something else answered, a NETRC the client found for itself.
-  // USER and PASS give the same answers later in a session.
-  let expect = dds.has('NETRC') ? null : 'logon', user = null;
-  const passwords = [];
-  const password = (dd, line, value) => { if (!PLACEHOLDER.test(value)) passwords.push({ dd, line, user }); };
-  const local = (name) => {
-    const dd = /^\/\/DD:([A-Z$#@][A-Z0-9$#@]{0,7})$/i.exec(name);
-    if (dd) return { dd: dd[1].toUpperCase(), dsns: (dds.get(dd[1].toUpperCase()) || []).map((d) => d.dsn).filter(Boolean) };
-    const quoted = /^'(.+)'$/.exec(name);
-    if (quoted) return { dsns: [quoted[1].toUpperCase()] };
-    if (name.startsWith('/') || unixDir) return { dsns: [] };
-    return lcd ? { dsns: [`${lcd}.${name.toUpperCase()}`] } : { dsns: [], relative: name };
-  };
-  for (const dd of input) {
-    if (!dd.inStream) { if (dd.dsn && !inputUnread) inputUnread = dd.dsn; continue; }
-    for (const { line, text } of dd.inStream) {
-      const [verb, ...args] = text.slice(0, 72).trim().split(/\s+/);
-      if (!verb) continue;
-      if (hostLine && !host) { host = verb; continue; }
-      const sub = subcommandOf(verb);
-      if (expect && !sub) {
-        if (expect === 'logon') user = verb;
-        if (expect === 'password') password(dd, line, verb);
-        else if (args[0]) password(dd, line, args[0]);
-        expect = expect === 'logon' && !args[0] ? 'password' : null;
-        continue;
+// Utilities whose commands come from a DD: a cataloged data set there is commands nobody reviewed
+// with the job; a MODS exit is a load module the sort runs; TSO commands say what else runs.
+function controlCardFindings(r, path, findings) {
+  for (const step of r.steps) {
+    const pgm = step.pgm ? step.pgm.toUpperCase() : null;
+    if (!pgm) continue;
+    const groups = ddGroups(step);
+    const cards = (name) => (groups.get(name) || []).flatMap((dd) => dd.inStream || []);
+    const ddNames = new Set(CONTROL_DDS.filter(([pgms]) => pgms.has(pgm)).flatMap(([, dds]) => dds));
+    if (sqlInput(step, groups).length) ddNames.add('SYSIN');
+    for (const name of ddNames) {
+      for (const dd of groups.get(name) || []) {
+        if (dd.dsn && !dd.inStream && !dd.temporary) {
+          findings.push({ rule: 'control-cards-from-dataset', path, line: dd.line, step: step.name,
+            detail: `step ${step.name || '(unnamed)'} runs ${pgm} with its ${name} commands in ${dd.dsn}, which is not part of this job` });
+        }
       }
-      expect = null;
-      if (sub === 'USER' && args[0]) {
-        user = args[0];
-        if (args[1]) password(dd, line, args[1]); else expect = 'password';
-        continue;
+    }
+    if (SORTS.has(pgm) || pgm === 'ICETOOL') {
+      for (const l of [...cards('SYSIN'), ...cards('DFSPARM')]) {
+        const mods = /^\s*MODS\s+(.*)$/i.exec(l.text);
+        if (!mods) continue;
+        for (const ex of mods[1].matchAll(/\b(E\d\d)\s*=\s*\(\s*([A-Z$#@][A-Z0-9$#@]{0,7})/gi)) {
+          findings.push({ rule: 'sort-exit-named', path, line: l.line, step: step.name, detail: `step ${step.name || '(unnamed)'} names ${ex[1].toUpperCase()} exit ${ex[2].toUpperCase()} on a MODS statement` });
+        }
       }
-      if (sub === 'PASS' && args[0]) { password(dd, line, args[0]); continue; }
-      if (/^LCD?$/i.test(verb) && args[0]) {
-        const quoted = /^'(.+)'$/.exec(args[0]);
-        unixDir = args[0].startsWith('/');
-        lcd = quoted ? quoted[1].toUpperCase() : lcd && !unixDir ? `${lcd}.${args[0].toUpperCase()}` : null;
-        continue;
+    }
+    if (TSO.has(pgm)) {
+      const runs = cards('SYSTSIN').map((l) => TSO_RUNS.exec(l.text)).filter(Boolean).map((m) => m[1].toUpperCase());
+      if (runs.length) {
+        findings.push({ rule: 'tso-batch-runs-program', path, line: step.line, step: step.name,
+          detail: `step ${step.name || '(unnamed)'} runs ${runs.length} TSO command(s) that start other work: ${[...new Set(runs)].join(', ')}` });
       }
-      const sent = FTP_SENDS.find(([re]) => re.test(verb));
-      if (!sent) continue;
-      for (const name of sent[1] === 'MPUT' ? args : args.slice(0, 1)) sends.push({ line, verb: sent[1], name, ...local(name) });
     }
   }
-  const cleartext = tls === 'never' || (!tls && !configUnread);
-  return { host, cleartext, configUnread, inputUnread, logon: dds.has('NETRC') ? 'NETRC' : null, sends, passwords, input };
 }
 
 // Credentials, which are the reason this rule set is critical rather than merely interesting.
@@ -181,7 +143,7 @@ const CREDENTIAL = /\b(?:PASSWORD|PASSWRD|PHRASE|PASSPHRASE)\s*(?:\(\s*)?(?:=\s*
 const SECURITY_COMMAND = /^\s*(?:TSS\s+)?(ADDUSER|ALTUSER|DELUSER|ADDGROUP|ALTGROUP|CONNECT|REMOVE|PERMIT|SETROPTS|RDEFINE|RALTER|RDELETE|ADDSD|ALTDSD|DELDSD|PASSWORD|ACFNRULE)\b/i;
 
 // ACF2 speaks differently: a mode change followed by INSERT or CHANGE against a logonid.
-const ACF2_COMMAND = /^\s*(?:SET\s+(?:LID|RULE|PROFILE)|INSERT\s+[A-Z0-9$#@]+|CHANGE\s+[A-Z0-9$#@]+\s)/i;
+const ACF2_COMMAND = /^\s*(?:SET\s+(?:LID|RULE|PROFILE)|INSERT\s+(?!INTO\b)[A-Z0-9$#@]+|CHANGE\s+[A-Z0-9$#@]+\s)/i;
 
 // Destruction in-stream. DELETE with a dataset name, or a REPRO that replaces its target. SQL's
 // DELETE FROM removes rows through a database utility, not a catalogued dataset.
@@ -304,9 +266,10 @@ export function scanJcl(root, opts = {}) {
       if (d && !stats.firstReadInPart) stats.firstReadInPart = `${path}${d.line ? `:${d.line}` : ''}: ${d.text}`;
     }
 
-    const ftpSteps = new Map(r.steps.filter((s) => s.pgm && s.pgm.toUpperCase() === 'FTP' && !defined.has('FTP')).map((s) => [s, ftpSession(s)]));
+    const ftpSteps = new Map(r.steps.filter((s) => s.pgm && s.pgm.toUpperCase() === 'FTP' && !defined.has('FTP')).map((s) => [s, ftpSession(s, ddGroups(s))]));
     const ftpInput = new Map();
     for (const ftp of ftpSteps.values()) for (const dd of ftp.input) ftpInput.set(dd, ftp);
+    const sqlDds = new Set(r.steps.flatMap((s) => sqlInput(s, ddGroups(s))));
 
     for (const dd of r.dds) {
       if (!dd.inStream) continue;
@@ -332,13 +295,13 @@ export function scanJcl(root, opts = {}) {
             detail: `in-stream FTP input gives the password ${logon.user ? `for ${logon.user} ` : ''}that step ${dd.step || '(none)'} logs on ${ftp.host ? `to ${ftp.host} ` : ''}with, in a file anyone who can read the repository can read` });
           continue;
         }
-        const sec = text.match(SECURITY_COMMAND);
+        const sec = !sqlDds.has(dd) && text.match(SECURITY_COMMAND);
         if (sec) {
           findings.push({ rule: 'jcl-instream-security-command', path, line: l.line, step: dd.step,
             detail: `in-stream data issues ${sec[1].toUpperCase()}, which grants or alters authority, from step ${dd.step || '(none)'}` });
           continue;
         }
-        if (ACF2_COMMAND.test(text)) {
+        if (!sqlDds.has(dd) && ACF2_COMMAND.test(text)) {
           findings.push({ rule: 'jcl-instream-security-command', path, line: l.line, step: dd.step,
             detail: `in-stream data issues an ACF2 administration command from step ${dd.step || '(none)'}` });
           continue;
@@ -414,6 +377,7 @@ export function scanJcl(root, opts = {}) {
         }
       }
     }
+    controlCardFindings(r, path, findings);
     return src.length;
   }, { label: 'jcl', maxBytes: opts.maxSourceBytes ?? Infinity });
 

package/lib/sets/recon.mjs

@@ -17,6 +17,7 @@ import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
 import { loadSite, classifyPath, productionQualifierOf, SITE_FILE } from '../site.mjs';
 import { parseJcl, dispositionOf } from '../jcl.mjs';
+import { statementCard } from '../card.mjs';
 
 
 // The production-dataset rules claim more than the name rule: that the job acts on production, not
@@ -93,7 +94,7 @@ function routableAddresses(text, jcl) {
   if (jcl) {
     for (const line of text.split(/\r?\n/)) {
       if (/^\/\/\*/.test(line)) continue;
-      collect(line.slice(0, 72), out);
+      collect(statementCard(line).text, out);
     }
     return out;
   }