@portll/cobolwork 0.5.0 → 0.6.0

package/lib/sets/cics.mjs

@@ -4,7 +4,7 @@ import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
 import { dirname, join } from 'node:path';
 import { parseSource } from '../parser.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { buildControl, hasFact } from '../control.mjs';
 import { parseCsd } from '../csd.mjs';
 
@@ -201,7 +201,7 @@ function conditionsIgnored(p, path) {
 // away from, and this set kept doing it: on a 100,000-program repository it exhausted an 8 GB heap
 // and took a corpus run with it. What survives per program is an identifier, a communication area's
 // size, one boolean, and a short list of transfers.
-export function scanCics(root, opts = {}) {
+export function* scanCicsSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const files = tree.list().filter(isProgram).filter(inScope(opts));
   // Identifier to the size of that program's DFHCOMMAREA, which is all a caller needs of a callee.
@@ -364,7 +364,7 @@ export function scanCics(root, opts = {}) {
   // Releasing each tree bounds what one program costs; it does not bound what a hundred thousand of
   // them cost together. The shared loop watches the heap and stops before V8 cannot collect, and
   // names every file it did not reach.
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     if (!/EXEC\s+CICS/i.test(src) && !/(?:^|[\s.])COPY\s+/i.test(src)) return src.length;
@@ -445,3 +445,5 @@ export function scanCics(root, opts = {}) {
 
   return report('cics', { rules: CICS_RULES, findings, stats, run });
 }
+
+export const scanCics = (root, opts = {}) => drive(scanCicsSteps(root, opts));

package/lib/sets/compile.mjs

@@ -13,7 +13,7 @@ import { basename, extname } from 'node:path';
 import { inScope, isProgram, relPath, readSource } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 
 // construct: no input has to reach it. med: certain once every copybook is found, and it means the
 // tree does not hold what runs, but it is not exploitable in itself. CWE-1127: MITRE discourages
@@ -102,7 +102,7 @@ function truncated(ref, lines) {
   return any && !whole;
 }
 
-export function scanCompile(root, opts = {}) {
+export function* scanCompileSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const files = tree.list().filter(isProgram).filter(inScope(opts));
   const findings = [];
@@ -117,7 +117,7 @@ export function scanCompile(root, opts = {}) {
   const textOf = (p) => (tree.contains(p) ? tree.text(p) : readSource(p)).text;
   const candidates = [];
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     if (LISTING.test(src)) { stats.filesScanned++; stats.listingsSkipped = (stats.listingsSkipped || 0) + 1; return src.length; }
@@ -186,3 +186,5 @@ export function scanCompile(root, opts = {}) {
   if (stats.filesUnparsed) stats.coverageIncomplete = true;
   return report('compile', { rules: COMPILE_RULES, findings, stats, run });
 }
+
+export const scanCompile = (root, opts = {}) => drive(scanCompileSteps(root, opts));

package/lib/sets/copybook.mjs

@@ -6,7 +6,7 @@ import { inScope, isCopybook, isProgram, relPath } from '../sources.mjs';
 import { finish } from '../kernel/findings.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { cobolCard } from '../cards.mjs';
 
 // A copybook is resolved by name along a search path: the program's own directory first, then the
@@ -71,7 +71,7 @@ export function systemLayoutDeparture(name, text) {
   return null;
 }
 
-export function scanCopybooks(root, opts = {}) {
+export function* scanCopybooksSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const all = tree.list().filter(inScope(opts));
   const findings = [];
@@ -100,7 +100,7 @@ export function scanCopybooks(root, opts = {}) {
   // is what stops the reading before the heap does.
   let run = { skipped: [], stoppedBy: null, peakHeapBytes: 0, note: null };
   if (candidates.size) {
-    run = eachWithinMemory(all.filter(isProgram), (p) => {
+    run = yield loopOver(all.filter(isProgram), (p) => {
       let src;
       try { src = tree.text(p).text; } catch (e) { noteUnread(stats, tree, p, e); return 0; }
       let r;
@@ -173,3 +173,5 @@ export function scanCopybooks(root, opts = {}) {
   // that looks unshadowed may only look that way.
   return report('copybook', { rules: COPYBOOK_RULES, findings, stats, run });
 }
+
+export const scanCopybooks = (root, opts = {}) => drive(scanCopybooksSteps(root, opts));

package/lib/sets/flow.mjs

@@ -1,5 +1,8 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { analyze } from '../dataflow.mjs';
+import { resolve } from 'node:path';
+import { directoryTree } from '../kernel/source-tree.mjs';
+import { analyzeSteps } from '../dataflow.mjs';
+import { drive } from '../kernel/shared-pass.mjs';
 import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from '../version.mjs';
 import { finish, withMeta, sortFindings } from '../kernel/findings.mjs';
 
@@ -365,11 +368,11 @@ function mergeRoutes(a, b) {
   };
 }
 // One graph per repository, for the reason scanAll gives: names are only unique within one.
-function analyzeEach(root, opts) {
-  if (!opts.repos || opts.repos.length < 2) return analyze(root, opts);
+function* analyzeEach(root, opts) {
+  if (!opts.repos || opts.repos.length < 2) return yield* analyzeSteps(root, opts);
   const merged = { findings: [], constructs: [], unstarted: [], startedBy: {}, stats: {} };
   for (const repo of opts.repos) {
-    const one = analyze(root, { ...opts, repos: [repo] });
+    const one = yield* analyzeSteps(root, { ...opts, repos: [repo] });
     merged.findings.push(...one.findings);
     merged.constructs.push(...one.constructs);
     merged.unstarted.push(...one.unstarted);
@@ -386,8 +389,9 @@ function analyzeEach(root, opts) {
   return merged;
 }
 
-export function scan(root, opts = {}) {
-  const res = analyzeEach(root, opts);
+export function* scanSteps(root, opts = {}) {
+  root = resolve(root);
+  const res = yield* analyzeEach(root, opts);
   const findings = [];
   for (const f of res.findings) {
     const meta = RULES[f.rule] || DEFAULT;
@@ -531,7 +535,8 @@ export function scan(root, opts = {}) {
       // happened to compute the right answer for the whole report - it ORs filesOverBudget in
       // separately - so nothing was wrong downstream, and nothing would have been until someone
       // called this set on its own and believed a silence.
-      coverageIncomplete: (res.stats.overBudget || 0) > 0
+      coverageIncomplete: (res.stats.files === 0 && directoryTree(root, opts).list().length === 0)
+        || (res.stats.overBudget || 0) > 0
         || (res.stats.filesNotReached || 0) > 0
         || (res.stats.unreadable || 0) > 0
         || (res.stats.threw || 0) > 0
@@ -570,6 +575,8 @@ export function scan(root, opts = {}) {
   return report;
 }
 
+export const scan = (root, opts = {}) => drive(scanSteps(root, opts));
+
 // Findings counted by the transaction or job that reaches them: the unit a mainframe team triages
 // in. A finding several entries reach counts under each.
 export function rollup(findings) {

package/lib/sets/hlasm.mjs

@@ -6,11 +6,14 @@
 //
 // It also answers the question the COBOL side cannot: a CALL 'SUBRTN' with no COBOL program of that
 // name may be an assembler module, and a CSECT or ENTRY of that name here is where it is.
-import { inScope, isAssembler, isProgram, relPath } from '../sources.mjs';
+import { basename, extname } from 'node:path';
+import { inScope, isAssembler, isProgram, relPath, HLASM_EXT, COPY_EXT } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
 import { eachWithinMemory } from '../kernel/memory.mjs';
 import { readHlasm, OPERATIONS } from '../hlasm.mjs';
+import { readHlasmStatements, parseHlasmStatement, NAMED_MACROS } from '../hlasm/read.mjs';
+import { macroOperands, splitOperands } from '../hlasm/operands.mjs';
 
 export const HLASM_RULES = {
   'hlasm-supervisor-state-change': {
@@ -31,6 +34,12 @@ export const HLASM_RULES = {
     impact: 'A program call, a program transfer or a change of secondary address space moves execution or data access across address spaces, under the authority the linkage tables grant rather than the caller\'s own',
     remedy: 'Confirm the entry tables and authorization indexes grant only the address spaces and keys the service needs, and that every PC routine validates what its caller passes',
   },
+  'hlasm-runtime-module-name': {
+    sev: 'med', evidence: 'construct', cwe: 'CWE-470',
+    text: 'Assembler source loads or calls a module whose name it reads from storage at run time',
+    impact: 'EPLOC= and DE= give LINK, XCTL, LOAD and ATTACH the address of the name rather than the name, so whatever writes that storage chooses the program that runs, with the caller\'s authority',
+    remedy: 'Name the module with EP= where it is fixed; where it must vary, check the name against a list of the modules the program may run before the call',
+  },
   'hlasm-calls-security-product': { sev: 'info', evidence: 'context', cwe: null, text: 'Assembler source calls the security product directly' },
   'hlasm-provides-called-module': { sev: 'info', evidence: 'context', cwe: null, text: 'An assembler CSECT or ENTRY is a module a COBOL program in this tree CALLs' },
 };
@@ -45,8 +54,17 @@ const STATE = { KEY: 'PSW key zero', EXTKEY: 'PSW key zero', MODE: 'supervisor s
 
 const where = (use) => (use.inMacro ? ' It is in a macro definition, so it is in every program that uses the macro.' : '');
 
-function detail(use) {
-  const written = `${use.name}${use.field ? ` ${use.field}` : ''}`;
+// EX and EXRL name their target instruction; when it is labelled in the same file, the finding says
+// which instruction runs with the register's length or mask.
+function executed(use, labels) {
+  if (use.class !== 'built-instruction') return '';
+  const target = (splitOperands(use.field || '')[1] || '').trim().toUpperCase();
+  const t = /^[A-Z$#@_][A-Z0-9$#@_]*$/.test(target) ? labels.get(target) : null;
+  return t ? ` runs ${t.operation}${t.field ? ` ${t.field}` : ''}` : '';
+}
+
+function detail(use, labels) {
+  const written = `${use.name}${use.field ? ` ${use.field}` : ''}${executed(use, labels)}`;
   if (use.class === 'security-product') {
     const request = /\bREQUEST=([A-Z]+)/i.exec(use.field || '');
     return `${request ? `${use.name} REQUEST=${request[1].toUpperCase()}` : use.name} calls the security product from assembler rather than through a service that checks on the program's behalf.${where(use)}`;
@@ -72,6 +90,32 @@ export function scanHlasm(root, opts = {}) {
   const all = tree.list().filter(inScope(opts));
   const findings = [];
   const stats = { filesScanned: 0, filesUnreadable: 0, assemblerFiles: 0, bmsFiles: 0, imsFiles: 0, unrecognisedFiles: 0, readNotAssembled: true };
+  const members = new Set(all.filter((f) => [...HLASM_EXT, ...COPY_EXT].some((e) => f.toLowerCase().endsWith(e))).map((f) => basename(f, extname(f)).toUpperCase()));
+  const refused = {};
+  const macroCalls = new Map();
+  const copyNotFound = new Set();
+  // Counts, for the coverage it reports, what the statement reader refused and which macros and COPY
+  // members a file uses.
+  function readStatements(src, path) {
+    const labels = new Map();
+    for (const st of readHlasmStatements(src).statements) {
+      if (st.inMacro) continue;
+      if (st.kind === 'statement' && st.name && !st.name.startsWith('&') && !st.name.startsWith('.')) labels.set(st.name.toUpperCase(), st);
+      const res = parseHlasmStatement(st);
+      if (res.status === 'unparsed' || res.status === 'unknown') (refused[res.kind] ||= { count: 0, first: `${path}:${st.line}` }).count++;
+      const name = res.kind === 'MACRO CALL' ? st.operation : NAMED_MACROS.has(res.kind) ? res.kind : null;
+      if (name) {
+        const m = macroCalls.get(name) || { name, calls: 0, definedInTree: members.has(name) };
+        m.calls++;
+        macroCalls.set(name, m);
+      }
+      if (res.kind === 'COPY') {
+        const member = String(res.status === 'parsed' ? res.node.member : (/^\s*([A-Z$#@][A-Z0-9$#@_]*)/i.exec(st.field || '') || [])[1] || '').toUpperCase();
+        if (member && !members.has(member)) copyNotFound.add(member);
+      }
+    }
+    return labels;
+  }
   const defines = new Map();
 
   const run = eachWithinMemory(all.filter(isAssembler), (f) => {
@@ -85,13 +129,20 @@ export function scanHlasm(root, opts = {}) {
     stats.assemblerFiles++;
     const path = relPath(root, f);
     for (const d of r.defines) if (!defines.has(d.name)) defines.set(d.name, { ...d, path });
+    const labels = readStatements(src, path);
     for (const use of r.operations) {
+      if (use.class === 'module-load') {
+        const { keywords } = macroOperands(use.field || '');
+        const at = keywords.has('EPLOC') ? 'EPLOC' : keywords.has('DE') ? 'DE' : null;
+        if (at) findings.push({ rule: 'hlasm-runtime-module-name', path, line: use.line, detail: `${use.name} ${at}=${keywords.get(at)} names the module by the address of its name, so the program that runs is whatever name that storage holds.${where(use)}` });
+        continue;
+      }
       if (use.name === 'MODESET') {
         if (use.stateChange) findings.push({ rule: 'hlasm-supervisor-state-change', path, line: use.line, detail: `MODESET ${use.stateChange} switches to ${STATE[use.stateChange.split('=')[0]]}.${where(use)}` });
         continue;
       }
       const rule = RULE_OF_CLASS[use.class];
-      if (rule) findings.push({ rule, path, line: use.line, detail: detail(use) });
+      if (rule) findings.push({ rule, path, line: use.line, detail: detail(use, labels) });
     }
     return src.length;
   }, { label: 'hlasm', maxBytes: opts.maxSourceBytes ?? Infinity });
@@ -117,5 +168,10 @@ export function scanHlasm(root, opts = {}) {
     }
   }
 
+  // Each statement the reader refused, each macro whose expansion was not read and each COPY member
+  // the tree does not hold is named: an operation inside any of them is one this set did not see.
+  if (Object.keys(refused).length) { stats.statementsNotRead = refused; stats.coverageIncomplete = true; }
+  if (copyNotFound.size) { stats.copyNotFound = [...copyNotFound].sort(); stats.coverageIncomplete = true; }
+  if (macroCalls.size) stats.macrosNotExpanded = [...macroCalls.values()].sort((a, b) => b.calls - a.calls || a.name.localeCompare(b.name)).slice(0, 50);
   return report('hlasm', { rules: HLASM_RULES, findings, stats, run });
 }

package/lib/sets/log.mjs

@@ -23,7 +23,7 @@ import { parseJcl } from '../jcl.mjs';
 import { compileStepOptions } from '../options.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 
 
 export const LOG_RULES = {
@@ -337,7 +337,7 @@ const memberOf = (file) => basename(String(file || ''), extname(String(file || '
 const runByJob = (batch, p, file) => batch.unsure
   || [String(p.id || '').toUpperCase(), memberOf(file)].some((n) => n && (batch.runs.has(n) || batch.runs.has(n.slice(0, 8))));
 
-export function scanLog(root, opts = {}) {
+export function* scanLogSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const files = tree.list().filter(isProgram).filter(inScope(opts));
   const findings = [];
@@ -364,7 +364,7 @@ export function scanLog(root, opts = {}) {
     return { rule: 'display-echoes-a-credential', detail: `${p.id} shows ${w.name} on the terminal to the person who typed it: not a log, but the screen and its scrollback keep it` };
   };
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     // Every EXEC CICS WRITE form, not the two this set started with. WRITE OPERATOR was added as a
@@ -404,3 +404,5 @@ export function scanLog(root, opts = {}) {
 
   return report('log', { rules: LOG_RULES, findings, stats, run });
 }
+
+export const scanLog = (root, opts = {}) => drive(scanLogSteps(root, opts));

package/lib/sets/priv.mjs

@@ -19,7 +19,7 @@
 import { inScope, isProgram, relPath } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { parseCsd } from '../csd.mjs';
 import { parseJcl } from '../jcl.mjs';
 import { loadSite } from '../site.mjs';
@@ -101,7 +101,7 @@ function under(dsn, prefixes) {
   });
 }
 
-export function scanPriv(root, opts = {}) {
+export function* scanPrivSteps(root, opts = {}) {
   const site = loadSite(root, opts.site || null, opts.tree);
   const tree = treeFor(root, opts);
   const files = tree.list().filter(inScope(opts));
@@ -140,7 +140,7 @@ export function scanPriv(root, opts = {}) {
   const diagnostics = [];
   const uss = [];
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     const path = relPath(root, f);
@@ -320,3 +320,5 @@ export function scanPriv(root, opts = {}) {
 
   return report('priv', { rules: PRIV_RULES, findings, stats, run });
 }
+
+export const scanPriv = (root, opts = {}) => drive(scanPrivSteps(root, opts));

package/lib/sets/secrets.mjs

@@ -0,0 +1,110 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Credentials written into COBOL source. The shapes are the ones rules/gitleaks-mainframe.toml gives
+// gitleaks, read from that file, so one list serves both and the build gate does not depend on
+// gitleaks being installed. Go's RE2 syntax is taken only where it means the same in JavaScript; any
+// other construct is refused when the file is read.
+import { readFileSync } from 'node:fs';
+import { inScope, isProgram, isCopybook, relPath } from '../sources.mjs';
+import { report } from '../kernel/ruleset.mjs';
+import { treeFor, noteUnread } from '../kernel/source-tree.mjs';
+import { eachWithinMemory } from '../kernel/memory.mjs';
+
+export const SECRETS_RULES = {
+  'credential-in-source': {
+    sev: 'high', evidence: 'construct', cwe: 'CWE-798', measured: false,
+    text: 'A credential is written into the source',
+    impact: 'Everyone who can read the source, its history or a listing can sign on with it, and changing it means changing and redeploying the program',
+    remedy: 'Read the credential at run time from where the site keeps them (a RACF-protected data set, a PassTicket, a Db2 trusted context or a CICS-supplied identity), and change the one already exposed',
+  },
+};
+
+// The TOML ids of the shapes that occur in COBOL programs and copybooks, and how a finding names each.
+const SHAPES = {
+  'cobol-value-credential': (m) => `${dataName(m) || 'a data item named for a credential'} has a literal VALUE`,
+  'embedded-sql-connect-password': () => 'EXEC SQL CONNECT gives a literal password',
+  'cics-signon-password': (m) => `EXEC CICS ${/\b(SIGNON|VERIFY|CHANGE)\b/i.exec(m)[1].toUpperCase()} gives a literal PASSWORD`,
+  'cics-new-password': (m) => `EXEC CICS ${/\b(SIGNON|CHANGE)\b/i.exec(m)[1].toUpperCase()} gives a literal NEWPASSWORD`,
+};
+
+const dataName = (m) => (/\b(?:0?[1-9]|[1-4][0-9]|77)\s+([A-Z0-9][A-Z0-9-]*)/i.exec(m) || [])[1]?.toUpperCase();
+
+const value = (raw) => {
+  const t = raw.trim();
+  if (t.startsWith("'''")) return t.slice(3, t.lastIndexOf("'''"));
+  if (t.startsWith('"')) return JSON.parse(t);
+  if (t.startsWith('[')) return [...t.matchAll(/'''(.*?)'''|"((?:[^"\\]|\\.)*)"/g)].map((x) => x[1] ?? JSON.parse(`"${x[2]}"`));
+  if (/^\d+$/.test(t)) return Number(t);
+  if (t === 'true' || t === 'false') return t === 'true';
+  throw new Error(`gitleaks-mainframe.toml: value not read: ${t}`);
+};
+
+// The [[rules]] tables of the file, each with its allowlist folded in.
+export function readToml(text) {
+  const rules = [];
+  let into = null;
+  for (const line of text.split('\n')) {
+    const l = line.trim();
+    if (!l || l.startsWith('#')) continue;
+    if (l === '[[rules]]') { rules.push((into = { allowlist: {} })); continue; }
+    if (l === '[rules.allowlist]') { into = rules.at(-1).allowlist; continue; }
+    if (l.startsWith('[')) { into = null; continue; }
+    const kv = /^([A-Za-z]+)\s*=\s*(.+)$/.exec(l);
+    if (kv && into) into[kv[1]] = value(kv[2]);
+  }
+  return rules;
+}
+
+// A gitleaks regex as a JavaScript one: a leading (?flags) group becomes the flags.
+export function jsRegex(re, extra = '') {
+  const lead = /^\(\?([ims]+)\)/.exec(re);
+  const body = lead ? re.slice(lead[0].length) : re;
+  if (/\(\?[a-zA-Z]|\(\?P<|\\[AzZpPQE]/.test(body)) throw new Error(`gitleaks-mainframe.toml: no JavaScript equivalent for ${re}`);
+  return new RegExp(body, [...new Set([...(lead ? lead[1] : ''), ...extra])].join(''));
+}
+
+const secretOf = (m, group) => (group ? m[group] : m.slice(1).find((g) => g !== undefined)) ?? m[0];
+
+export const SHAPE_RULES = readToml(readFileSync(new URL('../../rules/gitleaks-mainframe.toml', import.meta.url), 'utf8'))
+  .filter((r) => r.id in SHAPES)
+  .map((r) => ({
+    id: r.id,
+    pattern: jsRegex(r.regex, 'g'),
+    secretGroup: r.secretGroup || 0,
+    keywords: (r.keywords || []).map((k) => k.toLowerCase()),
+    allow: (r.allowlist.regexes || []).map((x) => jsRegex(x)),
+    allowTarget: r.allowlist.regexTarget || 'secret',
+  }));
+
+// Every shape in the text, at the line its first non-blank character is on.
+export function credentialsIn(text) {
+  const lower = text.toLowerCase();
+  const out = [];
+  for (const s of SHAPE_RULES) {
+    if (s.keywords.length && !s.keywords.some((k) => lower.includes(k))) continue;
+    for (const m of text.matchAll(s.pattern)) {
+      const target = s.allowTarget === 'match' ? m[0] : secretOf(m, s.secretGroup);
+      if (s.allow.some((a) => a.test(target))) continue;
+      const start = m.index + (m[0].length - m[0].trimStart().length);
+      out.push({ shape: s.id, line: text.slice(0, start).split('\n').length, detail: SHAPES[s.id](m[0]) });
+    }
+  }
+  return out;
+}
+
+export function scanSecrets(root, opts = {}) {
+  const tree = treeFor(root, opts);
+  const files = tree.list().filter((f) => isProgram(f) || isCopybook(f)).filter(inScope(opts));
+  const findings = [];
+  const stats = { filesScanned: 0, filesUnreadable: 0 };
+
+  const run = eachWithinMemory(files, (f) => {
+    let src;
+    try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
+    stats.filesScanned++;
+    const path = relPath(root, f);
+    for (const c of credentialsIn(src)) findings.push({ rule: 'credential-in-source', path, line: c.line, detail: `${c.detail} (${c.shape})` });
+    return src.length;
+  }, { label: 'secrets', maxBytes: opts.maxSourceBytes ?? Infinity });
+
+  return report('secrets', { rules: SECRETS_RULES, findings, stats, run });
+}

package/lib/sets/semantics.mjs

@@ -9,7 +9,7 @@ import { basename, extname } from 'node:path';
 import { inScope, isProgram, isJcl, relPath, ebcdicByte } from '../sources.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 import { loadSite } from '../site.mjs';
 import { parseJcl } from '../jcl.mjs';
 import { optionCards, optionTokens, compileStepOptions } from '../options.mjs';
@@ -206,7 +206,7 @@ function thruPairs(toks) {
 }
 const WHEN_WORDS = new Set(['THRU', 'THROUGH', 'ALSO', 'OR', 'NOT', 'ANY', 'OTHER', 'TRUE', 'FALSE']);
 
-export function scanSemantics(root, opts = {}) {
+export function* scanSemanticsSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const all = tree.list().filter(inScope(opts));
   const files = all.filter(isProgram);
@@ -359,7 +359,7 @@ export function scanSemantics(root, opts = {}) {
     }
   }
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     let r;
@@ -375,3 +375,5 @@ export function scanSemantics(root, opts = {}) {
 
   return report('semantics', { rules: SEMANTICS_RULES, findings, stats, run });
 }
+
+export const scanSemantics = (root, opts = {}) => drive(scanSemanticsSteps(root, opts));

package/lib/sets/web.mjs

@@ -22,7 +22,7 @@ import { parseCsd } from '../csd.mjs';
 import { classify } from './log.mjs';
 import { report } from '../kernel/ruleset.mjs';
 import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
-import { eachWithinMemory } from '../kernel/memory.mjs';
+import { drive, loopOver } from '../kernel/shared-pass.mjs';
 
 
 export const WEB_RULES = {
@@ -168,7 +168,7 @@ const HTTP_METHOD = /^(GET|POST|PUT|PATCH|DELETE)$/i;
 // the absence and the explicit NO are one finding, and PROTOCOL only changes what is in the clear.
 const encrypted = (svc) => /^(YES|CLIENTAUTH|CLIENTCERT)$/i.test(String(svc.ssl || ''));
 
-export function scanWeb(root, opts = {}) {
+export function* scanWebSteps(root, opts = {}) {
   const tree = treeFor(root, opts);
   const all = tree.list().filter(inScope(opts));
   const files = all.filter(isProgram);
@@ -323,7 +323,7 @@ export function scanWeb(root, opts = {}) {
     }
   }
 
-  const run = eachWithinMemory(files, (f) => {
+  const run = yield loopOver(files, (f) => {
     let src;
     try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
     // The whole set is about EXEC CICS WEB and EXEC CICS DOCUMENT. Nothing else needs parsing.
@@ -444,3 +444,5 @@ export function scanWeb(root, opts = {}) {
 
   return report('web', { rules: WEB_RULES, findings, stats, run });
 }
+
+export const scanWeb = (root, opts = {}) => drive(scanWebSteps(root, opts));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@portll/cobolwork",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "COBOL, JCL and CICS security analysis with zero runtime dependencies: reference-format parser, cross-program data flow, mainframe credential rules. Plugs into commitwork.",
   "license": "AGPL-3.0-or-later",
   "author": "Portll <john@portll.net>",