@portll/cobolwork 0.2.140 → 0.3.0

package/README.md

@@ -34,7 +34,7 @@ follows. To run from a checkout instead:
 
     git clone https://github.com/Portll/cobolwork && cd cobolwork && npm link
 
-Node 18 or later. No dependencies, runtime or development. `npm link` puts `cobolwork` on your PATH;
+Node 22 or later. No dependencies, runtime or development. `npm link` puts `cobolwork` on your PATH;
 adding `bin/` to PATH does the same thing. GnuCOBOL is needed only to regrade the parser or validate
 benchmark cases. In a GitHub workflow, `uses: Portll/cobolwork@main` runs the build gate on a pull
 request and writes SARIF for code scanning: [docs/github-action.md](docs/github-action.md).
@@ -187,6 +187,11 @@ since a test that missed the route does not show it safe.
 `summary.handling`, because it is then a list of what to attack first. Upload it only where the RACF
 facts behind it may go. [docs/spec/reach.md](docs/spec/reach.md) §9 is the full specification.
 
+`COBOLWORK_EXECUTION` names reports `ironwork run --coverage` wrote from runs of the estate's own
+tests. A finding inside a paragraph they cover carries `executed`, the paragraph and how often the
+runs entered it, and `summary.byExecution` counts the findings in paragraphs entered and never
+entered ([docs/spec/evidence.md](docs/spec/evidence.md) §13.5).
+
 ### What each finding lets someone do, and the fix
 
 A defect finding carries two more facts, the same for every finding of its rule: what someone can do
@@ -339,13 +344,20 @@ the guard off, it just stops the host's load from deciding. A spawned scan inher
 
 ## Compliance
 
-Every rule is mapped to the clause of each framework that makes it an obligation, quoted verbatim:
+Every rule is mapped to the clause of each framework that makes it an obligation, quoted verbatim,
+and to a COBIT 2019 practice by identifier alone:
 
 | File | Instrument | Rules mapped |
 |---|---|---|
 | `rules/compliance-dora.json` | Regulation (EU) 2022/2554 (DORA) | 203 |
 | `rules/compliance-ffiec.json` | FFIEC IT Examination Handbook | 201, and 2 recorded as unmapped |
 | `rules/compliance-nist80053.json` | NIST SP 800-53 Rev. 5.2.0 | 201, and 2 recorded as unmapped |
+| `rules/compliance-cobit2019.json` | COBIT 2019 (ISACA), identifiers only | 201, and 2 recorded as unmapped |
+
+A COBIT 2019 row names the objective or practice and gives this project's own rationale; no ISACA
+text is reproduced, so reading what a practice says needs a copy of the framework. The practice is
+chosen at the NIST control, through a crosswalk in `diag/map-compliance.mjs`, so it cannot drift
+from the rule's NIST clause.
 
 `node diag/map-compliance.mjs` refuses to write a quote the cached instrument does not contain, and
 every scan carries the mapping as `ruleCompliance`. The clause choice is a judgement that no

package/THIRD-PARTY-NOTICES.md

@@ -89,8 +89,9 @@ its source URL and retrieval date:
 - **NIST SP 800-53 Rev. 5** — a work of the US Government.
 - **FFIEC IT Examination Handbook** — a work of the US Government.
 
-PCI DSS and the COBIT-derived SOX material are deliberately absent: they may not be redistributed,
-so they belong in a licensed feed rather than in this repository. Full instrument texts are never
+COBIT 2019 appears as objective and practice identifiers only, with this project's own rationale;
+no text of COBIT, which is ISACA's, is reproduced. PCI DSS is deliberately absent: it may not be
+redistributed. Full instrument texts are never
 committed (`feed/sources/` is ignored); the feed ships citations, not text.
 
 ## Vulnerability data

package/bin/cobolwork.mjs

@@ -16,6 +16,8 @@ import { build, buildSarif, buildSummaryLine } from '../lib/build.mjs';
 import { capabilities } from '../lib/capabilities.mjs';
 import { commitAt, revisionOf, toolRevision } from '../lib/revision.mjs';
 import { stampFingerprints } from '../lib/kernel/identity.mjs';
+import { pdsExportTree } from '../lib/kernel/source-tree.mjs';
+import { SITE_FILE } from '../lib/site.mjs';
 import { tally } from '../lib/kernel/findings.mjs';
 import { loadBaseline, applyBaseline, baselineEntries, BASELINE_FILE, SUPPRESSING } from '../lib/baseline.mjs';
 import { readReport } from '../lib/tui/model.mjs';
@@ -86,6 +88,9 @@ Options
                         baseline: the judgement each new entry records, and which rules it covers
   --advisories <file>[,<file>]  a customer's own advisory extract (JSON), loaded for this scan only;
                         never kept in the tree
+  --pds-export          scan: the directory holds partitioned data sets' members as files, as
+                        zowe zos-files download all-members writes them (hlq/llq/member.txt) or in a
+                        directory named for each data set; findings name DATA.SET/MEMBER
   --copylib <dir>[,<dir>]  copy libraries the estate keeps outside the repository, searched after the
                         tree's own copybooks, as COBCPY is; a copybook found there is read, not reported missing
   --report <file>       tui, explain: read a stored scan report instead of scanning
@@ -150,6 +155,7 @@ function parseArgs(argv) {
     else if (a === '--action') opts.action = value();
     else if (a === '--rule') opts.rule = list();
     else if (a === '--advisories') opts.advisoryFeeds = [...new Set(list().map(x => resolve(x)))];
+    else if (a === '--pds-export') opts.pdsExport = true;
     else if (a === '--copylib') opts.copylib = [...new Set(list().map(x => resolve(x)))];
     else if (a === '--report') opts.report = value();
     else if (a === '--keys') opts.keys = value();
@@ -198,6 +204,7 @@ function warnCoverage(report) {
 }
 
 const isLink = (p) => { try { return lstatSync(p).isSymbolicLink(); } catch { return false; } };
+const isFile = (p) => { try { return statSync(p).isFile(); } catch { return false; } };
 
 let journal = null;
 
@@ -262,6 +269,8 @@ if (opts.copylib && opts._.length && !COPYLIB_COMMANDS.includes(opts._[0])) {
 const notDirs = (opts.copylib || []).filter((d) => { try { return !statSync(d).isDirectory(); } catch { return true; } });
 if (notDirs.length) { process.stderr.write(`cobolwork: --copylib ${notDirs.join(', ')} is not a directory\n`); process.exit(2); }
 const systemDirs = opts.copylib || [];
+if (opts.pdsExport && opts._.length && opts._[0] !== 'scan') { process.stderr.write('cobolwork: --pds-export is for scan only\n'); process.exit(2); }
+if (opts.pdsExport && opts.repos) { process.stderr.write('cobolwork: --pds-export reads one export; --repos does not apply\n'); process.exit(2); }
 if (opts.json && opts._.length && opts._[0] !== 'capabilities') {
   process.stderr.write(`cobolwork: --json is for capabilities; every other command writes JSON unless --format says otherwise\n`);
   process.exit(2);
@@ -332,7 +341,13 @@ try {
     process.exitCode = evidenceCommand(target, opts, { toolVersion: VERSION, write: (s) => process.stdout.write(s) });
   } else if (command === 'scan' || command === 'flow') {
     const flowOpts = { repos, fullTrace: opts.fullTrace === true, systemDirs };
-    const report = command === 'flow' ? scanFlow(root, flowOpts) : scanAll(root, { ...flowOpts, only: opts.only, advisoryFeeds: opts.advisoryFeeds });
+    // The site file sits beside the members and is not one of them, so it is named rather than found.
+    const site = resolve(root, SITE_FILE);
+    const pds = opts.pdsExport ? pdsExportTree(root, { systemDirs }) : null;
+    const report = command === 'flow' ? scanFlow(root, flowOpts)
+      : pds ? scanAll(pds.root, { ...flowOpts, only: opts.only, advisoryFeeds: opts.advisoryFeeds, tree: pds, feedRoot: root, site: isFile(site) && !isLink(site) ? site : null })
+      : scanAll(root, { ...flowOpts, only: opts.only, advisoryFeeds: opts.advisoryFeeds });
+    if (pds) report.summary.pdsExport = pds.export;
     if (command === 'flow') {
       report.summary.identity = stampFingerprints(report.findings, { root });
       stampFingerprints(report.checked, { root });

package/lib/baseline.mjs

@@ -16,7 +16,7 @@
 // because the finding it would hide is evidence that the tree is not to be trusted. A baseline
 // passed from outside the tree can.
 import { readFileSync, existsSync } from 'node:fs';
-import { join } from 'node:path';
+import { join, resolve } from 'node:path';
 import { tally } from './kernel/findings.mjs';
 import { resolvesInside } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
@@ -54,22 +54,24 @@ export function validateEntry(e) {
 
 // { path, source, entries, problems }. An absent file is an empty baseline, not an error; an
 // unreadable one is a problem the report carries, never a silent empty.
-export function loadBaseline(root, { explicit = null, use = true, mayBeAbsent = false } = {}) {
+// A source tree held in memory (a git revision) supplies its own baseline file.
+export function loadBaseline(root, { explicit = null, use = true, mayBeAbsent = false, tree = null } = {}) {
   if (!use) return null;
-  const path = explicit || join(root, BASELINE_FILE);
-  if (!existsSync(path)) {
+  const held = !explicit && tree && tree.kind !== 'directory';
+  const path = explicit || (held ? resolve(tree.root, BASELINE_FILE) : join(root, BASELINE_FILE));
+  if (held ? !tree.contains(path) : !existsSync(path)) {
     if (!explicit) return null;
     if (mayBeAbsent) return { path, source: 'explicit', entries: [], problems: [] };
     return { path, source: 'explicit', entries: [], problems: [`${path} does not exist, so nothing was suppressed`] };
   }
   const out = { path, source: explicit ? 'explicit' : 'tree', entries: [], problems: [] };
   // The tree's own baseline is the tree's, and a link out of it is not followed.
-  if (!explicit && !resolvesInside(root, path)) {
+  if (!explicit && !held && !resolvesInside(root, path)) {
     out.problems.push(`${BASELINE_FILE} is a link that leads outside the tree, so nothing was suppressed`);
     return out;
   }
   let raw;
-  try { raw = JSON.parse(readFileSync(path, 'utf8')); } catch (err) {
+  try { raw = JSON.parse(held ? tree.bytes(path).toString('utf8') : readFileSync(path, 'utf8')); } catch (err) {
     out.problems.push(`${path} is not readable as JSON: ${printable(err.message, 120)}`);
     return out;
   }

package/lib/bms.mjs

@@ -15,7 +15,7 @@
 //
 // This reads text and opens nothing; the caller reads the file through the source tree, so
 // containment stays where it is enforced.
-import { parseOperands, splitOperands } from './jcl.mjs';
+import { parseOperands, splitOperands } from './cards.mjs';
 
 // Column 72 marks a continuation and 73 to 80 are the sequence field, so a statement ends at 71.
 const END_COLUMN = 71;

package/lib/capabilities.mjs

@@ -17,7 +17,7 @@ export const CAPABILITIES_SCHEMA_VERSION = 1;
 
 // Each command, what it takes and what it writes.
 export const COMMANDS = {
-  scan: { args: ['<path>'], options: ['--format', '--out', '--repos', '--only', '--quiet', '--full-trace', '--baseline', '--no-baseline', '--advisories', '--copylib', '--evidence'], documents: ['cobolwork', 'sarif'] },
+  scan: { args: ['<path>'], options: ['--format', '--out', '--repos', '--only', '--quiet', '--full-trace', '--baseline', '--no-baseline', '--advisories', '--copylib', '--pds-export', '--evidence'], documents: ['cobolwork', 'sarif'] },
   flow: { args: ['<path>'], options: ['--out', '--repos', '--quiet', '--full-trace', '--baseline', '--no-baseline', '--copylib', '--evidence'], documents: ['cobolwork-flow'] },
   inventory: { args: ['<path>'], options: ['--out', '--quiet', '--copylib', '--evidence'], documents: ['cobolwork-inventory'] },
   parse: { args: ['<file>'], options: ['--out'], documents: ['cobolwork-parse'] },
@@ -68,6 +68,7 @@ export function capabilities() {
       jcl: JCL_EXT,
       bms: BMS_EXT,
       extensionless: 'sniffed by content: a file with no extension is classified by what it holds',
+      pdsExport: 'scan --pds-export: every member classified by content, whatever extension the download gave it; text, --binary and --record downloads',
       zowe: ['zowe.config.json', 'zowe.config.user.json', '.mcp.json', '.vscode/mcp.json', '.cursor/mcp.json', '.claude/settings.json', '.claude/settings.local.json', '.gemini/settings.json', 'claude_desktop_config.json', '.vscode/settings.json', 'systems.json under a mock directory'],
     },
     evidenceKinds: Object.keys(EVIDENCE),

package/lib/cards.mjs

@@ -0,0 +1,66 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Card-image source: where a line's fields sit on the card, and how an operand field splits. COBOL
+// in fixed or variable reference format has a sequence area (columns 1-6), an indicator (7), program
+// text (8-72, or 8-250 in variable format) and what follows, which the compiler ignores. A JCL or
+// utility control statement is columns 1-72, with its sequence number after. Free-format COBOL is
+// text from end to end.
+
+export const TEXT_END = { fixed: 72, variable: 250 };
+export const STATEMENT_END = 72;
+
+export function cobolCard(line, format) {
+  const end = TEXT_END[format];
+  if (!end) return { sequence: '', indicator: '', text: line, ignored: '' };
+  return { sequence: line.slice(0, 6), indicator: line[6] || ' ', text: line.slice(7, end), ignored: line.slice(end) };
+}
+
+export function statementCard(line) {
+  return { text: line.slice(0, STATEMENT_END), sequence: line.slice(STATEMENT_END) };
+}
+
+// Splits an operand field on commas that are not inside parentheses or quotes. JCL, BMS and HLASM nest
+// both, and a naive split on comma turns DISP=(NEW,CATLG,DELETE) into three operands.
+export function splitOperands(s) {
+  const out = [];
+  let depth = 0, quoted = false, start = 0;
+  for (let i = 0; i < s.length; i++) {
+    const c = s[i];
+    if (quoted) { if (c === "'") { if (s[i + 1] === "'") i++; else quoted = false; } continue; }
+    if (c === "'") quoted = true;
+    else if (c === '(') depth++;
+    else if (c === ')') depth = Math.max(0, depth - 1);
+    else if (c === ',' && depth === 0) { out.push(s.slice(start, i)); start = i + 1; }
+  }
+  out.push(s.slice(start));
+  return out.filter((x, i, a) => x.length || i < a.length - 1);
+}
+
+// Operands are positional until the first KEY=VALUE, and keyword after it: JCL's EXEC takes its
+// procedure name positionally, and everything interesting on a DD is a keyword.
+export function parseOperands(field) {
+  const positional = [];
+  const keywords = new Map();
+  for (const raw of splitOperands(field)) {
+    const part = raw.trim();
+    if (!part) continue;
+    const eq = keywordSplit(part);
+    if (eq < 0) positional.push(part);
+    else keywords.set(part.slice(0, eq).toUpperCase(), part.slice(eq + 1));
+  }
+  return { positional, keywords };
+}
+
+// The first '=' that is not inside parentheses or quotes. DCB=(RECFM=FB,LRECL=80) is one keyword
+// whose value happens to contain more of them.
+function keywordSplit(part) {
+  let depth = 0, quoted = false;
+  for (let i = 0; i < part.length; i++) {
+    const c = part[i];
+    if (quoted) { if (c === "'") { if (part[i + 1] === "'") i++; else quoted = false; } continue; }
+    if (c === "'") quoted = true;
+    else if (c === '(') depth++;
+    else if (c === ')') depth = Math.max(0, depth - 1);
+    else if (c === '=' && depth === 0) return i;
+  }
+  return -1;
+}

package/lib/compliance.mjs

@@ -5,10 +5,10 @@
 // against each. The difference between them is this file, and it is the difference between a
 // tooling budget and a compliance budget.
 //
-// DORA, NIST SP 800-53 and the FFIEC IT Examination Handbook ship here because their instruments
-// may be reproduced: one is EU law, the other two US Government works. PCI DSS and the COBIT-derived SOX material may not be
-// redistributed, so those mappings belong in the licensed feed rather than in an AGPL repository.
-// The split is a licensing fact before it is a product decision.
+// DORA, NIST SP 800-53 and the FFIEC IT Examination Handbook ship with quotes because their
+// instruments may be reproduced: one is EU law, the other two US Government works. COBIT 2019 ships
+// as identifiers with this project's own rationale and no ISACA text. PCI DSS may not be
+// redistributed and is not mapped.
 //
 // A framework may decline to cover a rule. Neither NIST nor FFIEC has a control that genuinely
 // covers committing an LPAR name to a repository, and that is recorded as an unmapped rule with a
@@ -19,6 +19,7 @@ const FRAMEWORKS = [
   { id: 'dora', file: '../rules/compliance-dora.json' },
   { id: 'nist-800-53r5', file: '../rules/compliance-nist80053.json' },
   { id: 'ffiec', file: '../rules/compliance-ffiec.json' },
+  { id: 'cobit-2019', file: '../rules/compliance-cobit2019.json' },
 ];
 
 // A framework that will not parse is a framework this scan cannot map against, which is the same
@@ -64,7 +65,7 @@ const byRule = new Map();
 for (const d of loaded) {
   for (const r of d.rows) {
     if (!byRule.has(r.ruleId)) byRule.set(r.ruleId, []);
-    byRule.get(r.ruleId).push({ framework: r.framework, clause: r.clause, title: r.title });
+    byRule.get(r.ruleId).push({ framework: r.framework, clause: r.clause, ...(r.title ? { title: r.title } : {}) });
   }
 }