@portll/cobolwork 0.6.0 → 0.7.0

package/lib/db2/stmt/view.mjs

@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Db2 for z/OS CREATE VIEW, after the Db2 13 SQL Reference: the view's name and columns, its common
+// table expressions and fullselect kept as tokens, the check option, and the names its FROM clauses read.
+
+import { Db2Syntax } from '../cursor.mjs';
+import { readName } from './table.mjs';
+
+const isName = (t) => !!t && (t.t === 'word' || t.t === 'ident');
+const isWord = (t, u) => !!t && t.t === 'word' && (Array.isArray(u) ? u.includes(t.u) : t.u === u);
+const isOpen = (t) => !!t && t.t === 'op' && t.v === '(';
+const isClose = (t) => !!t && t.t === 'op' && t.v === ')';
+
+// What another dialect's view looks like at the token where a Db2 for z/OS one stops.
+function hint(t, after) {
+  if (!t) return '';
+  if (t.t === 'op' && t.v === '/') return " ('/' qualifies names in Db2 for i system naming, not Db2 for z/OS)";
+  if (t.t !== 'word') return '';
+  if (t.u === 'IF' && (isWord(after, 'NOT') || isWord(after, 'EXISTS'))) return ' (IF [NOT] EXISTS is not Db2 for z/OS syntax)';
+  if (t.u === 'MATERIALIZED') return ' (Db2 for z/OS has materialized query tables, made by CREATE TABLE, not materialized views)';
+  if (t.u === 'READ' && isWord(after, 'ONLY')) return ' (WITH READ ONLY is Oracle, not Db2 for z/OS)';
+  if (t.u === 'CONSTRAINT') return ' (a named check option is Oracle, not Db2 for z/OS)';
+  if (t.u === 'WITH' && isOpen(after)) return ' (view options in parentheses are not Db2 for z/OS syntax)';
+  return '';
+}
+
+function refuse(c, what) {
+  const t = c.peek();
+  throw new Db2Syntax(`expected ${what}${t ? ` at '${t.v ?? t.t}'` : ' at end of statement'}${hint(t, c.peek(1))}`, t || c.peek(-1));
+}
+
+function closing(toks, i) {
+  let depth = 0;
+  for (let k = i; k < toks.length; k++) {
+    if (isOpen(toks[k])) depth++;
+    else if (isClose(toks[k]) && --depth === 0) return k;
+  }
+  return toks.length;
+}
+
+const atTop = (toks, words) => {
+  let depth = 0;
+  return toks.some((t) => {
+    if (isOpen(t)) depth++;
+    else if (isClose(t)) depth--;
+    return depth === 0 && isWord(t, words);
+  });
+};
+
+// The words that end a FROM clause; FOR does too unless a period specification follows the table.
+const FROM_END = ['WHERE', 'GROUP', 'HAVING', 'ORDER', 'FETCH', 'OFFSET', 'LIMIT', 'UNION', 'EXCEPT', 'INTERSECT', 'WITH', 'OPTIMIZE', 'QUERYNO', 'SKIP'];
+
+// One FROM clause from i: a name that starts a table reference is recorded; TABLE (…), XMLTABLE (…) and
+// the like are read only for the subqueries inside them. Returns the index where the clause ends.
+function fromClause(toks, i, out) {
+  let start = true;
+  for (; i < toks.length; i++) {
+    const t = toks[i];
+    if (isWord(t, FROM_END) || (isWord(t, 'FOR') && !isWord(toks[i + 1], ['SYSTEM_TIME', 'BUSINESS_TIME']))) return i;
+    if ((t.t === 'op' && t.v === ',') || isWord(t, 'JOIN')) { start = true; continue; }
+    if (isOpen(t)) {
+      const j = closing(toks, i);
+      const inner = toks.slice(i + 1, j);
+      if (start && !atTop(inner, ['SELECT', 'UNION', 'EXCEPT', 'INTERSECT'])) fromClause(inner, 0, out);
+      else fromNames(inner, out);
+      i = j;
+      start = false;
+      continue;
+    }
+    if (!start) continue;
+    start = false;
+    if (!isName(t) || isWord(t, ['TABLE', 'FINAL', 'OLD']) || isOpen(toks[i + 1])) continue;
+    const parts = [t.v];
+    while (toks[i + 1] && toks[i + 1].t === 'op' && toks[i + 1].v === '.' && isName(toks[i + 2])) { parts.push(toks[i + 2].v); i += 2; }
+    out.push(parts.join('.'));
+  }
+  return i;
+}
+
+// The names in the FROM clause of every subselect in toks, subqueries included; a FROM that no SELECT
+// at its own depth precedes (EXTRACT (YEAR FROM D), IS DISTINCT FROM) is not one.
+function fromNames(toks, out) {
+  let select = false;
+  for (let i = 0; i < toks.length; i++) {
+    const t = toks[i];
+    if (isOpen(t)) {
+      const j = closing(toks, i);
+      fromNames(toks.slice(i + 1, j), out);
+      i = j;
+    } else if (isWord(t, 'SELECT')) select = true;
+    else if (isWord(t, 'FROM') && select && !isWord(toks[i - 1], 'DISTINCT')) i = fromClause(toks, i + 1, out) - 1;
+  }
+}
+
+function readColumnNames(c) {
+  return c.items().map((item) => {
+    if (item.length !== 1 || !isName(item[0])) throw new Db2Syntax(`expected an unqualified column name${item[0] ? ` at '${item[0].v}'` : ''}`, item[0] || c.peek(-1));
+    return item[0].v;
+  });
+}
+
+function readCommonTableExpression(c) {
+  const t = c.next();
+  if (!isName(t)) { c.pos -= t ? 1 : 0; refuse(c, 'a common table expression name'); }
+  const cte = { name: t.v, columns: c.isOp('(') ? readColumnNames(c) : null, fullselect: null };
+  c.expectWord('AS');
+  if (!c.isOp('(')) refuse(c, "'(' and the common table expression's fullselect");
+  cte.fullselect = c.group();
+  return cte;
+}
+
+function readView(c) {
+  c.expectWord('CREATE');
+  if (c.isWord('OR') && c.isWord('REPLACE', 1)) refuse(c, 'VIEW (Db2 for z/OS has no CREATE OR REPLACE VIEW)');
+  if (!c.isWord('VIEW')) refuse(c, 'VIEW');
+  c.next();
+  if (c.isWord('IF') && c.isWord(['NOT', 'EXISTS'], 1)) refuse(c, 'a view name');
+  const parts = readName(c);
+  if (parts.length > 3) refuse(c, 'a view name of at most three parts');
+  const node = { kind: 'CREATE VIEW', name: parts.join('.'), columns: null, with: [], fullselect: null, checkOption: null, tables: [] };
+  if (c.isOp('(')) node.columns = readColumnNames(c);
+  if (!c.word('AS')) refuse(c, node.columns ? 'AS' : "'(' or AS");
+  if (c.word('WITH')) {
+    if (c.isWord('RECURSIVE')) refuse(c, 'a common table expression (WITH RECURSIVE is not Db2 for z/OS syntax)');
+    do node.with.push(readCommonTableExpression(c)); while (c.op(','));
+  }
+  if (!c.isWord('SELECT') && !c.isOp('(')) refuse(c, 'a fullselect');
+  node.fullselect = c.until((t) => isWord(t, 'WITH'));
+  if (c.word('WITH')) {
+    const how = c.word(['CASCADED', 'LOCAL']);
+    if (!c.isWord('CHECK')) refuse(c, how ? 'CHECK OPTION' : 'CASCADED, LOCAL or CHECK OPTION');
+    c.next();
+    c.expectWord('OPTION');
+    node.checkOption = how ? how.u : 'CASCADED';
+    if (!c.done()) refuse(c, 'the end of the statement after CHECK OPTION');
+  }
+  const names = [];
+  for (const cte of node.with) fromNames(cte.fullselect, names);
+  fromNames(node.fullselect, names);
+  const local = new Set(node.with.map((cte) => cte.name.toUpperCase()));
+  node.tables = [...new Set(names)].filter((n) => n.includes('.') || !local.has(n.toUpperCase()));
+  return node;
+}
+
+export const parsers = {
+  'CREATE VIEW': (c) => readView(c),
+};

package/lib/diff.mjs

@@ -9,7 +9,7 @@ import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 // A diff finding is located by program rather than by line, so it orders on its own key. The text
 // comparison underneath it is the shared one.
 import { byText, evidenceMap } from './kernel/findings.mjs';
-import { directoryTree, gitTree } from './kernel/source-tree.mjs';
+import { directoryTree, gitTree, pdsExportRevision, pdsExportTree } from './kernel/source-tree.mjs';
 import { printable } from './kernel/printable.mjs';
 import { loadSite, SITE_FILE } from './site.mjs';
 import { loadBaseline, BASELINE_FILE, SUPPRESSING } from './baseline.mjs';
@@ -258,6 +258,7 @@ function siteChange(baseAt, headAt) {
   if (b.allowUnvalidatedPacks !== h.allowUnvalidatedPacks) parts.push(`allowUnvalidatedPacks ${h.allowUnvalidatedPacks}`);
   if (JSON.stringify(b.runtimeVersions) !== JSON.stringify(h.runtimeVersions)) parts.push('runtimeVersions');
   if (h.problems.length && h.problems.join() !== b.problems.join()) parts.push(`now reports: ${h.problems[0]}`);
+  if (h.warnings.length && h.warnings.join() !== b.warnings.join()) parts.push(`now warns: ${h.warnings[0]}`);
   return parts.length ? { file: SITE_FILE, change: printable(parts.join('; '), 400) } : null;
 }
 
@@ -307,14 +308,25 @@ export function withRefs(repo, baseRef, headRef, opts, fn) {
   }
 }
 
+// The members of a PDS export's working tree whose files git tracks or would track: an archive's
+// members by the archive's file.
+function exportScope(repo, tree) {
+  const scope = gitScope(repo);
+  return new Set(tree.list().filter((p) => scope.has(resolve(repo, tree.origin(p).replace(/\([^)]*\)$/, '')))));
+}
+
 // Reads each revision out of git without writing it anywhere; the head is the working tree when
-// headRef is null.
+// headRef is null. With pdsExport each side is read as a PDS export, members named for their data
+// sets on both.
 export function diffRefs(repo, baseRef, headRef = null, opts = {}) {
   const trees = { systemDirs: opts.systemDirs || [] };
-  const base = gitTree(repo, baseRef, trees);
-  const head = headRef ? gitTree(repo, headRef, trees) : repo;
-  const res = diffTrees(base, head, headRef ? opts : { ...opts, allow: gitScope(repo) });
+  const revision = (ref) => (opts.pdsExport ? pdsExportRevision(repo, ref, trees) : gitTree(repo, ref, trees));
+  const base = revision(baseRef);
+  const head = headRef ? revision(headRef) : opts.pdsExport ? pdsExportTree(repo, trees) : repo;
+  const allow = headRef ? null : opts.pdsExport ? exportScope(repo, head) : gitScope(repo);
+  const res = diffTrees(base, head, allow ? { ...opts, allow } : opts);
   res.summary.base = baseRef;
   res.summary.head = headRef || WORKING_TREE;
+  if (opts.pdsExport) res.summary.pdsExport = { base: base.export, head: head.export };
   return res;
 }

package/lib/evidence/cli.mjs

@@ -1,15 +1,26 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // `cobolwork evidence verify|seal|anchor`: the command half of docs/spec/evidence.md §7-9. Returns
 // the exit status; writes one JSON document through `write`.
-import { existsSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs';
+import { existsSync, readFileSync, readdirSync, unlinkSync, writeFileSync } from 'node:fs';
 import { join, resolve } from 'node:path';
 import { anchorGit, envelopeFor, seal, sealFiles, timeStampRequest } from './seal.mjs';
 import { parseAllowedSigners } from './sshsig.mjs';
 import { EvidenceRefusal, createExclusive, evidenceDirFrom, syncClose, writeAll } from './store.mjs';
 import { verifyEvidence } from './verify.mjs';
+import { EVIDENCE_RESULT_SCHEMA_VERSION } from '../version.mjs';
+import { evidenceClausesFor } from '../compliance.mjs';
 
 const VERBS = ['verify', 'seal', 'anchor', 'sign'];
 
+// The clauses a verified directory's evidence answers: a run journal's records, and the chain once
+// it verifies.
+function complianceOf(dir, verdict) {
+  const runs = join(dir, 'runs');
+  const journals = existsSync(runs) && readdirSync(runs).some((n) => n.endsWith('.jsonl'));
+  const present = journals ? ['journal-records', ...(verdict.verified ? ['journal-chain'] : [])] : [];
+  return present.flatMap((evidenceId) => evidenceClausesFor(evidenceId).map((c) => ({ evidenceId, ...c })));
+}
+
 function anchorNewest(dir, opts) {
   const seals = sealFiles(dir);
   if (!seals.length) throw new EvidenceRefusal('there is no seal to anchor; run evidence seal first');
@@ -46,7 +57,7 @@ export function evidenceCommand(verb, opts, { toolVersion, write, env = process.
   }
   const dir = evidenceDirFrom(opts, env);
   if (!dir) throw new EvidenceRefusal('evidence needs --evidence <dir> or COBOLWORK_EVIDENCE');
-  const emit = (doc) => write(`${JSON.stringify({ tool: 'cobolwork-evidence', verb, ...doc }, null, opts.quiet ? 0 : 1)}\n`);
+  const emit = (doc) => write(`${JSON.stringify({ tool: 'cobolwork-evidence', schemaVersion: EVIDENCE_RESULT_SCHEMA_VERSION, verb, ...doc }, null, opts.quiet ? 0 : 1)}\n`);
 
   if (verb === 'verify') {
     const allowed = opts.allowedSigners ? parseAllowedSigners(readFileSync(resolve(opts.allowedSigners), 'utf8')) : null;
@@ -56,7 +67,7 @@ export function evidenceCommand(verb, opts, { toolVersion, write, env = process.
     const file = (x) => (x ? resolve(x) : null);
     const cosign = opts.cosignBundle ? { bundle: file(opts.cosignBundle), key: file(opts.cosignKey), identity: opts.certificateIdentity || null, issuer: opts.certificateOidcIssuer || null, trustedRoot: file(opts.trustedRoot), ignoreTlog: opts.insecureIgnoreTlog === true } : null;
     const verdict = verifyEvidence(dir, { allowed, anchorGit: file(opts.anchorGit), anchorRef: opts.ref || null, anchorPin: opts.anchorPin || null, tsr: file(opts.tsr), tsaCa: file(opts.tsaCa), cosign, maxUnsealed: max, env });
-    emit(verdict);
+    emit({ ...verdict, compliance: complianceOf(dir, verdict) });
     return verdict.exit;
   }
   if (verb === 'seal') {

package/lib/evidence/record.mjs

@@ -33,7 +33,7 @@ export const KINDS = {
   verdict: { fields: { verdict: str, checks: any, relaxed: strings, exit: int }, required: ['verdict'] },
   output: { fields: { name: str, sha256: hex64, bytes: int, path: str, stdout: (x) => x === true }, required: ['name', 'sha256'] },
   // ironwork's run journal closes with the program's true RETURN-CODE, which may be negative.
-  close: { fields: { exit: (x) => x === null || Number.isSafeInteger(x), counts: any, durationMs: int, ledger: str }, required: ['exit'] },
+  close: { fields: { exit: (x) => x === null || Number.isSafeInteger(x), counts: any, durationMs: int, ledger: str, executor: (x) => ['vm', 'interpreter'].includes(x) }, required: ['exit'] },
   // Written by ironwork's run journal, in this format, so one verifier reads both tools.
   dd: { fields: { dd: str, event: (x) => ['open', 'close', 'end'].includes(x), mode: str, sha256: hex64, bytes: int }, required: ['dd', 'event'] },
   call: { fields: { program: str, from: str, sha256: hex64 }, required: ['program'] },

package/lib/evidence/store.mjs

@@ -1,8 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // The evidence directory on disk: where it may be, how its files are opened (never through a
 // symbolic link, owner-only), the ledger lock, and reading a chain file back line by line.
-import { randomBytes } from 'node:crypto';
-import { closeSync, constants, existsSync, fstatSync, fsyncSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, realpathSync, renameSync, statSync, unlinkSync, writeSync } from 'node:fs';
+import { closeSync, constants, existsSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, realpathSync, renameSync, statSync, unlinkSync, writeSync } from 'node:fs';
 import { basename, dirname, join, resolve, sep } from 'node:path';
 
 const { O_WRONLY, O_CREAT, O_EXCL, O_APPEND, O_NOFOLLOW } = constants;
@@ -139,47 +138,61 @@ export function lastLine(path) {
 const pidAlive = (pid) => { try { process.kill(pid, 0); return true; } catch (e) { return e.code === 'EPERM'; } };
 const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 
-// Moves the stale lock `seen` aside before removing it, so a lock a peer took after it was read is
-// put back instead of removed. Returns whether the stale lock was the one broken.
-export function breakStale(path, seen) {
-  const aside = `${path}.${process.pid}.${randomBytes(4).toString('hex')}`;
-  try { renameSync(path, aside); } catch { return false; }
-  let same = false;
-  try { same = statSync(aside).ino === seen.ino && readFileSync(aside, 'utf8') === seen.text; } catch { /* not the same */ }
-  if (!same) { try { linkSync(aside, path); } catch { /* a newer lock is in place */ } }
-  try { unlinkSync(aside); } catch { /* already gone */ }
-  return same;
+// A lock file as read: its inode and text, the pid it names and its age, or null when there is none.
+// A lock whose pid and time cannot be read was left by a writer that died between creating it and
+// writing them; it is aged from its modification time.
+function readLock(path, now) {
+  let fd;
+  try { fd = openRead(path); } catch { return null; }
+  try {
+    const st = fstatSync(fd);
+    const text = readAll(fd).toString('utf8');
+    const [p, t] = text.trim().split(/\s+/).map(Number);
+    const pid = Number.isSafeInteger(p) && p > 0 ? p : null;
+    const since = Number.isSafeInteger(t) && t > 0 ? t : null;
+    return { ino: st.ino, text, pid, age: pid !== null && since !== null ? now() - since : now() - st.mtimeMs };
+  } finally { closeSync(fd); }
 }
 
-// Takes ledger.lock, breaking one that is older than staleMs and whose holder is gone. A lock whose
-// pid and time cannot be read was left by a writer that died between creating it and writing them;
-// it is aged from its modification time. Returns { release, broken } or null when the lock could
-// not be had within waitMs.
+const isStale = (lock, staleMs) => lock !== null && lock.age > staleMs && (lock.pid === null || !pidAlive(lock.pid));
+
+// Replaces the stale lock `seen` with this writer's own. Breakers hold ledger.lock.break while they
+// break, so one breaks at a time, and rename it over the lock only while the lock is still the file
+// judged stale: a lock a peer took since is never moved. Returns whether this writer holds the lock.
+export function breakStale(path, seen, { staleMs = 60000, now = Date.now } = {}) {
+  const claim = `${path}.break`;
+  let fd;
+  try { fd = createExclusive(claim); } catch (e) {
+    if (e.code !== 'EEXIST') throw e;
+    if (isStale(readLock(claim, now), staleMs)) { try { unlinkSync(claim); } catch { /* already gone */ } }
+    return false;
+  }
+  writeAll(fd, `${process.pid} ${now()}\n`);
+  closeSync(fd);
+  const still = readLock(path, now);
+  if (still && still.ino === seen.ino && still.text === seen.text) {
+    try { renameSync(claim, path); return true; } catch { /* the claim is removed below */ }
+  }
+  try { unlinkSync(claim); } catch { /* already gone */ }
+  return false;
+}
+
+// Takes ledger.lock, breaking one that is older than staleMs and whose holder is gone. Returns
+// { release, broken } or null when the lock could not be had within waitMs.
 export function takeLock(path, { waitMs = 5000, staleMs = 60000, now = Date.now } = {}) {
   const deadline = now() + waitMs;
-  let broken = null;
+  const held = (broken) => ({ release: () => { try { unlinkSync(path); } catch { /* already gone */ } }, broken });
   for (;;) {
     try {
       const fd = createExclusive(path);
       writeAll(fd, `${process.pid} ${now()}\n`);
       closeSync(fd);
-      return { release: () => { try { unlinkSync(path); } catch { /* already gone */ } }, broken };
+      return held(null);
     } catch (e) {
       if (e.code !== 'EEXIST') throw e;
     }
-    let st = null;
-    let text = null;
-    try { st = statSync(path); text = readFileSync(path, 'utf8'); } catch { /* removed between the open and the read */ }
-    if (text !== null) {
-      const [p, t] = text.trim().split(/\s+/).map(Number);
-      const pid = Number.isSafeInteger(p) && p > 0 ? p : null;
-      const since = Number.isSafeInteger(t) && t > 0 ? t : null;
-      const age = pid !== null && since !== null ? now() - since : now() - st.mtimeMs;
-      if (age > staleMs && (pid === null || !pidAlive(pid)) && breakStale(path, { text, ino: st.ino })) {
-        broken = { holderPid: pid, ageMs: Math.max(0, Math.trunc(age)) };
-        continue;
-      }
-    }
+    const seen = readLock(path, now);
+    if (isStale(seen, staleMs) && breakStale(path, seen, { staleMs, now })) return held({ holderPid: seen.pid, ageMs: Math.max(0, Math.trunc(seen.age)) });
     if (now() >= deadline) return null;
     sleep(25);
   }

package/lib/exec-reading.mjs

@@ -0,0 +1,51 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// One reading of an EXEC block for every rule set and the flow engine, and the one the precompiler's
+// table names: each option's argument tokens, every word in order, and for EXEC CICS the command the
+// table names it and the direction each option's data moves. Cached per block, so a worker that
+// rebuilds a program's parse reads each block exactly as the main thread does.
+import { cicsCommand } from './precompile-cics.mjs';
+import { CICS_COMMANDS, CICS_EVERY_COMMAND } from './cics-commands.mjs';
+
+const readings = new WeakMap();
+
+// `opts` maps each option to the tokens inside its parentheses, the first time it is written; `words`
+// is every word of the block. `verb` and `sub` are the command's first two words: the table's name
+// where it has one (SEND MAP however its options are ordered), else the block's first two words.
+export function execReading(exec) {
+  let r = readings.get(exec);
+  if (r) return r;
+  const toks = exec.toks;
+  const opts = new Map();
+  const words = [];
+  const top = [];
+  let depth = 0;
+  for (let i = 0; i < toks.length; i++) {
+    const t = toks[i];
+    if (t.t === 'sep') { depth += t.v === '(' ? 1 : t.v === ')' ? -1 : 0; continue; }
+    if (t.t !== 'word') continue;
+    words.push(t.u);
+    if (depth === 0) top.push({ word: t.u });
+    if (toks[i + 1] && toks[i + 1].t === 'sep' && toks[i + 1].v === '(') {
+      const inner = [];
+      let d = 1;
+      for (let k = i + 2; k < toks.length && d > 0; k++) {
+        if (toks[k].t === 'sep') { d += toks[k].v === '(' ? 1 : -1; if (d === 0) break; continue; }
+        inner.push(toks[k]);
+      }
+      if (!opts.has(t.u)) opts.set(t.u, inner);
+    }
+  }
+  const command = exec.kind === 'CICS' && top.length ? cicsCommand(top) : null;
+  const named = command ? command.split(' ') : [];
+  const entry = command ? CICS_COMMANDS[command] : null;
+  r = {
+    opts, words, command,
+    verb: named[0] || words[0],
+    sub: named.length > 1 ? named[1] : words[1],
+    // The way an option's data moves, as the table records it: sends, receives, both, label or none;
+    // null for an option or a command the table does not hold.
+    direction: (option) => (entry ? entry.options[option] || CICS_EVERY_COMMAND[option] || entry.anyOption || null : null),
+  };
+  readings.set(exec, r);
+  return r;
+}

package/lib/execution.mjs

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // Execution: whether the estate's own runs entered the paragraph a finding is in, from the
 // reports `ironwork run --coverage` writes. A finding in a paragraph a run entered is code the tests
-// exercise; one in a paragraph no run entered is code they never reach. See docs/spec/evidence.md §13.4 (execution coverage).
+// exercise; one in a paragraph no run entered is code they never reach. See docs/spec/evidence.md §13.5 (execution coverage).
 import { readFileSync } from 'node:fs';
 import { basename, delimiter, join } from 'node:path';
 
@@ -11,7 +11,8 @@ export function executionFeedPaths(opts = {}) {
 }
 
 // One report: per PROGRAM-ID, each paragraph's source line, name and how often a run entered it.
-// Shape (ironwork's): { programs: [{ program, detail: [{ name, line, entered }] }] }.
+// Shape (ironwork's): { programs: [{ program, detail: [{ name, line, entered }] }] }, as
+// schema/cobolwork-execution.schema.json describes it.
 export function loadExecutionFeed(path) {
   const feed = { file: basename(path), programs: new Map(), problem: null };
   let doc;

package/lib/explain.mjs

@@ -8,6 +8,7 @@ import { REGISTRY } from './kernel/registry.mjs';
 import { WHO_ACTS } from './tui/model.mjs';
 import { verificationPlan } from './verify.mjs';
 import { cobolCard, statementCard } from './cards.mjs';
+import { EXPLAIN_SCHEMA_VERSION } from './version.mjs';
 
 const HIDDEN_RULES = new Set(Object.keys(REGISTRY.find((s) => s.name === 'hidden').rules));
 
@@ -151,6 +152,7 @@ export function explainFinding(report, fingerprint, { root }) {
 
   return {
     tool: 'cobolwork-explain',
+    schemaVersion: EXPLAIN_SCHEMA_VERSION,
     carriesSource: true,
     note: NOTE,
     fingerprint,

package/lib/exploitability.mjs

@@ -13,6 +13,7 @@ import { kindsOf } from './consequence.mjs';
 import { WHO } from './sets/flow.mjs';
 import { entryName, accessOfEntry, accessFromEntry, privilegedFromEntry } from './reach.mjs';
 import { verifyEvidence } from './evidence/verify.mjs';
+import { versionProblem, unknownKeys, ignoredKey } from './kernel/version-key.mjs';
 
 export const HANDLING = 'This report joins routes an attacker can drive with the estate\'s own access facts. It holds no source '
   + 'text, but it is a list of what to attack first: keep it where the access records it was built from may go, and do not '
@@ -20,7 +21,7 @@ export const HANDLING = 'This report joins routes an attacker can drive with the
 
 // A system response is not input, but the caller who provokes the failure is the one who reads it back.
 const DRIVER = { ...WHO, 'system-response': 'A web caller, who can provoke the failure and reads the code sent back' };
-const AT_REST = { 'file-record': 'the file record', database: 'the database row' };
+const AT_REST = { 'file-record': 'the file record', 'cics-queue': 'the queue item', database: 'the database row' };
 
 const at = (x) => `${x.file}:${x.line}`;
 
@@ -167,6 +168,9 @@ export function witnessFeedPaths(opts = {}) {
 }
 
 const OUTCOMES = new Set(['reproduced', 'not-reproduced']);
+export const WITNESS_VERSION = 1;
+const WITNESS_KEYS = new Set(['$schema', 'version', 'witness', 'recorded', 'results']);
+const RESULT_KEYS = new Set(['outcome', 'by', 'on', 'system', 'reference', 'evidence']);
 const str = (v) => typeof v === 'string' && v.length > 0;
 const isoDate = (v) => typeof v === 'string' && /^\d{4}-\d{2}-\d{2}$/.test(v);
 
@@ -198,7 +202,7 @@ function witnessedRun(ev, base, verdicts) {
 //   { witness, recorded, results: { FINGERPRINT: { outcome: "reproduced"|"not-reproduced", by, on, system, reference? } } }
 // Only those fields are read. What was sent to the system is not one of them.
 export function loadWitnessFeed(path, { root = null } = {}) {
-  const feed = { file: basename(path), witness: null, recorded: null, results: new Map(), refused: [], problem: null };
+  const feed = { file: basename(path), witness: null, recorded: null, results: new Map(), refused: [], warnings: [], problem: null };
   let real;
   try { real = realpathSync(path); } catch (e) { feed.problem = `could not be read (${e.code || e.name})`; return feed; }
   if (root) {
@@ -207,11 +211,16 @@ export function loadWitnessFeed(path, { root = null } = {}) {
   }
   let doc;
   try { doc = JSON.parse(readFileSync(real, 'utf8')); } catch (e) { feed.problem = `is not JSON (${e.message})`; return feed; }
+  if (!doc || typeof doc !== 'object' || Array.isArray(doc)) { feed.problem = 'is not a JSON object'; return feed; }
+  feed.problem = versionProblem(doc, WITNESS_VERSION);
+  if (feed.problem) return feed;
   if (!str(doc.witness)) { feed.problem = 'does not say whose tests it holds'; return feed; }
   if (!isoDate(doc.recorded)) { feed.problem = 'does not say when it was recorded'; return feed; }
   if (!doc.results || typeof doc.results !== 'object' || Array.isArray(doc.results)) { feed.problem = 'holds no results'; return feed; }
   feed.witness = doc.witness;
   feed.recorded = doc.recorded;
+  const extra = new Set(Object.values(doc.results).flatMap((r) => (r && typeof r === 'object' ? unknownKeys(r, RESULT_KEYS) : [])));
+  feed.warnings = [...unknownKeys(doc, WITNESS_KEYS).map((k) => ignoredKey(feed.file, k)), ...[...extra].sort().map((k) => ignoredKey(`${feed.file} results`, k))];
   const verdicts = new Map();
   for (const [fingerprint, r] of Object.entries(doc.results)) {
     const why = !r || typeof r !== 'object' ? 'is not an object'

package/lib/hlasm/asm/data.mjs

@@ -3,7 +3,7 @@
 // type and type extension, program type, length, scale and exponent modifiers, and nominal value.
 // https://www.ibm.com/docs/en/hla-and-tf/1.6.0?topic=statements-dc-instruction
 import { HlasmSyntax, splitOperands } from '../operands.mjs';
-import { readExpression } from '../expr.mjs';
+import { parseExpression, readExpression } from '../expr.mjs';
 
 const EXTENSIONS = {
   C: 'AEU', G: '', X: '', B: '', F: 'D', H: '', E: 'HBD', D: 'HBD', L: 'HBDQ', P: '', Z: '',
@@ -110,7 +110,19 @@ function operands(st, nominalRequired) {
   });
 }
 
+// A channel command word: command code, data address, flags and data count, each an expression.
+// https://www.ibm.com/docs/en/hla-and-tf/1.6.0?topic=statements-ccw-ccw0-instructions
+function ccw(st) {
+  const list = splitOperands(st.field || '');
+  if (list.length !== 4) throw new HlasmSyntax(`${st.operation} takes a command code, data address, flags and data count, not ${list.length} operands`);
+  const [command, address, flags, count] = list.map((text) => parseExpression(text.trim()));
+  return { kind: st.operation, command, address, flags, count };
+}
+
 export const parsers = {
   DC: (st) => ({ kind: 'DC', operands: operands(st, true) }),
   DS: (st) => ({ kind: 'DS', operands: operands(st, false) }),
+  CCW: ccw,
+  CCW0: ccw,
+  CCW1: ccw,
 };

package/lib/hlasm/asm/sections.mjs

@@ -129,7 +129,9 @@ function dxd(st) {
 
 export const parsers = {
   CSECT: (st) => noOperand('CSECT', st, false),
-  DSECT: (st) => noOperand('DSECT', st, true),
+  // An unnamed dummy section, like an unnamed COM, is defined when the name entry is omitted.
+  // https://www.ibm.com/docs/en/hla-and-tf/1.6.0?topic=sections-unnamed-section
+  DSECT: (st) => noOperand('DSECT', st, false),
   RSECT: (st) => noOperand('RSECT', st, false),
   COM: (st) => noOperand('COM', st, false),
   START: (st) => {

package/lib/hlasm/instr.mjs

@@ -27,6 +27,26 @@ function shapeOf(pattern) {
 
 export const formatFamily = (format) => String(format || '').replace(/-[a-z]$/, '');
 
+// An explicit length is assembled into a length field: 8 bits for L, 4 bits for L1 and L2, each
+// holding the length less one, so 0 to 256 and 0 to 16 bytes. Checked when the length is a
+// constant; a symbol's value needs the symbol table.
+const LENGTH_FIELD = { L: 256, L1: 16, L2: 16 };
+function constant(e) {
+  if (!e) return null;
+  if (e.t === 'num') return e.v;
+  if (e.t === 'sdt' && e.type === 'X') return /^[0-9A-F]+$/i.test(e.text) ? parseInt(e.text, 16) : null;
+  if (e.t === 'sdt' && e.type === 'B') return /^[01]+$/.test(e.text) ? parseInt(e.text, 2) : null;
+  if (e.t === 'paren') return constant(e.e);
+  if (e.t === 'unary') { const v = constant(e.e); return v === null ? null : e.op === '-' ? -v : v; }
+  if (e.t === 'bin') {
+    const l = constant(e.l);
+    const r = constant(e.r);
+    if (l === null || r === null) return null;
+    return { '+': l + r, '-': l - r, '*': l * r, '/': r ? Math.trunc(l / r) : 0 }[e.op] ?? null;
+  }
+  return null;
+}
+
 export function parseInstruction(st, row) {
   const shape = shapeOf(row.operands || '');
   // An instruction with no operands takes whatever follows it as remarks: SAM24  Back to AMODE 24.
@@ -42,6 +62,11 @@ export function parseInstruction(st, row) {
     if (!s.parts) return { operand: s.name, e: parseExpression(text) };
     const a = parseAddress(text);
     if (a.parts && a.parts.length > s.parts) throw new HlasmSyntax(`operand ${i + 1} of ${st.operation} (${s.name}) has ${a.parts.length} parts in parentheses, not at most ${s.parts}`);
+    const field = /^D\d*\((L\d?),/.exec(s.name)?.[1];
+    const length = field && a.parts ? constant(a.parts[0]) : null;
+    if (length !== null && (length < 0 || length > LENGTH_FIELD[field])) {
+      throw new HlasmSyntax(`operand ${i + 1} of ${st.operation} has length ${length}, and ${field} holds 0 to ${LENGTH_FIELD[field]}`);
+    }
     return { operand: s.name, ...a };
   });
   return { kind: 'INSTRUCTION', mnemonic: st.operation, format: row.format, length: row.length, operands };

package/lib/hlasm/macro/authorization.mjs

@@ -1,18 +1,24 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
 // Parsers for the z/OS authorization macros MODESET, TESTAUTH, and RACROUTE.
-import { HlasmSyntax, NAME } from '../operands.mjs';
+import { HlasmSyntax, NAME, remoteForm } from '../operands.mjs';
+import { MVS38_EXTKEYS, mvs38Form } from '../mvs38.mjs';
 
 const MODESET_EXTKEYS = new Set(['ZERO', 'TCB', 'RBT1', 'RBT234', 'KEY2', 'KEY3', 'KEY4', 'KEY7']);
 const MODESET_KEYS = new Set(['ZERO', 'NZERO']);
 const MODESET_MODES = new Set(['PROB', 'SUP']);
-const MODESET_KNOWN = new Set(['EXTKEY', 'KEYADDR', 'KEYREG', 'SAVEKEY', 'WORKREG', 'RELATED', 'KEY', 'MODE']);
+const MODESET_KNOWN = new Set(['EXTKEY', 'KEYADDR', 'KEYREG', 'SAVEKEY', 'WORKREG', 'RELATED', 'KEY', 'MODE', 'MF']);
+// The list and execute forms of the MODESET that generates an SVC (z/OS 3.1 MVS Programming:
+// Authorized Assembler Services Reference, Volume 3): the list form holds KEY and MODE, the execute
+// form names the list.
+const MODESET_LIST_KNOWN = new Set(['KEY', 'MODE', 'RELATED', 'MF']);
+const MODESET_EXECUTE_KNOWN = new Set(['RELATED', 'MF']);
 
 const TESTAUTH_KNOWN = new Set(['FCTN', 'STATE', 'KEY', 'RBLEVEL', 'BRANCH']);
 const TESTAUTH_STATES = new Set(['YES', 'NO']);
 
 const RACROUTE_REQUESTS = new Set(['AUTH', 'FASTAUTH', 'DEFINE', 'DIRAUTH', 'EXTRACT', 'LIST', 'SIGNON', 'STAT', 'TOKENBLD', 'TOKENMAP', 'TOKENXTR', 'VERIFY', 'VERIFYX', 'AUDIT']);
 
-function parseModeset(st, ops) {
+function parseModeset(st, ops, opts) {
   const { keywords } = ops;
   const node = { kind: 'MODESET', key: null, mode: null, extkey: null, keywords: {} };
 
@@ -23,9 +29,23 @@ function parseModeset(st, ops) {
     node.keywords[k] = v;
   }
 
+  if (keywords.has('MF')) {
+    const { form, list } = remoteForm(keywords.get('MF'), 'MF');
+    const known = form === 'L' ? MODESET_LIST_KNOWN : MODESET_EXECUTE_KNOWN;
+    for (const k of keywords.keys()) {
+      if (!known.has(k)) throw new HlasmSyntax(`MODESET: the ${form === 'L' ? 'list' : 'execute'} form does not take ${k}`);
+    }
+    if (form === 'L' && !keywords.has('KEY') && !keywords.has('MODE')) throw new HlasmSyntax('MODESET: the list form needs KEY or MODE');
+    node.mf = form;
+    if (list) node.list = list;
+  }
+
   if (keywords.has('EXTKEY')) {
     const v = keywords.get('EXTKEY').trim().toUpperCase();
-    if (!MODESET_EXTKEYS.has(v)) {
+    if (MVS38_EXTKEYS.has(v)) {
+      mvs38Form(opts, node, 'MODESET', `EXTKEY=${v}`);
+      node.pswKey = MVS38_EXTKEYS.get(v);
+    } else if (!MODESET_EXTKEYS.has(v)) {
       throw new HlasmSyntax(`MODESET: invalid EXTKEY value ${v}`);
     }
     node.extkey = v;