@portll/cobolwork 0.0.1 → 0.2.75

package/lib/tui/screen.mjs

@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { printable } from '../kernel/printable.mjs';
+
+export function createScreen(cols, rows) {
+  return {
+    cols, rows,
+    chars: Array.from({ length: rows }, () => Array(cols).fill(' ')),
+    styles: Array.from({ length: rows }, () => Array(cols).fill('')),
+  };
+}
+
+export function put(screen, row, col, text, style = '') {
+  if (row < 0 || row >= screen.rows) return;
+  const s = printable(text, Infinity);
+  for (let i = 0; i < s.length; i++) {
+    const c = col + i;
+    if (c < 0) continue;
+    if (c >= screen.cols) break;
+    screen.chars[row][c] = s[i];
+    screen.styles[row][c] = style;
+  }
+}
+
+export function styleRow(screen, row, style) {
+  if (row < 0 || row >= screen.rows) return;
+  screen.styles[row].fill(style);
+}
+
+export const screenText = (screen) => screen.chars.map((r) => r.join('').replace(/\s+$/, '')).join('\n');
+
+const SGR = {
+  color: { '': '', title: '\x1b[1m', warn: '\x1b[33m', dim: '\x1b[2m', sel: '\x1b[7m', hi: '\x1b[36m', crit: '\x1b[31;1m', high: '\x1b[31m' },
+  plain: { '': '', title: '\x1b[1m', warn: '\x1b[1m', dim: '', sel: '\x1b[7m', hi: '', crit: '\x1b[1m', high: '' },
+};
+
+function rowAnsi(screen, r, table) {
+  let out = '';
+  let current = null;
+  for (let c = 0; c < screen.cols; c++) {
+    const st = screen.styles[r][c];
+    if (st !== current) { out += '\x1b[0m' + (table[st] || ''); current = st; }
+    out += screen.chars[r][c];
+  }
+  return out + '\x1b[0m';
+}
+
+const rowKey = (screen, r) => screen.chars[r].join('') + '\u0000' + screen.styles[r].join(',');
+
+// Escape sequences that turn `prev` (or a blank terminal) into `next`, changed rows only.
+export function redraw(prev, next, { color = true } = {}) {
+  const table = color ? SGR.color : SGR.plain;
+  const whole = !prev || prev.cols !== next.cols || prev.rows !== next.rows;
+  let out = whole ? '\x1b[2J' : '';
+  for (let r = 0; r < next.rows; r++) {
+    if (!whole && rowKey(prev, r) === rowKey(next, r)) continue;
+    out += `\x1b[${r + 1};1H` + rowAnsi(next, r, table);
+  }
+  return out;
+}

package/lib/tui/terminal.mjs

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { emitKeypressEvents } from 'node:readline';
+import { keyToken } from './keys.mjs';
+
+const ALT_ON = '\x1b[?1049h';
+const ALT_OFF = '\x1b[?1049l';
+const CURSOR_HIDE = '\x1b[?25l';
+const CURSOR_SHOW = '\x1b[?25h';
+const RESET = '\x1b[0m';
+
+export function nodeTerminal({ input = process.stdin, output = process.stdout } = {}) {
+  let started = false;
+  const keyHandlers = [];
+  const resizeHandlers = [];
+  const size = () => ({ cols: output.columns || 80, rows: output.rows || 24 });
+  const onKeypress = (str, key) => {
+    const token = keyToken(str, key || {});
+    if (token) for (const h of keyHandlers) h(token);
+  };
+  const onResize = () => { for (const h of resizeHandlers) h(size()); };
+  return {
+    size,
+    start() {
+      if (started) return;
+      started = true;
+      emitKeypressEvents(input);
+      if (input.isTTY && input.setRawMode) input.setRawMode(true);
+      input.on('keypress', onKeypress);
+      if (output.on) output.on('resize', onResize);
+      input.resume();
+      output.write(ALT_ON + CURSOR_HIDE);
+    },
+    stop() {
+      if (!started) return;
+      started = false;
+      input.off('keypress', onKeypress);
+      if (output.off) output.off('resize', onResize);
+      if (input.isTTY && input.setRawMode) input.setRawMode(false);
+      input.pause();
+      output.write(RESET + CURSOR_SHOW + ALT_OFF);
+    },
+    write: (text) => { if (text) output.write(text); },
+    onKey: (h) => { keyHandlers.push(h); },
+    onResize: (h) => { resizeHandlers.push(h); },
+  };
+}

package/lib/utilities.mjs

@@ -0,0 +1,296 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// How the common z/OS utilities move data between their DD names. None of it can be derived from
+// the JCL: SYSUT1 is IEBGENER's input and SYSUT2 its output only because IBM says so, and a SORT
+// step's outputs are named in its control statements rather than on any DD. So it is a table, kept
+// by hand, and each entry names the pages it was written from.
+//
+// A program the table does not know has no copies, rather than guessed ones. A step whose control
+// statements are not in the job - SYSIN naming a library member - says so, because its copies are
+// then only the ones its DD names decide on their own.
+
+const upper = (s) => String(s).toUpperCase();
+
+// The DDs of a step by name, each followed by the datasets concatenated to it.
+export function ddGroups(step) {
+  const out = new Map();
+  let current = null;
+  for (const dd of step.dds) {
+    if (dd.name) {
+      current = upper(dd.name);
+      if (!out.has(current)) out.set(current, []);
+    }
+    if (current) out.get(current).push(dd);
+  }
+  return out;
+}
+
+// The in-stream lines of a control DD, and the first dataset in its concatenation, whose lines are
+// statements this reader never sees.
+function cards(group) {
+  const lines = [];
+  let unread = null;
+  for (const dd of group || []) {
+    if (dd.inStream) lines.push(...dd.inStream);
+    else if (dd.dsn && !unread) unread = dd.dsn;
+  }
+  return { lines, unread };
+}
+
+// Up to the first blank outside quotes: an operand field, where whatever follows is a remark.
+function upToBlank(s) {
+  let quoted = false;
+  for (let i = 0; i < s.length; i++) {
+    if (s[i] === "'") quoted = !quoted;
+    else if (!quoted && /\s/.test(s[i])) return s.slice(0, i);
+  }
+  return s;
+}
+
+// Splits on commas outside parentheses.
+function topLevel(s) {
+  const out = [];
+  let depth = 0, start = 0;
+  for (let i = 0; i < s.length; i++) {
+    if (s[i] === '(') depth++;
+    else if (s[i] === ')') depth = Math.max(0, depth - 1);
+    else if (s[i] === ',' && depth === 0) { out.push(s.slice(start, i)); start = i + 1; }
+  }
+  out.push(s.slice(start));
+  return out.map((x) => x.trim()).filter(Boolean);
+}
+
+const unwrap = (v) => (/^\(.*\)$/.test(v) ? v.slice(1, -1) : v);
+
+// DFSORT and DFSMSdfp utility statements: a label in column 1, or a comment if column 1 holds an
+// asterisk; the operation; operands up to the first blank, after which comes a remark. `continues`
+// says whether a line's operands carry on to the next line.
+function utilityStatements(lines, continues) {
+  const out = [];
+  let cur = null;
+  for (const { line, text } of lines) {
+    const s = text.slice(0, 71);
+    if (cur) {
+      const more = upToBlank(s.trimStart());
+      cur.operands += more;
+      if (!continues(more, text)) { out.push(cur); cur = null; }
+      continue;
+    }
+    if (/^\*/.test(s) || !s.trim()) continue;
+    const m = /^(\S+)?\s+(\S+)\s*(.*)$/.exec(s);
+    const [, label, op, rest] = m || [null, null, s.trim(), ''];
+    cur = { line, label: label || null, op: upper(op), operands: upToBlank(rest) };
+    if (!continues(cur.operands, text)) { out.push(cur); cur = null; }
+  }
+  if (cur) out.push(cur);
+  return out;
+}
+
+// IDCAMS and ADRDSSU commands: a hyphen at the end of a line continues the command after a blank,
+// a plus continues it and the value it is in, and /* */ encloses a comment.
+function commands(lines) {
+  const out = [];
+  let cur = null;
+  let comment = false;
+  for (const { line, text } of lines) {
+    let t = '';
+    const s = text.slice(0, 72);
+    for (let i = 0; i < s.length; i++) {
+      if (comment) { if (s[i] === '*' && s[i + 1] === '/') { comment = false; i++; } continue; }
+      if (s[i] === '/' && s[i + 1] === '*') { comment = true; i++; continue; }
+      t += s[i];
+    }
+    t = t.trim();
+    const join = /-$/.test(t) ? ' ' : /\+$/.test(t) ? '' : null;
+    if (join !== null) t = t.slice(0, -1).trim();
+    if (!cur) {
+      if (!t) continue;
+      cur = { line, text: t, join: '' };
+    } else cur.text += cur.join + t;
+    if (join === null) { out.push(cur); cur = null; } else cur.join = join;
+  }
+  if (cur) out.push(cur);
+  return out;
+}
+
+// The values of the first of `names` given as a keyword in a command, as in INFILE(IN) or
+// OUTDDNAME(TAPE1,TAPE2). A name only counts at the start of a word, so INDD is not LOGINDD.
+function param(text, names) {
+  const m = new RegExp(`(?:^|[\\s,])(?:${names.join('|')})\\s*\\(([^)]*)\\)`, 'i').exec(text);
+  return m ? m[1].split(/[\s,]+/).filter(Boolean).map((v) => upper(v.replace(/^'|'$/g, ''))) : null;
+}
+
+const who = (step) => `step ${step.name || '(unnamed)'}`;
+
+// ---------------------------------------------------------------------------------------------
+// The table. Each entry turns a step into moves: data read from `from` and written to `to`, where
+// an end is a DD name or, when a control statement names one itself, a dataset.
+
+// IEBGENER copies SYSUT1 to SYSUT2, and ICEGENER, DFSORT's replacement for it, reads the same DDs.
+// Control statements only edit records or split members, so the copy is the DDs' alone.
+//   https://www.ibm.com/docs/api/v1/content/zosbasics/com.ibm.zos.zdatamgmt/zsysprogc_utilities_IEBGENER.htm
+//   https://www.ibm.com/docs/en/zos/3.1.0?topic=performance-use-icegener-instead-iebgener
+function generCopy(step, dds) {
+  if (!dds.has('SYSUT1') || !dds.has('SYSUT2')) return { moves: [], notes: [] };
+  return { moves: [{ from: [{ dd: 'SYSUT1' }], to: [{ dd: 'SYSUT2' }], line: step.line }], notes: [] };
+}
+
+// DFSORT reads SORTIN for a sort or a copy and SORTINnn, nn from 00 to 99, for a merge, and writes
+// SORTOUT. OUTFIL writes the DDs its FNAMES operand names, SORTOFx or SORTOFxx for each suffix its
+// FILES operand gives, or SORTOUT when it gives neither. SORTDD= and SORTOUT= in DFSPARM rename the
+// defaults, and are not read. A line whose operand field ends in a comma, semicolon or colon
+// continues.
+//   https://www.ibm.com/docs/en/zos/3.1.0?topic=statements-program-dd
+//   https://www.ibm.com/docs/en/zos/2.5.0?topic=sets-jcl-merging-data-directly
+//   https://www.ibm.com/docs/en/zos/2.2.0?topic=statements-outfil-control
+//   https://www.ibm.com/docs/en/zos/3.1.0?topic=statements-general-coding-rules
+// SYNCSORT is here because it takes the same DD names and statements; this was not checked against
+// Precisely's own guide, which could not be retrieved.
+function sortCopies(step, dds) {
+  const { lines, unread } = cards(dds.get('SYSIN'));
+  const statements = utilityStatements(lines, (ops) => /[,;:]$/.test(ops));
+  const merge = statements.some((st) => st.op === 'MERGE' && !/(?:^|,)FIELDS=COPY\b/i.test(st.operands));
+  const inputs = [...dds.keys()].filter((n) => (unread
+    ? n === 'SORTIN' || /^SORTIN\d\d$/.test(n)
+    : merge ? /^SORTIN\d\d$/.test(n) : n === 'SORTIN'));
+  const from = inputs.map((dd) => ({ dd }));
+  const notes = unread ? [{ line: step.line, text: `${who(step)} runs ${step.pgm} with control statements in ${unread}, which this reader cannot see, so whether it merges and which OUTFIL data sets it writes are not known` }] : [];
+  const moves = dds.has('SORTOUT') ? [{ from, to: [{ dd: 'SORTOUT' }], line: step.line }] : [];
+  for (const st of statements) {
+    if (st.op !== 'OUTFIL') continue;
+    const fnames = /(?:^|,)FNAMES=(\([^)]*\)|[^,()]+)/i.exec(st.operands);
+    const files = /(?:^|,)FILES=(\([^)]*\)|[^,()]+)/i.exec(st.operands);
+    const to = [
+      ...(fnames ? topLevel(unwrap(fnames[1])).map((n) => ({ dd: upper(n) })) : []),
+      ...(files ? topLevel(unwrap(files[1])).map((x) => ({ dd: `SORTOF${upper(x)}` })) : []),
+    ];
+    moves.push({ from, to: to.length ? to : [{ dd: 'SORTOUT' }], line: st.line });
+  }
+  return { moves, notes };
+}
+
+// IDCAMS REPRO copies INFILE(ddname) or INDATASET(entryname) to OUTFILE(ddname) or
+// OUTDATASET(entryname), abbreviated IFILE, IDS, OFILE and ODS. The dataset forms name no DD:
+// IDCAMS allocates those itself.
+//   https://www.ibm.com/docs/en/zos/3.1.0?topic=commands-repro
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.3.0/com.ibm.zos.v2r3.idai200/da6i2253.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.idai200/continu.htm
+function reproCopies(step, dds) {
+  const { lines, unread } = cards(dds.get('SYSIN'));
+  const notes = unread ? [{ line: step.line, text: `${who(step)} runs IDCAMS with commands in ${unread}, which this reader cannot see, so any REPRO it runs is not known` }] : [];
+  const moves = [];
+  for (const cmd of commands(lines)) {
+    if (!/^REPRO\b/i.test(cmd.text)) continue;
+    const end = (ddNames, dsNames) => {
+      const dd = param(cmd.text, ddNames);
+      if (dd) return [{ dd: dd[0] }];
+      const ds = param(cmd.text, dsNames);
+      return ds ? [{ dsn: ds[0] }] : [];
+    };
+    moves.push({ from: end(['INFILE', 'IFILE'], ['INDATASET', 'IDS']), to: end(['OUTFILE', 'OFILE'], ['OUTDATASET', 'ODS']), line: cmd.line });
+  }
+  return { moves, notes };
+}
+
+// IEBCOPY COPY and COPYGRP name their output with OUTDD and their inputs with INDD, where an input
+// written (ddname,R) replaces members of the same name. An INDD= on a record of its own begins
+// another step of the COPY before it. A statement continues after a comma or a mark in column 72.
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_3.1.0/com.ibm.zos.v3r1.idau100/copy.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_3.1.0/com.ibm.zos.v3r1.idau100/copygrp.htm
+//   https://www.ibm.com/docs/en/zos/2.4.0?topic=ie-example-15-copy-groups-from-pdse-pdse-replace
+function iebcopyCopies(step, dds) {
+  const { lines, unread } = cards(dds.get('SYSIN'));
+  const notes = unread ? [{ line: step.line, text: `${who(step)} runs IEBCOPY with statements in ${unread}, which this reader cannot see, so what it copies is not known` }] : [];
+  const moves = [];
+  let out = null;
+  const statements = utilityStatements(lines, (ops, text) => /,$/.test(ops) || (text.length > 71 && text[71] !== ' '));
+  for (const st of statements) {
+    // A record holding only INDD= has no operation, so what reads as its operation is its operand.
+    const alone = /^INDD=/.test(st.op);
+    if (st.op === 'COPY' || st.op === 'COPYGRP') out = null;
+    else if (!alone) {
+      if (st.op !== 'SELECT' && st.op !== 'EXCLUDE') out = null;
+      continue;
+    }
+    const kw = new Map(topLevel(alone ? st.op + st.operands : st.operands).map((o) => {
+      const eq = o.indexOf('=');
+      return [upper(o.slice(0, eq)), o.slice(eq + 1)];
+    }));
+    if (kw.has('OUTDD')) out = upper(kw.get('OUTDD'));
+    if (!out || !kw.has('INDD')) continue;
+    const from = topLevel(unwrap(kw.get('INDD'))).map((i) => ({ dd: upper(unwrap(i).split(',')[0].trim()) }));
+    moves.push({ from, to: [{ dd: out }], line: st.line });
+  }
+  return { moves, notes };
+}
+
+// ADRDSSU DUMP reads the INDDNAME volume, or the LOGINDDNAME volumes of a logical dump, and writes
+// each OUTDDNAME, up to 255 of them. RESTORE reads the dump data set INDDNAME names and writes the
+// OUTDDNAME volumes. With no input DD a logical dump chooses its datasets by filter, and with no
+// output DD a logical restore puts them where the catalogue says, neither of which the job shows.
+//   https://www.ibm.com/docs/en/zos/3.1.0?topic=commands-command-syntax
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.adru000/r2319.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.adru000/r2321.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.4.0/com.ibm.zos.v2r4.adru000/r2327.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.adru000/dgt3u2165.htm
+//   https://www.ibm.com/docs/api/v1/content/SSLTBW_2.2.0/com.ibm.zos.v2r2.adru000/dgt3u2174.htm
+function dssCopies(step, dds) {
+  const { lines, unread } = cards(dds.get('SYSIN'));
+  const notes = unread ? [{ line: step.line, text: `${who(step)} runs ADRDSSU with commands in ${unread}, which this reader cannot see, so what it dumps or restores is not known` }] : [];
+  const moves = [];
+  for (const cmd of commands(lines)) {
+    const op = /^(DUMP|RESTORE)\b/i.exec(cmd.text);
+    if (!op) continue;
+    const dump = upper(op[1]) === 'DUMP';
+    const ins = param(cmd.text, ['INDDNAME', 'INDD', 'IDD']) || (dump && param(cmd.text, ['LOGINDDNAME', 'LOGINDD', 'LIDD'])) || [];
+    const outs = param(cmd.text, ['OUTDDNAME', 'OUTDD', 'ODD']) || [];
+    if (dump && !ins.length) notes.push({ line: cmd.line, text: `${who(step)} dumps datasets chosen by filter from the catalogue, so which ones it copies is not known here` });
+    if (!dump && !outs.length) notes.push({ line: cmd.line, text: `${who(step)} restores datasets to where the catalogue puts them, so what it writes is not known here` });
+    moves.push({ from: ins.map((dd) => ({ dd })), to: outs.map((dd) => ({ dd })), line: cmd.line });
+  }
+  return { moves, notes };
+}
+
+const TABLE = [
+  [['IEBGENER', 'ICEGENER'], generCopy],
+  [['SORT', 'ICEMAN', 'DFSORT', 'SYNCSORT'], sortCopies],
+  [['IDCAMS'], reproCopies],
+  [['IEBCOPY'], iebcopyCopies],
+  [['ADRDSSU'], dssCopies],
+];
+const BY_PROGRAM = new Map(TABLE.flatMap(([names, fn]) => names.map((n) => [n, fn])));
+
+export const UTILITIES = [...BY_PROGRAM.keys()];
+
+// What a step's program copies, as { copies, uses, notes }: the copies from dataset to dataset, how
+// the utility uses each DD statement it names ('read' or 'write'), and what could not be read. Null
+// for a program the table does not know.
+export function copiesOf(step) {
+  const utility = step.pgm ? upper(step.pgm) : null;
+  const fn = utility && BY_PROGRAM.get(utility);
+  if (!fn) return null;
+  const dds = ddGroups(step);
+  const { moves, notes } = fn(step, dds);
+
+  // An end naming a DD stands for every dataset concatenated under that name. A DD the step does
+  // not define is kept, with no dataset: a statement naming it still says where the data went.
+  const expand = (end) => {
+    if (!end.dd) return [{ dd: null, dsn: end.dsn }];
+    const group = dds.get(end.dd);
+    if (!group) return [{ dd: end.dd, dsn: null }];
+    return group.map((dd) => ({ dd: end.dd, dsn: dd.dsn || null, ...(dd.inStream ? { inStream: true } : {}) }));
+  };
+
+  const copies = [];
+  const uses = new Map();
+  const mark = (end, use) => {
+    for (const dd of (end.dd && dds.get(end.dd)) || []) if (uses.get(dd) !== 'write') uses.set(dd, use);
+  };
+  for (const m of moves) {
+    for (const end of m.from) mark(end, 'read');
+    for (const end of m.to) mark(end, 'write');
+    for (const from of m.from.flatMap(expand)) {
+      for (const to of m.to.flatMap(expand)) copies.push({ step: step.name, utility, from, to, line: m.line });
+    }
+  }
+  return { copies, uses, notes };
+}

package/lib/version.mjs

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+import { readFileSync } from 'node:fs';
+
+// Bumped when a field a consumer reads changes meaning or shape, so a reader can refuse a report it
+// was not written for rather than misread it. 2: filesScanned is the widest rule set, not programs.
+// 3: findings carry `evidence`, and setsIncomplete entries a `kind`.
+export const SCHEMA_VERSION = 3;
+
+// Names how taint moves through groups, so a finding that appears or disappears between two reports
+// can be attributed to the model rather than to the code scanned. 'byte-range': a tainted field
+// taints those bytes of its group, and the bytes are followed through MOVE, REDEFINES and CALL.
+export const FLOW_MODEL = 'byte-range';
+
+// The package version, read once: a row says which cobolwork produced it.
+export const TOOL_VERSION = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version;