@portll/cobolwork 0.5.0 → 0.7.0

package/lib/parser.mjs

@@ -210,7 +210,8 @@ function evaluateCondition(text, defines) {
   return null;
 }
 
-export function normalize(src, format, defines, std) {
+// A debugging line (D in the indicator) is source under WITH DEBUGGING MODE and a comment otherwise.
+export function normalize(src, format, defines, std, debugging = false) {
   // Micro Focus reads a free-form line with * or / in column 1 as a comment, as cobc -std=mf does.
   const columnOneComments = std === 'mf' || std === 'mf-strict';
   const phys = src.split(/\r?\n/);
@@ -310,7 +311,7 @@ export function normalize(src, format, defines, std) {
       text = l.slice(7, 7 + width);
       if (!FIXED_INDICATORS.has(indicator)) { diags.push({ kind: 'invalid-indicator', line }); indicator = ' '; }
     }
-    if (indicator === '*' || indicator === '/' || indicator === 'D' || indicator === 'd') continue;
+    if (indicator === '*' || indicator === '/' || ((indicator === 'D' || indicator === 'd') && !debugging)) continue;
     if (fmt === 'free' && (/^\s*\*>/.test(text) || (columnOneComments && /^[*/]/.test(text)))) continue;
     // AUTHOR, INSTALLATION, DATE-WRITTEN, DATE-COMPILED, SECURITY and REMARKS take a comment-entry:
     // any text, a COPY or an apostrophe included. It runs to the end of the header's line, and in
@@ -409,8 +410,9 @@ export function tokenize(norm, file) {
       if (c === ' ' || c === '\t' || c === '\f' || c === '\r') { i++; continue; }
       if (exec && exec.kind === 'SQL' && c === '-' && s[i + 1] === '-') break;
       if (picPending) {
+        // = is no PICTURE symbol, so == closes the pseudo-text a picture string ends.
         let j = i;
-        while (j < n && s[j] !== ' ') j++;
+        while (j < n && s[j] !== ' ' && !(s[j] === '=' && s[j + 1] === '=')) j++;
         let pic = s.slice(i, j);
         if (pic.toUpperCase() === 'IS') { emit(tok('word', pic, line, file)); i = j; continue; }
         let trailingPeriod = false;
@@ -853,7 +855,7 @@ function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
   // A copybook is read in the format in force where the COPY statement sits, which a >>SOURCE
   // directive earlier in the including file may have changed from the file's starting format.
   const fmt = detectFormat(src) === 'terminal' && ctx.copyFormat === 'auto' ? 'terminal' : (at.fmt || (ctx.copyFormat !== 'auto' ? ctx.copyFormat : (inheritedFormat || ctx.mainFormat)));
-  const norm = normalize(src, fmt, ctx.defines, ctx.std);
+  const norm = normalize(src, fmt, ctx.defines, ctx.std, ctx.debugging);
   // A tag written against other text - :TAG:-FIELD, FS-(), 'X'-CLE - is replaced in the text, as the
   // compiler does, so the text around it joins the replacement into one word. A literal is taken as a
   // tag only where it touches a word; elsewhere it is replaced token for token like any operand.
@@ -896,6 +898,53 @@ function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
   return out;
 }
 
+// REPLACE ==:TAG:== BY ==text== replaces the colon-delimited operand wherever the program text holds
+// it, inside a word or a PICTURE as much as standing alone (IBM Enterprise COBOL, REPLACE statement,
+// replacement rules). Like a tag in COPY REPLACING it is replaced in the text, from the statement on
+// until REPLACE OFF or a REPLACE that does not name it again; a REPLACE itself is left as written.
+const TAG = /^:[A-Za-z0-9_-]+:$/;
+
+function replaceStatementAt(text, at) {
+  const rest = text.slice(at);
+  const off = /^REPLACE\s+OFF\s*\./i.exec(rest);
+  if (off) return { end: at + off[0].length, off: true, tags: [] };
+  const head = /^REPLACE(\s+ALSO)?\s*/i.exec(rest);
+  let k = head[0].length;
+  const tags = [];
+  let pairs = 0;
+  for (;;) {
+    const pair = /^(?:(?:LEADING|TRAILING)\s+)?==([\s\S]*?)==\s*BY\s*==([\s\S]*?)==\s*/i.exec(rest.slice(k));
+    if (!pair) break;
+    pairs++;
+    if (TAG.test(pair[1].trim())) tags.push([pair[1].trim(), pair[2].trim().replace(/\s+/g, ' ')]);
+    k += pair[0].length;
+  }
+  if (!pairs) return null;
+  if (rest[k] === '.') k++;
+  return { end: at + k, also: !!head[1], tags };
+}
+
+function replaceTags(norm) {
+  const text = norm.entries.map((e) => e.text).join('\n');
+  if (!/==\s*:[A-Za-z0-9_-]+:\s*==/.test(text)) return;
+  let active = [];
+  let out = '';
+  let from = 0;
+  // nosemgrep: javascript.lang.security.audit.detect-non-literal-regexp.detect-non-literal-regexp -- TAG admits no metacharacter
+  const swap = (chunk) => active.reduce((t, [tag, to]) => t.replace(new RegExp(tag, 'gi'), to), chunk);
+  for (const m of text.matchAll(/\bREPLACE\b/gi)) {
+    if (m.index < from) continue;
+    const st = replaceStatementAt(text, m.index);
+    if (!st) continue;
+    out += swap(text.slice(from, m.index)) + text.slice(m.index, st.end);
+    active = st.off ? [] : st.also ? [...active, ...st.tags] : st.tags;
+    from = st.end;
+  }
+  out += swap(text.slice(from));
+  const lines = out.split('\n');
+  if (lines.length === norm.entries.length) norm.entries.forEach((e, i) => { e.text = lines[i]; });
+}
+
 // A REPLACE governs the source up to the next one, which replaces it unless ALSO; OFF ends it; output is not rescanned.
 function applyReplaceStatements(tokens, ctx) {
   let active = [];
@@ -976,6 +1025,8 @@ function parseDataEntry(toks, section, file) {
       continue;
     }
     if (u === 'PIC' || u === 'PICTURE') { j++; if (toks[j] && toks[j].u === 'IS') j++; if (toks[j]) item.picture = toks[j].v; j++; continue; }
+    // PIC U BYTE-LENGTH n: a UTF-8 item of n bytes holding as many characters as fit.
+    if (u === 'BYTE-LENGTH') { j++; if (toks[j] && toks[j].u === 'IS') j++; if (toks[j] && /^\d+$/.test(toks[j].v)) { item.byteLength = Number(toks[j].v); j++; } continue; }
     if (u === 'USAGE') { j++; if (toks[j] && toks[j].u === 'IS') j++; if (toks[j]) { item.usage = toks[j].u; j++; } continue; }
     if (USAGE_WORDS.has(u)) { item.usage = u; j++; continue; }
     // OCCURS DYNAMIC [CAPACITY IN name] [FROM n] [TO m]: sized at its most, none without TO; the
@@ -1048,6 +1099,9 @@ function parseDataEntry(toks, section, file) {
     if (u === 'CONSTANT') { item.constant = true; j++; if (toks[j] && toks[j].u === 'AS') j++; while (toks[j] && (toks[j].t === 'num' || toks[j].t === 'lit' || (toks[j].t === 'word' && /^\d+$/.test(toks[j].v)))) item.values.push(toks[j++]); continue; }
     if (u === 'VALUE' || u === 'VALUES') {
       j++;
+      // A screen item's VALUE is one literal; what follows it is the item's other clauses, and a later
+      // VALUE (two entries run together by a missing period) replaces it.
+      if (section === 'SCREEN') { if (toks[j] && toks[j].u === 'IS') j++; if (toks[j]) { item.values = [toks[j]]; j++; } continue; }
       while (toks[j] && (toks[j].t !== 'word' || !(DATA_CLAUSE_WORDS.has(toks[j].u) || (report && REPORT_CLAUSE_WORDS.has(toks[j].u))) || toks[j].u === 'IS')) { item.values.push(toks[j]); j++; }
       continue;
     }
@@ -1073,6 +1127,25 @@ function debugItem(at) {
   return [root, ...root.children];
 }
 
+// DEBUG-CONTENTS is X(n), n left to the compiler (IBM, DEBUG-ITEM). GnuCOBOL makes it as long as the
+// longest item USE FOR DEBUGGING names, a CD's records included, and never shorter than 30.
+function sizeDebugContents(prog, tokens, from, to, scheme, constants) {
+  const named = [];
+  for (let i = from; i < to - 3; i++) {
+    if (tokens[i].u !== 'USE' || tokens[i + 1].u !== 'FOR' || tokens[i + 2].u !== 'DEBUGGING') continue;
+    for (let k = i + 3; k < to && tokens[k].t !== 'period'; k++) if (tokens[k].t === 'word') named.push(tokens[k].u);
+  }
+  const sizes = named.flatMap((n) => prog.items.filter((it) => it.name === n && it.level < 50 && !it.implicit).map((it) => it.size || 0)
+    .concat(prog.items.filter((it) => it.cd === n).map((it) => it.size || 0)));
+  const longest = Math.max(30, ...sizes);
+  if (longest === 30) return;
+  const root = prog.items.find((it) => it.implicit && it.name === 'DEBUG-ITEM');
+  const contents = root && root.children.find((c) => c.name === 'DEBUG-CONTENTS');
+  if (!contents) return;
+  contents.picture = `X(${longest})`;
+  computeSizes([root], scheme, constants);
+}
+
 // TYPE gives an item the picture, usage and subordinate items of the TYPEDEF it names. The listing
 // prints the typed item alone; what it holds is implied, reachable by qualification (RE OF WS-Z),
 // so the copies are returned as items marked typeClone.
@@ -1169,6 +1242,7 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
           if (s[m]) f.assign = { t: s[m].t, v: s[m].t === 'word' ? s[m].u : s[m].v };
         }
         for (let m = n + 1; m < s.length; m++) if (s[m].t === 'word') f.envRefs.push(s[m]);
+        f.lineSequential = s.some((t, m) => t.u === 'LINE' && s[m + 1] && s[m + 1].u === 'SEQUENTIAL');
         prog.files.push(f);
       } else {
         for (const t of s) if (t.t === 'word') prog.refs.push({ tok: t, zone: 'env' });
@@ -1182,6 +1256,7 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
     let stack = [];
     let currentFd = null;
     let currentRd = null;
+    let currentCd = null;
     const dataSentences = sentences(tokens, dataStart, dataEnd);
     for (let s of dataSentences) {
       // What an EXEC SQL INCLUDE brings in follows it in the same sentence, up to its own first period.
@@ -1196,6 +1271,7 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
         section = first.u === 'RD' ? 'REPORT' : 'COMMUNICATION';
         currentFd = null;
         currentRd = first.u === 'RD' ? { name: s[1].u, line: first.line, file: first.file, groups: [] } : null;
+        currentCd = first.u === 'CD' ? s[1].u : null;
         if (currentRd) (prog.reports ||= []).push(currentRd);
         else (prog.cds ||= []).push(s[1].u);
         stack = [];
@@ -1204,12 +1280,7 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
       if (first.t === 'word' && (first.u === 'FD' || first.u === 'SD') && s[1]) {
         section = 'FILE';
         currentRd = null;
-        currentFd = { kind: first.u, name: s[1].u, line: first.line, file: first.file, records: [], fdTok: s[1], declaredMax: 0, varying: false };
-        const recAt = s.findIndex(x => x.t === 'word' && x.u === 'RECORD');
-        if (recAt >= 0) for (let m = recAt + 1; m < s.length && !['LABEL', 'BLOCK', 'DATA', 'VALUE', 'RECORDING', 'CODE-SET', 'LINAGE', 'REPORT', 'REPORTS', 'DEPENDING'].includes(s[m].u); m++) {
-          if (s[m].u === 'VARYING') currentFd.varying = true;
-          if (/^\d+$/.test(s[m].v)) currentFd.declaredMax = Math.max(currentFd.declaredMax, Number(s[m].v));
-        }
+        currentFd = { kind: first.u, name: s[1].u, line: first.line, file: first.file, records: [], fdTok: s[1], ...recordClause(s) };
         const repAt = s.findIndex(x => x.t === 'word' && (x.u === 'REPORT' || x.u === 'REPORTS'));
         if (repAt >= 0) {
           currentFd.reports = [];
@@ -1225,6 +1296,7 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
       }
       if ((first.t === 'word' || first.t === 'num') && /^\d{1,2}$/.test(first.v)) {
         const item = parseDataEntry(s, section, first.file);
+        if (section === 'COMMUNICATION' && item.level === 1) item.cd = currentCd;
         if (item.level === 88 || item.level === 66) {
           const parent = item.level === 88 ? stack[stack.length - 1] : null;
           if (parent) { parent.children.push(item); item.parent = parent; }
@@ -1283,6 +1355,7 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
         || siblings.find(x => x !== it && x.name === it.redefines) || null;
     }
     computeSizes(roots, scheme, constants);
+    if (prog.debuggingMode && procAt >= 0) sizeDebugContents(prog, tokens, procAt, procEnd, scheme, constants);
     const subtreeCache = new Map();
     const subtree = (record) => {
       if (subtreeCache.has(record)) return subtreeCache.get(record);
@@ -1319,12 +1392,18 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
       it.renamesSpan = all.filter(x => x.offset >= from && x.offset + x.contributes <= to && x !== it);
     }
     for (const rd of prog.reports || []) layoutReport(rd);
-    // A file is as large as the longer of its declared length and its record description; the
-    // compiler warns when a record exceeds the declared maximum but still uses the record. A report
-    // file's record is as wide as the widest line of the reports written to it.
+    // A file's record is as long as its RECORD clause allows, or its longest record description
+    // where that is longer or there is no clause; a record longer than the clause still wins, as the
+    // compiler warns and uses it. The FD of a line-sequential file, which GnuCOBOL has and IBM does
+    // not, is sized by its records whatever RECORD CONTAINS says, though RECORD IS VARYING still sets
+    // its maximum. A report file's record is as wide as the widest line of the reports written to it.
     for (const fd of prog.fds || []) {
       const reports = (fd.reports || []).map(n => (prog.reports || []).find(r => r.name === n)).filter(Boolean);
-      fd.size = Math.max(fd.declaredMax || 0, 0, ...fd.records.map(r => r.size || 0), ...reports.map(r => r.width || 0));
+      const described = Math.max(0, ...fd.records.map(recordLength), ...reports.map(r => r.width || 0));
+      const select = prog.files.find(f => f.name === fd.name);
+      const ignored = select && select.lineSequential && fd.kind === 'FD' && fd.recordFormat !== 3;
+      const clause = ignored ? 0 : fd.fixedLength || fd.varyingMax || fd.rangeMax || 0;
+      fd.size = Math.max(clause, described);
     }
   }
 
@@ -1334,6 +1413,34 @@ function parseProgram(tokens, from, to, scheme, defines, hostVariables = true) {
   return prog;
 }
 
+const FD_CLAUSES = new Set(['LABEL', 'BLOCK', 'DATA', 'VALUE', 'RECORDING', 'CODE-SET', 'LINAGE', 'REPORT', 'REPORTS', 'DEPENDING', 'RECORD']);
+
+// The RECORD clause of an FD in IBM's three formats, CONTAINS n, CONTAINS n TO m and IS VARYING ...
+// TO m, each giving the longest record the file holds. LABEL RECORD and DATA RECORD are other clauses
+// that share the word.
+function recordClause(s) {
+  for (let at = 1; at < s.length; at++) {
+    if (s[at].u !== 'RECORD' || ['LABEL', 'DATA'].includes(s[at - 1].u)) continue;
+    let varying = false;
+    let to = false;
+    const nums = [];
+    let toNum = null;
+    for (let m = at + 1; m < s.length && !FD_CLAUSES.has(s[m].u); m++) {
+      if (s[m].u === 'VARYING') varying = true;
+      else if (s[m].u === 'TO') to = true;
+      else if (/^\d+$/.test(s[m].v)) { nums.push(Number(s[m].v)); if (to && toNum == null) toNum = Number(s[m].v); }
+    }
+    if (varying) return { recordFormat: 3, varyingMax: toNum };
+    if (nums.length === 1 && !to) return { recordFormat: 1, fixedLength: nums[0] };
+    if (nums.length) return { recordFormat: 2, rangeMax: nums.at(-1) };
+  }
+  return { recordFormat: null };
+}
+
+// A record's length in its file. OCCURS on a level-01 record is a GnuCOBOL extension IBM refuses;
+// GnuCOBOL sizes the file by one occurrence.
+const recordLength = (r) => (r.occurs > 1 ? r.size / r.occurs : r.size || 0);
+
 export function segmentEnd(tokens, i, to) {
   let depth = 0;
   for (let k = i + 1; k < to; k++) {
@@ -1485,10 +1592,15 @@ function parseProcedure(tokens, procAt, to, prog, hostVariables) {
         const constText = target.t === 'word' && prog.constants && prog.constants.text
           ? prog.constants.text.get(target.u) : undefined;
         const resolved = target.t === 'lit' ? target.v : constText;
+        // A literal program name is padded to its field; the compiler drops the trailing spaces. A
+        // literal holding a path calls the program its last component names, as the runtime loads it.
+        const literal = resolved === undefined ? null : String(resolved).trimEnd();
+        const name = literal === null ? target.u : literal.split(/[\\/]/).pop();
         prog.calls.push({
-          // A literal program name is padded to its field; the compiler drops the trailing spaces.
           kind: resolved === undefined ? 'I' : 'L',
-          name: resolved === undefined ? target.u : String(resolved).trimEnd(),
+          name,
+          ...(literal !== null && name !== literal ? { calledAs: literal } : {}),
+          ...(target.t === 'lit' && target.prefix === 'X' ? { hex: true } : {}),
           ...(constText !== undefined ? { viaConstant: target.u } : {}),
           line: t.line, file: t.file, targetTok: target, using: args, stmtIndex: prog.statements.length - 1,
         });
@@ -1953,8 +2065,15 @@ export function parseSource(src, file, opts = {}) {
     // Where a copybook's text comes from: the disk, or the source tree the program was read from.
     readText: opts.readText || ((p) => readSource(p).text) };
   collectCopybookDefines(src, format, ctx, 0);
-  const norm = normalize(src, format, ctx.defines, opts.std);
+  let norm = normalize(src, format, ctx.defines, opts.std);
+  // WITH DEBUGGING MODE in SOURCE-COMPUTER makes the debugging lines of the program and its
+  // copybooks source; it is read from the text with those lines still comments.
+  if (/\bSOURCE-COMPUTER\s*\.[^.]*\bDEBUGGING\s+MODE\b/i.test(norm.entries.map(e => e.text).join(' '))) {
+    ctx.debugging = true;
+    norm = normalize(src, format, ctx.defines, opts.std, true);
+  }
   ctx.mainFormat = norm.finalFormat;
+  replaceTags(norm);
   const { tokens: raw, diags } = tokenize(norm, file);
   ctx.diags.push(...norm.diags.map(d => ({ ...d, file })), ...diags);
   const expanded = stripDirecting(applyReplaceStatements(expand(raw, ctx, [resolve(file)], norm.finalFormat), ctx));

package/lib/pli/cursor.mjs

@@ -0,0 +1,15 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// The PL/I statement cursor: the shared token cursor, failing with PliSyntax.
+import { tokenCursor, split } from '../statement-cursor.mjs';
+
+export class PliSyntax extends Error {
+  constructor(message, tok) {
+    super(message);
+    this.name = 'PliSyntax';
+    this.line = tok ? tok.line : null;
+    this.col = tok ? tok.col : null;
+  }
+}
+
+export const cursor = (toks) => tokenCursor(toks, PliSyntax);
+export { split };

package/lib/pli/expr.mjs

@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// PL/I expressions and references, as the statement parsers consume them: an operator tree by
+// PL/I precedence, every token consumed, and the references the expression reads.
+import { cursor } from './cursor.mjs';
+
+// Higher binds tighter. Infix ¬ is exclusive-or, at the level of |.
+const PREC = {
+  '**': 7, '*': 6, '/': 6, '+': 5, '-': 5, '||': 4,
+  '=': 3, '¬=': 3, '<': 3, '>': 3, '<=': 3, '>=': 3, '¬<': 3, '¬>': 3, '<>': 3,
+  '&': 2, '|': 1, '¬': 1,
+};
+const PREFIX = new Set(['+', '-', '¬']);
+
+// A reference: a name, then any mix of (arguments), '.' name, and '->' or '=>' locator
+// qualification. args holds one list per parenthesised group, each argument an expression or
+// { t: 'star' } for '*'.
+export function parseReference(c) {
+  const start = c.pos;
+  const first = c.word();
+  if (!first) c.fail('a reference');
+  const path = [first.u];
+  const locators = [];
+  const args = [];
+  for (;;) {
+    if (c.isOp('(')) { args.push(c.items().map(argument)); continue; }
+    if (c.isOp('.') && c.isWord(undefined, 1)) { c.next(); path.push(c.next().u); continue; }
+    if (c.isOp(['->', '=>']) && c.isWord(undefined, 1)) { c.next(); locators.push(path.length); path.push(c.next().u); continue; }
+    break;
+  }
+  return { t: 'ref', name: path[path.length - 1], path, locators, args, toks: c.slice(start) };
+}
+
+function argument(toks) {
+  if (toks.length === 1 && toks[0].t === 'op' && toks[0].v === '*') return { t: 'star', toks };
+  const s = cursor(toks);
+  const e = parseExpression(s);
+  if (!s.done()) s.fail('the end of the argument');
+  return e;
+}
+
+function primary(c) {
+  const t = c.peek();
+  if (!t) c.fail('an operand');
+  if (t.t === 'num') return { t: 'num', tok: c.next() };
+  if (t.t === 'lit') return { t: 'lit', tok: c.next() };
+  if (t.t === 'word') return parseReference(c);
+  if (t.t === 'op' && t.v === '(') {
+    c.next();
+    const inner = parseExpression(c);
+    c.expectOp(')');
+    // (3)'AB' and (N)'0'B repeat the literal: a parenthesis is never otherwise followed by one.
+    if (c.peek() && c.peek().t === 'lit') return { t: 'lit', tok: c.next(), factor: inner };
+    return { t: 'paren', expr: inner };
+  }
+  return c.fail('an operand');
+}
+
+// A prefix operator applies to its operand's ** chain: -A**2 is -(A**2).
+function unary(c) {
+  const t = c.peek();
+  if (t && t.t === 'op' && PREFIX.has(t.v)) {
+    c.next();
+    return { op: `prefix${t.v}`, args: [binary(c, 7)] };
+  }
+  return primary(c);
+}
+
+function binary(c, min, stopOps = []) {
+  let left = unary(c);
+  for (;;) {
+    const t = c.peek();
+    if (!t || t.t !== 'op' || !(t.v in PREC) || stopOps.includes(t.v)) return left;
+    const p = PREC[t.v];
+    if (p < min) return left;
+    c.next();
+    const right = binary(c, t.v === '**' ? p : p + 1, stopOps);
+    left = { op: t.v, args: [left, right] };
+  }
+}
+
+function collect(node, out) {
+  if (!node) return;
+  if (node.t === 'ref') {
+    for (const at of node.locators) out.push(node.path.slice(0, at).join('.'));
+    out.push(node.path.join('.'));
+    for (const list of node.args) for (const a of list) collect(a.tree ?? a, out);
+  } else if (node.t === 'paren') collect(node.expr.tree, out);
+  else if (node.t === 'lit' && node.factor) collect(node.factor.tree, out);
+  else if (node.args) for (const a of node.args) collect(a, out);
+}
+
+// An expression, stopping before the first token that cannot continue it: ',' or ')', a word after a
+// complete operand (so a caller's stopWords need no test), an assignment operator, or an op in
+// stopOps. { t: 'expr', tree, toks, refs }.
+export function parseExpression(c, { stopOps = [] } = {}) {
+  const start = c.pos;
+  const tree = binary(c, 1, stopOps);
+  const refs = [];
+  collect(tree, refs);
+  return { t: 'expr', tree, toks: c.slice(start), refs: [...new Set(refs)] };
+}

package/lib/pli/include.mjs

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// %INCLUDE expansion at the token level, as the preprocessor does it before statements exist: a
+// member can hold the rest of a DECLARE (`DCL 1 REC, %INCLUDE RECFLDS;`), so a member's tokens
+// replace the directive in the token stream and statements are split afterwards. Every token keeps
+// the file and line it was read from.
+import { basename, dirname, extname } from 'node:path';
+import { tokenize, statements } from './lex.mjs';
+
+// Upper-case member name -> paths of the files that could hold it, in path order.
+export function membersOf(paths) {
+  const out = new Map();
+  for (const p of [...paths].sort()) {
+    const name = basename(p, extname(p)).toUpperCase();
+    if (!out.has(name)) out.set(name, []);
+    out.get(name).push(p);
+  }
+  return out;
+}
+
+// The member a directive in `from` means: one in the same directory if there is one, else the first.
+export function chooseMember(candidates, from) {
+  if (!candidates || !candidates.length) return null;
+  return candidates.find((p) => dirname(p) === dirname(from)) || candidates[0];
+}
+
+// The members a %INCLUDE or %XINCLUDE directive names: name, 'name' or ddname(name), comma-separated.
+function directiveMembers(toks) {
+  const names = [];
+  for (let k = 2; k < toks.length; k++) {
+    const t = toks[k];
+    if (t.t === 'op' && t.v === ',') continue;
+    if (t.t === 'word' && toks[k + 1]?.t === 'op' && toks[k + 1].v === '(' && toks[k + 3]?.t === 'op' && toks[k + 3].v === ')') {
+      const m = toks[k + 2];
+      names.push(String(m.u ?? m.v).toUpperCase());
+      k += 3;
+    } else if (t.t === 'word') names.push(t.u);
+    else if (t.t === 'lit') names.push(t.v.toUpperCase());
+  }
+  return names;
+}
+
+// readMember(name, fromFile) -> { path, text } or null.
+export function expandIncludes(tokens, { file, readMember, maxDepth = 16 }) {
+  const included = [];
+  const unresolved = [];
+  const cycles = [];
+  const seen = new Set();
+  const walk = (toks, from, chain) => {
+    const out = [];
+    for (let i = 0; i < toks.length; i++) {
+      const t = toks[i];
+      const d = toks[i + 1];
+      if (!(t.t === 'op' && t.v === '%' && d && d.t === 'word' && (d.u === 'INCLUDE' || d.u === 'XINCLUDE'))) { out.push(t); continue; }
+      let end = i;
+      while (end < toks.length && toks[end].t !== 'semi') end++;
+      const directive = toks.slice(i, end + 1);
+      const keep = [];
+      for (const name of directiveMembers(directive)) {
+        if (d.u === 'XINCLUDE' && seen.has(name)) continue;
+        if (chain.includes(name)) { cycles.push({ chain: [...chain, name] }); continue; }
+        if (chain.length >= maxDepth) { unresolved.push({ name, file: t.file ?? from, line: t.line, why: 'too deep' }); continue; }
+        const m = readMember(name, from);
+        if (!m) { unresolved.push({ name, file: t.file ?? from, line: t.line }); keep.push(name); continue; }
+        seen.add(name);
+        included.push({ name, path: m.path, from: { file: t.file ?? from, line: t.line } });
+        out.push(...walk(tokenize(m.text, { file: m.path }).tokens, m.path, [...chain, name]));
+      }
+      // An unresolved member keeps its directive, so the statement is still there to be counted.
+      if (keep.length) out.push(...directive);
+      i = end;
+    }
+    return out;
+  };
+  return { tokens: walk(tokens, file, []), included, unresolved, cycles };
+}
+
+// readPli with %INCLUDE members spliced in.
+export function readPliExpanded(text, { file, readMember }) {
+  const { tokens, diags, process, margins } = tokenize(text, { file });
+  const ex = expandIncludes(tokens, { file, readMember });
+  return { statements: statements(ex.tokens), diags, process, margins, included: ex.included, unresolved: ex.unresolved, cycles: ex.cycles };
+}

package/lib/pli/layout.mjs

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// PL/I structure mapping: DECLARE items into structure trees, and each member's offset and size by the
+// Enterprise PL/I Language Reference's rules, innermost minor structures first, every pair of units
+// placed with the first shifted toward the second as far as its alignment allows.
+import { storageOf } from './storage.mjs';
+
+const DW = 64;
+const mod = (a, m) => ((a % m) + m) % m;
+
+// The trees a DECLARE's items describe. A member's logical level is one deeper than the structure
+// that holds it, whatever level numbers the source wrote.
+export function structuresOf(items) {
+  const roots = [];
+  const stack = [];
+  for (const it of items) {
+    const node = { name: it.name, level: it.level, dims: it.dims || [], attributes: it.attributes || [], line: it.line, ...(it.file ? { file: it.file } : {}), children: [] };
+    if (it.level == null || it.level <= 1) { roots.push(node); stack.length = 0; if (it.level != null) stack.push(node); node.logical = 1; continue; }
+    while (stack.length && stack[stack.length - 1].level >= it.level) stack.pop();
+    const parent = stack[stack.length - 1];
+    if (!parent) { roots.push(node); node.logical = 1; node.orphan = true; stack.push(node); continue; }
+    node.logical = parent.logical + 1;
+    parent.children.push(node);
+    stack.push(node);
+  }
+  return roots;
+}
+
+// The number of elements a dimension list gives, or null when a bound is not a constant.
+function extent(dims) {
+  let n = 1;
+  for (const d of dims) {
+    const parts = [[]];
+    for (const t of d) { if (t.t === 'op' && t.v === ':') parts.push([]); else parts[parts.length - 1].push(t); }
+    const num = (p) => {
+      if (p.length === 1 && p[0].t === 'num' && /^\d+$/.test(p[0].v)) return Number(p[0].v);
+      if (p.length === 2 && p[0].t === 'op' && (p[0].v === '-' || p[0].v === '+') && p[1].t === 'num' && /^\d+$/.test(p[1].v)) return (p[0].v === '-' ? -1 : 1) * Number(p[1].v);
+      return null;
+    };
+    const [lo, hi] = parts.length === 2 ? [num(parts[0]), num(parts[1])] : [1, num(parts[0])];
+    if (lo == null || hi == null) return null;
+    n *= Math.max(0, hi - lo + 1);
+  }
+  return n;
+}
+
+const explicitAlignment = (node) => (node.attributes.some((a) => a.name === 'ALIGNED') ? true : node.attributes.some((a) => a.name === 'UNALIGNED') ? false : null);
+
+// Pairs two units: the first begins at its offset from a doubleword boundary, the second at the
+// first position after it that its alignment allows, then the first moves toward the second by
+// whole multiples of its own alignment. Positions are in bits.
+function pair(a, b) {
+  const end = a.offset + a.bits;
+  const start = b.structure ? end + mod(b.offset - end, b.align) : Math.ceil(end / b.align) * b.align;
+  const shift = Math.floor((start - end) / a.align) * a.align;
+  const first = a.offset + shift;
+  return { first, second: start, unit: { offset: mod(first, DW), bits: start + b.bits - first, align: Math.max(a.align, b.align), structure: true } };
+}
+
+// Maps one node: an element takes its storage; a structure maps its members into one unit, a union
+// overlays them. Every node gets `at`, its start in bits within the unit of its parent.
+function mapNode(node, inherited, problems) {
+  const own = explicitAlignment(node);
+  const inherit = own ?? inherited;
+  const count = node.dims.length ? extent(node.dims) : 1;
+  let unit;
+  if (!node.children.length) {
+    const s = storageOf(node.attributes, { inherited, name: node.name });
+    node.storage = s;
+    if (!s.known) problems.push({ name: node.name, line: node.line, why: s.type === 'TYPE' ? `type ${s.typeName} is defined by a DEFINE this reading does not resolve` : s.type ? `the extent of ${s.type} is not a constant` : 'no data attributes' });
+    unit = { offset: 0, bits: s.bits ?? 0, align: s.align, structure: false };
+  } else if (node.attributes.some((a) => a.name === 'UNION')) {
+    const members = node.children.map((ch) => mapNode(ch, inherit, problems));
+    let len = 0;
+    node.children.forEach((ch, k) => { ch.at = mod(members[k].offset, members[k].align); len = Math.max(len, ch.at + members[k].bits); });
+    unit = { offset: 0, bits: len, align: Math.max(...members.map((m) => m.align)), structure: true };
+  } else {
+    const members = node.children.map((ch) => mapNode(ch, inherit, problems));
+    let acc = { ...members[0] };
+    const starts = [members[0].offset];
+    for (let k = 1; k < members.length; k++) {
+      const { first, second, unit: u } = pair(acc, members[k]);
+      const moved = first - acc.offset;
+      for (let j = 0; j < k; j++) starts[j] += moved;
+      starts.push(starts[0] + (second - first));
+      acc = u;
+    }
+    const base = starts[0];
+    node.children.forEach((ch, k) => { ch.at = starts[k] - base; });
+    unit = { offset: acc.offset, bits: acc.bits, align: acc.align, structure: true };
+  }
+  if (count == null) problems.push({ name: node.name, line: node.line, why: 'a dimension bound is not a constant' });
+  node.count = count;
+  // Each element of an array starts on the same boundary, so an element is padded to its alignment.
+  const stride = count > 1 ? Math.ceil(unit.bits / unit.align) * unit.align : unit.bits;
+  node.strideBits = stride;
+  return count > 1 ? { ...unit, bits: stride * count } : unit;
+}
+
+// Byte and bit offsets from the start of the major structure, and sizes, set on every node.
+function place(node, atBits) {
+  node.offsetBits = atBits;
+  node.offset = Math.floor(atBits / 8);
+  if (atBits % 8) node.bitOffset = atBits % 8;
+  node.size = Math.ceil(node.strideBits / 8);
+  node.occurs = node.count ?? null;
+  for (const ch of node.children) place(ch, atBits + ch.at);
+}
+
+// Lays out every structure a DECLARE describes. Returns the trees, each node carrying offset (bytes
+// from its major structure), bitOffset when not on a byte, size (bytes of one element), occurs, and
+// storage for an element; problems lists what could not be sized.
+export function layout(items) {
+  const roots = structuresOf(items);
+  const problems = [];
+  for (const r of roots) {
+    const u = mapNode(r, null, problems);
+    // Storage begins at the byte holding the first bit; unaligned bits shifted toward their
+    // successor leave their padding there.
+    const lead = u.offset % 8;
+    r.totalBits = u.bits;
+    place(r, lead);
+    r.total = Math.ceil((lead + u.bits) / 8);
+  }
+  return { roots, problems };
+}