@portll/cobolwork 0.6.0 → 0.7.0

package/lib/pli/expr.mjs

@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// PL/I expressions and references, as the statement parsers consume them: an operator tree by
+// PL/I precedence, every token consumed, and the references the expression reads.
+import { cursor } from './cursor.mjs';
+
+// Higher binds tighter. Infix ¬ is exclusive-or, at the level of |.
+const PREC = {
+  '**': 7, '*': 6, '/': 6, '+': 5, '-': 5, '||': 4,
+  '=': 3, '¬=': 3, '<': 3, '>': 3, '<=': 3, '>=': 3, '¬<': 3, '¬>': 3, '<>': 3,
+  '&': 2, '|': 1, '¬': 1,
+};
+const PREFIX = new Set(['+', '-', '¬']);
+
+// A reference: a name, then any mix of (arguments), '.' name, and '->' or '=>' locator
+// qualification. args holds one list per parenthesised group, each argument an expression or
+// { t: 'star' } for '*'.
+export function parseReference(c) {
+  const start = c.pos;
+  const first = c.word();
+  if (!first) c.fail('a reference');
+  const path = [first.u];
+  const locators = [];
+  const args = [];
+  for (;;) {
+    if (c.isOp('(')) { args.push(c.items().map(argument)); continue; }
+    if (c.isOp('.') && c.isWord(undefined, 1)) { c.next(); path.push(c.next().u); continue; }
+    if (c.isOp(['->', '=>']) && c.isWord(undefined, 1)) { c.next(); locators.push(path.length); path.push(c.next().u); continue; }
+    break;
+  }
+  return { t: 'ref', name: path[path.length - 1], path, locators, args, toks: c.slice(start) };
+}
+
+function argument(toks) {
+  if (toks.length === 1 && toks[0].t === 'op' && toks[0].v === '*') return { t: 'star', toks };
+  const s = cursor(toks);
+  const e = parseExpression(s);
+  if (!s.done()) s.fail('the end of the argument');
+  return e;
+}
+
+function primary(c) {
+  const t = c.peek();
+  if (!t) c.fail('an operand');
+  if (t.t === 'num') return { t: 'num', tok: c.next() };
+  if (t.t === 'lit') return { t: 'lit', tok: c.next() };
+  if (t.t === 'word') return parseReference(c);
+  if (t.t === 'op' && t.v === '(') {
+    c.next();
+    const inner = parseExpression(c);
+    c.expectOp(')');
+    // (3)'AB' and (N)'0'B repeat the literal: a parenthesis is never otherwise followed by one.
+    if (c.peek() && c.peek().t === 'lit') return { t: 'lit', tok: c.next(), factor: inner };
+    return { t: 'paren', expr: inner };
+  }
+  return c.fail('an operand');
+}
+
+// A prefix operator applies to its operand's ** chain: -A**2 is -(A**2).
+function unary(c) {
+  const t = c.peek();
+  if (t && t.t === 'op' && PREFIX.has(t.v)) {
+    c.next();
+    return { op: `prefix${t.v}`, args: [binary(c, 7)] };
+  }
+  return primary(c);
+}
+
+function binary(c, min, stopOps = []) {
+  let left = unary(c);
+  for (;;) {
+    const t = c.peek();
+    if (!t || t.t !== 'op' || !(t.v in PREC) || stopOps.includes(t.v)) return left;
+    const p = PREC[t.v];
+    if (p < min) return left;
+    c.next();
+    const right = binary(c, t.v === '**' ? p : p + 1, stopOps);
+    left = { op: t.v, args: [left, right] };
+  }
+}
+
+function collect(node, out) {
+  if (!node) return;
+  if (node.t === 'ref') {
+    for (const at of node.locators) out.push(node.path.slice(0, at).join('.'));
+    out.push(node.path.join('.'));
+    for (const list of node.args) for (const a of list) collect(a.tree ?? a, out);
+  } else if (node.t === 'paren') collect(node.expr.tree, out);
+  else if (node.t === 'lit' && node.factor) collect(node.factor.tree, out);
+  else if (node.args) for (const a of node.args) collect(a, out);
+}
+
+// An expression, stopping before the first token that cannot continue it: ',' or ')', a word after a
+// complete operand (so a caller's stopWords need no test), an assignment operator, or an op in
+// stopOps. { t: 'expr', tree, toks, refs }.
+export function parseExpression(c, { stopOps = [] } = {}) {
+  const start = c.pos;
+  const tree = binary(c, 1, stopOps);
+  const refs = [];
+  collect(tree, refs);
+  return { t: 'expr', tree, toks: c.slice(start), refs: [...new Set(refs)] };
+}

package/lib/pli/include.mjs

@@ -0,0 +1,82 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// %INCLUDE expansion at the token level, as the preprocessor does it before statements exist: a
+// member can hold the rest of a DECLARE (`DCL 1 REC, %INCLUDE RECFLDS;`), so a member's tokens
+// replace the directive in the token stream and statements are split afterwards. Every token keeps
+// the file and line it was read from.
+import { basename, dirname, extname } from 'node:path';
+import { tokenize, statements } from './lex.mjs';
+
+// Upper-case member name -> paths of the files that could hold it, in path order.
+export function membersOf(paths) {
+  const out = new Map();
+  for (const p of [...paths].sort()) {
+    const name = basename(p, extname(p)).toUpperCase();
+    if (!out.has(name)) out.set(name, []);
+    out.get(name).push(p);
+  }
+  return out;
+}
+
+// The member a directive in `from` means: one in the same directory if there is one, else the first.
+export function chooseMember(candidates, from) {
+  if (!candidates || !candidates.length) return null;
+  return candidates.find((p) => dirname(p) === dirname(from)) || candidates[0];
+}
+
+// The members a %INCLUDE or %XINCLUDE directive names: name, 'name' or ddname(name), comma-separated.
+function directiveMembers(toks) {
+  const names = [];
+  for (let k = 2; k < toks.length; k++) {
+    const t = toks[k];
+    if (t.t === 'op' && t.v === ',') continue;
+    if (t.t === 'word' && toks[k + 1]?.t === 'op' && toks[k + 1].v === '(' && toks[k + 3]?.t === 'op' && toks[k + 3].v === ')') {
+      const m = toks[k + 2];
+      names.push(String(m.u ?? m.v).toUpperCase());
+      k += 3;
+    } else if (t.t === 'word') names.push(t.u);
+    else if (t.t === 'lit') names.push(t.v.toUpperCase());
+  }
+  return names;
+}
+
+// readMember(name, fromFile) -> { path, text } or null.
+export function expandIncludes(tokens, { file, readMember, maxDepth = 16 }) {
+  const included = [];
+  const unresolved = [];
+  const cycles = [];
+  const seen = new Set();
+  const walk = (toks, from, chain) => {
+    const out = [];
+    for (let i = 0; i < toks.length; i++) {
+      const t = toks[i];
+      const d = toks[i + 1];
+      if (!(t.t === 'op' && t.v === '%' && d && d.t === 'word' && (d.u === 'INCLUDE' || d.u === 'XINCLUDE'))) { out.push(t); continue; }
+      let end = i;
+      while (end < toks.length && toks[end].t !== 'semi') end++;
+      const directive = toks.slice(i, end + 1);
+      const keep = [];
+      for (const name of directiveMembers(directive)) {
+        if (d.u === 'XINCLUDE' && seen.has(name)) continue;
+        if (chain.includes(name)) { cycles.push({ chain: [...chain, name] }); continue; }
+        if (chain.length >= maxDepth) { unresolved.push({ name, file: t.file ?? from, line: t.line, why: 'too deep' }); continue; }
+        const m = readMember(name, from);
+        if (!m) { unresolved.push({ name, file: t.file ?? from, line: t.line }); keep.push(name); continue; }
+        seen.add(name);
+        included.push({ name, path: m.path, from: { file: t.file ?? from, line: t.line } });
+        out.push(...walk(tokenize(m.text, { file: m.path }).tokens, m.path, [...chain, name]));
+      }
+      // An unresolved member keeps its directive, so the statement is still there to be counted.
+      if (keep.length) out.push(...directive);
+      i = end;
+    }
+    return out;
+  };
+  return { tokens: walk(tokens, file, []), included, unresolved, cycles };
+}
+
+// readPli with %INCLUDE members spliced in.
+export function readPliExpanded(text, { file, readMember }) {
+  const { tokens, diags, process, margins } = tokenize(text, { file });
+  const ex = expandIncludes(tokens, { file, readMember });
+  return { statements: statements(ex.tokens), diags, process, margins, included: ex.included, unresolved: ex.unresolved, cycles: ex.cycles };
+}

package/lib/pli/layout.mjs

@@ -0,0 +1,125 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// PL/I structure mapping: DECLARE items into structure trees, and each member's offset and size by the
+// Enterprise PL/I Language Reference's rules, innermost minor structures first, every pair of units
+// placed with the first shifted toward the second as far as its alignment allows.
+import { storageOf } from './storage.mjs';
+
+const DW = 64;
+const mod = (a, m) => ((a % m) + m) % m;
+
+// The trees a DECLARE's items describe. A member's logical level is one deeper than the structure
+// that holds it, whatever level numbers the source wrote.
+export function structuresOf(items) {
+  const roots = [];
+  const stack = [];
+  for (const it of items) {
+    const node = { name: it.name, level: it.level, dims: it.dims || [], attributes: it.attributes || [], line: it.line, ...(it.file ? { file: it.file } : {}), children: [] };
+    if (it.level == null || it.level <= 1) { roots.push(node); stack.length = 0; if (it.level != null) stack.push(node); node.logical = 1; continue; }
+    while (stack.length && stack[stack.length - 1].level >= it.level) stack.pop();
+    const parent = stack[stack.length - 1];
+    if (!parent) { roots.push(node); node.logical = 1; node.orphan = true; stack.push(node); continue; }
+    node.logical = parent.logical + 1;
+    parent.children.push(node);
+    stack.push(node);
+  }
+  return roots;
+}
+
+// The number of elements a dimension list gives, or null when a bound is not a constant.
+function extent(dims) {
+  let n = 1;
+  for (const d of dims) {
+    const parts = [[]];
+    for (const t of d) { if (t.t === 'op' && t.v === ':') parts.push([]); else parts[parts.length - 1].push(t); }
+    const num = (p) => {
+      if (p.length === 1 && p[0].t === 'num' && /^\d+$/.test(p[0].v)) return Number(p[0].v);
+      if (p.length === 2 && p[0].t === 'op' && (p[0].v === '-' || p[0].v === '+') && p[1].t === 'num' && /^\d+$/.test(p[1].v)) return (p[0].v === '-' ? -1 : 1) * Number(p[1].v);
+      return null;
+    };
+    const [lo, hi] = parts.length === 2 ? [num(parts[0]), num(parts[1])] : [1, num(parts[0])];
+    if (lo == null || hi == null) return null;
+    n *= Math.max(0, hi - lo + 1);
+  }
+  return n;
+}
+
+const explicitAlignment = (node) => (node.attributes.some((a) => a.name === 'ALIGNED') ? true : node.attributes.some((a) => a.name === 'UNALIGNED') ? false : null);
+
+// Pairs two units: the first begins at its offset from a doubleword boundary, the second at the
+// first position after it that its alignment allows, then the first moves toward the second by
+// whole multiples of its own alignment. Positions are in bits.
+function pair(a, b) {
+  const end = a.offset + a.bits;
+  const start = b.structure ? end + mod(b.offset - end, b.align) : Math.ceil(end / b.align) * b.align;
+  const shift = Math.floor((start - end) / a.align) * a.align;
+  const first = a.offset + shift;
+  return { first, second: start, unit: { offset: mod(first, DW), bits: start + b.bits - first, align: Math.max(a.align, b.align), structure: true } };
+}
+
+// Maps one node: an element takes its storage; a structure maps its members into one unit, a union
+// overlays them. Every node gets `at`, its start in bits within the unit of its parent.
+function mapNode(node, inherited, problems) {
+  const own = explicitAlignment(node);
+  const inherit = own ?? inherited;
+  const count = node.dims.length ? extent(node.dims) : 1;
+  let unit;
+  if (!node.children.length) {
+    const s = storageOf(node.attributes, { inherited, name: node.name });
+    node.storage = s;
+    if (!s.known) problems.push({ name: node.name, line: node.line, why: s.type === 'TYPE' ? `type ${s.typeName} is defined by a DEFINE this reading does not resolve` : s.type ? `the extent of ${s.type} is not a constant` : 'no data attributes' });
+    unit = { offset: 0, bits: s.bits ?? 0, align: s.align, structure: false };
+  } else if (node.attributes.some((a) => a.name === 'UNION')) {
+    const members = node.children.map((ch) => mapNode(ch, inherit, problems));
+    let len = 0;
+    node.children.forEach((ch, k) => { ch.at = mod(members[k].offset, members[k].align); len = Math.max(len, ch.at + members[k].bits); });
+    unit = { offset: 0, bits: len, align: Math.max(...members.map((m) => m.align)), structure: true };
+  } else {
+    const members = node.children.map((ch) => mapNode(ch, inherit, problems));
+    let acc = { ...members[0] };
+    const starts = [members[0].offset];
+    for (let k = 1; k < members.length; k++) {
+      const { first, second, unit: u } = pair(acc, members[k]);
+      const moved = first - acc.offset;
+      for (let j = 0; j < k; j++) starts[j] += moved;
+      starts.push(starts[0] + (second - first));
+      acc = u;
+    }
+    const base = starts[0];
+    node.children.forEach((ch, k) => { ch.at = starts[k] - base; });
+    unit = { offset: acc.offset, bits: acc.bits, align: acc.align, structure: true };
+  }
+  if (count == null) problems.push({ name: node.name, line: node.line, why: 'a dimension bound is not a constant' });
+  node.count = count;
+  // Each element of an array starts on the same boundary, so an element is padded to its alignment.
+  const stride = count > 1 ? Math.ceil(unit.bits / unit.align) * unit.align : unit.bits;
+  node.strideBits = stride;
+  return count > 1 ? { ...unit, bits: stride * count } : unit;
+}
+
+// Byte and bit offsets from the start of the major structure, and sizes, set on every node.
+function place(node, atBits) {
+  node.offsetBits = atBits;
+  node.offset = Math.floor(atBits / 8);
+  if (atBits % 8) node.bitOffset = atBits % 8;
+  node.size = Math.ceil(node.strideBits / 8);
+  node.occurs = node.count ?? null;
+  for (const ch of node.children) place(ch, atBits + ch.at);
+}
+
+// Lays out every structure a DECLARE describes. Returns the trees, each node carrying offset (bytes
+// from its major structure), bitOffset when not on a byte, size (bytes of one element), occurs, and
+// storage for an element; problems lists what could not be sized.
+export function layout(items) {
+  const roots = structuresOf(items);
+  const problems = [];
+  for (const r of roots) {
+    const u = mapNode(r, null, problems);
+    // Storage begins at the byte holding the first bit; unaligned bits shifted toward their
+    // successor leave their padding there.
+    const lead = u.offset % 8;
+    r.totalBits = u.bits;
+    place(r, lead);
+    r.total = Math.ceil((lead + u.bits) / 8);
+  }
+  return { roots, problems };
+}

package/lib/pli/lex.mjs

@@ -0,0 +1,198 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// PL/I source as tokens and statements: the margins a file is read with, comments and literals
+// removed or kept as tokens, and the semicolon-terminated statements with their label and condition
+// prefixes. Classification by statement kind is in lib/pli/statements.mjs.
+
+const PROCESS_LINE = /^[*%]PROCESS\b(.*)$/i;
+const CARRIAGE = new Set([' ', '0', '1', '-', '+']);
+const SEQUENCE = /^[ A-Za-z0-9]{0,8}$/;
+
+// Enterprise PL/I reads columns 2 to 72 unless *PROCESS MARGINS says otherwise. Public source is
+// often written from column 1 to any width, and read with MARGINS(2,72) it loses its first column
+// and its tail, so the margins are taken from the file itself and the reason recorded.
+export function marginsOf(text) {
+  const lines = text.split(/\r?\n/);
+  for (const l of lines) {
+    const m = PROCESS_LINE.exec(l);
+    const opt = m && /\bMAR(?:GINS)?\s*\(\s*(\d+)\s*,\s*(\d+)\s*(?:,\s*(\d+)\s*)?\)/i.exec(m[1]);
+    if (opt) return { left: Number(opt[1]), right: Number(opt[2]), carriage: opt[3] ? Number(opt[3]) : 0, reason: 'process-option' };
+  }
+  const body = lines.filter((l) => !PROCESS_LINE.test(l));
+  const long = body.filter((l) => l.replace(/\s+$/, '').length > 72);
+  const sequenceLike = long.filter((l) => l.length <= 80 && SEQUENCE.test(l.slice(72).replace(/\s+$/, '')) && /\d/.test(l.slice(72))).length;
+  const sequenced = long.length > 0 && sequenceLike >= 0.95 * long.length;
+  const firstColumnFree = body.every((l) => !l.length || CARRIAGE.has(l[0]));
+  const left = firstColumnFree ? 2 : 1;
+  const right = long.length === 0 || sequenced ? 72 : Infinity;
+  const carriage = left === 2 && body.some((l) => l.length && l[0] !== ' ') ? 1 : 0;
+  const reason = right === Infinity ? 'text-beyond-72' : sequenced ? 'sequence-area' : 'default';
+  return { left, right, carriage, reason };
+}
+
+const NOT = new Set(['¬', '^', '~']);
+const OPS3 = ['||=', '**=', '¬=>', '^=>'];
+const OPS2 = ['->', '=>', '¬=', '^=', '~=', '<>', '<=', '>=', '¬<', '¬>', '^<', '^>', '~<', '~>', '||', '!!', '**', '+=', '-=', '*=', '/=', '|=', '&='];
+const OPS1 = '+-*/=<>&|!:.,()%?';
+
+const LIT_SUFFIX = /^(?:B[1-4]?X?|BX|XN|XU|GX|WX|UX|X|G|M|W|A|E|U)(?![A-Za-z0-9_$#@])/i;
+const NUMBER = /^(?:\d[\d_]*(?:\.\d*)?|\.\d+)(?:[EeSsDdQq][+-]?\d+)?(?:[Bb](?![A-Za-z0-9_$#@]))?(?:[Ii](?![A-Za-z0-9_$#@]))?/;
+const WORD_START = /[A-Za-z$#@_À-ɏ]/;
+const WORD_REST = /[A-Za-z0-9$#@_À-ɏ]/;
+
+// Tokens over the whole text, so a comment or literal that runs across lines is one token.
+// { t: 'word', v, u } | { t: 'lit', v, suffix } | { t: 'num', v } | { t: 'op', v } | { t: 'semi' },
+// each with line and col as the source has them.
+export function tokenize(text, { file = null, margins = marginsOf(text) } = {}) {
+  const diags = [];
+  const process = [];
+  const physical = text.split(/\r?\n/);
+  const rows = physical.map((l, k) => {
+    if (PROCESS_LINE.test(l)) { process.push({ line: k + 1, options: PROCESS_LINE.exec(l)[1].replace(/;\s*$/, '').trim() }); return ''; }
+    return l.slice(margins.left - 1, margins.right === Infinity ? undefined : margins.right).replace(/\t/g, ' ');
+  });
+  const tokens = [];
+  const at = (line, col) => ({ line, col: col + margins.left, ...(file ? { file } : {}) });
+  let line = 0;
+  let i = 0;
+  const source = rows;
+  while (line < source.length) {
+    const s = source[line];
+    if (i >= s.length) { line++; i = 0; continue; }
+    const c = s[i];
+    if (c === ' ' || c === '\f' || c === '\r' || c === '\v') { i++; continue; }
+    if (c === '/' && s[i + 1] === '*') {
+      const start = at(line + 1, i);
+      let l = line, k = i + 2, closed = false;
+      while (l < source.length) {
+        const e = source[l].indexOf('*/', k);
+        if (e >= 0) { line = l; i = e + 2; closed = true; break; }
+        l++; k = 0;
+      }
+      if (!closed) { diags.push({ sev: 'error', kind: 'unterminated-comment', ...start }); line = source.length; }
+      continue;
+    }
+    if (c === "'" || c === '"') {
+      const start = at(line + 1, i);
+      let v = '', l = line, k = i + 1, closed = false;
+      while (l < source.length) {
+        const row = source[l];
+        if (k >= row.length) { l++; k = 0; continue; }
+        if (row[k] === c) {
+          if (row[k + 1] === c) { v += c; k += 2; continue; }
+          closed = true; k++; break;
+        }
+        v += row[k++];
+      }
+      if (!closed) { diags.push({ sev: 'error', kind: 'unterminated-literal', ...start }); line = source.length; i = 0; tokens.push({ t: 'lit', v, suffix: '', ...start }); continue; }
+      line = l; i = k;
+      const suf = LIT_SUFFIX.exec(source[line].slice(i));
+      const suffix = suf ? suf[0].toUpperCase() : '';
+      if (suf) i += suf[0].length;
+      tokens.push({ t: 'lit', v, suffix, quote: c, ...start });
+      continue;
+    }
+    const num = /[0-9.]/.test(c) ? NUMBER.exec(s.slice(i)) : null;
+    if (num && !(c === '.' && !/[0-9]/.test(s[i + 1] || ''))) {
+      tokens.push({ t: 'num', v: num[0], ...at(line + 1, i) });
+      i += num[0].length;
+      continue;
+    }
+    if (WORD_START.test(c)) {
+      let j = i + 1;
+      while (j < s.length && WORD_REST.test(s[j])) j++;
+      const v = s.slice(i, j);
+      tokens.push({ t: 'word', v, u: v.toUpperCase(), ...at(line + 1, i) });
+      i = j;
+      continue;
+    }
+    if (c === ';') { tokens.push({ t: 'semi', ...at(line + 1, i) }); i++; continue; }
+    const three = s.slice(i, i + 3);
+    const two = s.slice(i, i + 2);
+    const op = OPS3.find((o) => o === three) || OPS2.find((o) => o === two) || (OPS1.includes(c) || NOT.has(c) ? c : null);
+    if (op) {
+      tokens.push({ t: 'op', v: normalOp(op), ...at(line + 1, i) });
+      i += op.length;
+      continue;
+    }
+    diags.push({ sev: 'warn', kind: 'unexpected-char', char: c, ...at(line + 1, i) });
+    i++;
+  }
+  return { tokens, diags, process, margins };
+}
+
+const normalOp = (op) => op.replace(/^[\^~]/, '¬').replace(/^!!$/, '||').replace(/^!$/, '|');
+
+// A leading `name:` is a label, `name(3):` a subscripted label, and `(SIZE, NOFOFL):` a condition
+// prefix. They may repeat and mix, and are taken off before the statement is classified.
+function prefixes(toks) {
+  const labels = [];
+  const conditions = [];
+  let i = 0;
+  for (;;) {
+    const a = toks[i], b = toks[i + 1];
+    if (a && a.t === 'word' && b && b.t === 'op' && b.v === ':') { labels.push({ name: a.u, line: a.line }); i += 2; continue; }
+    if (a && a.t === 'word' && b && b.t === 'op' && b.v === '(') {
+      const close = closing(toks, i + 1);
+      if (close > 0 && toks[close + 1] && toks[close + 1].t === 'op' && toks[close + 1].v === ':' && toks.slice(i + 2, close).every((t) => t.t === 'num' || (t.t === 'op' && (t.v === ',' || t.v === '-' || t.v === '+')))) {
+        labels.push({ name: a.u, line: a.line, subscript: toks.slice(i + 2, close).map((t) => t.v).join('') });
+        i = close + 2;
+        continue;
+      }
+    }
+    if (a && a.t === 'op' && a.v === '(') {
+      const close = closing(toks, i);
+      if (close > 0 && toks[close + 1] && toks[close + 1].t === 'op' && toks[close + 1].v === ':' && toks.slice(i + 1, close).every((t) => t.t === 'word' || (t.t === 'op' && t.v === ','))) {
+        for (const t of toks.slice(i + 1, close)) if (t.t === 'word') conditions.push(t.u);
+        i = close + 2;
+        continue;
+      }
+    }
+    return { labels, conditions, at: i };
+  }
+}
+
+// The index of the parenthesis closing the one at `open`, or -1 when it is not closed.
+export function closing(toks, open) {
+  let depth = 0;
+  for (let k = open; k < toks.length; k++) {
+    const t = toks[k];
+    if (t.t !== 'op') continue;
+    if (t.v === '(') depth++;
+    else if (t.v === ')' && --depth === 0) return k;
+  }
+  return -1;
+}
+
+// Statements split at semicolons. Each carries its prefixes and the tokens after them; a trailing
+// run with no semicolon is kept as a statement and flagged, never dropped.
+export function statements(tokens) {
+  const out = [];
+  let cur = [];
+  const close = (semi) => {
+    if (!cur.length && !semi) return;
+    const { labels, conditions, at } = prefixes(cur);
+    const toks = cur.slice(at);
+    const first = cur[0] || semi;
+    const last = cur[cur.length - 1] || semi;
+    out.push({ labels, conditions, toks, line: first.line, endLine: (semi || last).line, ...(first.file ? { file: first.file } : {}), ...(semi ? {} : { unterminated: true }) });
+    cur = [];
+  };
+  for (const t of tokens) {
+    if (t.t === 'semi') close(t);
+    else cur.push(t);
+  }
+  close(null);
+  return out;
+}
+
+// UTF-8 where the bytes are valid UTF-8, else Latin-1: a ¬ read as Latin-1 from UTF-8 is two
+// characters and moves every column after it.
+export function sourceText(buf) {
+  const utf8 = buf.toString('utf8');
+  return utf8.includes('�') ? buf.toString('latin1') : utf8;
+}
+
+export function readPli(text, opts = {}) {
+  const { tokens, diags, process, margins } = tokenize(text, opts);
+  return { statements: statements(tokens), diags, process, margins };
+}