@portll/cobolwork 0.2.140 → 0.3.0

package/lib/inventory.mjs

@@ -1,7 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-or-later
-import { dirname } from 'node:path';
-import { parseSource, buildFileIndex, detectFormat } from './parser.mjs';
-import { inScope, isCopybook, isJcl, isProgram, readSource, relPath } from './sources.mjs';
+import { detectFormat } from './parser.mjs';
+import { inScope, isCopybook, isJcl, isProgram, relPath } from './sources.mjs';
 import { SCHEMA_VERSION, TOOL_VERSION } from './version.mjs';
 import { treeFor } from './kernel/source-tree.mjs';
 
@@ -36,19 +35,19 @@ export function inventory(root, opts = {}) {
     out.summary.programFiles++;
     let src;
     try {
-      const s = readSource(f);
+      const s = tree.text(f);
       src = s.text;
-      if (s.encoding === 'ebcdic') { out.summary.filesEbcdic++; out.ebcdic.push(relPath(root, f)); }
-    } catch (e) { out.summary.filesUnreadable++; out.unreadable.push(`${relPath(root, f)}: ${e.code || e.name}`); continue; }
+      if (s.encoding === 'ebcdic') { out.summary.filesEbcdic++; out.ebcdic.push(tree.rel(f)); }
+    } catch (e) { out.summary.filesUnreadable++; out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`); continue; }
     // A file with a program extension and no program in it is usually a copybook named .cbl; it is
     // counted so the difference between program files and programs read is accounted for.
     if (!/PROCEDURE\s+DIVISION|PROGRAM-ID/i.test(src)) { out.summary.notPrograms++; continue; }
     let res;
     try {
-      res = parseSource(src, f, { format: 'auto', includeDirs: idx.copyDirs, fileIndex: idx.index, mainDir: dirname(f), copyFormat: 'auto', systemDirs: opts.systemDirs || [] });
+      res = tree.parse(f, src);
     } catch (e) {
       out.summary.filesUnreadable++;
-      out.unreadable.push(`${relPath(root, f)}: ${e.code || e.name}`);
+      out.unreadable.push(`${tree.rel(f)}: ${e.code || e.name}`);
       continue;
     }
     out.summary.filesScanned++;
@@ -60,7 +59,7 @@ export function inventory(root, opts = {}) {
     if (/EXEC\s+DLI/i.test(src)) out.summary.withDli++;
     for (const c of res.copies) {
       if (c.status === 'resolved') out.summary.copiesResolved++;
-      else if (String(c.status).startsWith('refused')) { out.summary.copiesRefused++; out.refusedCopies.push({ name: c.name, from: relPath(root, c.file), line: c.line, why: c.status }); }
+      else if (String(c.status).startsWith('refused')) { out.summary.copiesRefused++; out.refusedCopies.push({ name: c.name, from: tree.rel(c.file), line: c.line, why: c.status }); }
       // Before the system names, which a copybook in the tree is free to take.
       else if (c.status === 'expansion-limit') out.summary.copiesOverLimit = (out.summary.copiesOverLimit || 0) + 1;
       else if (c.status === 'system' || SYSTEM_COPY.test(c.name)) out.summary.copiesSystem++;

package/lib/jcl.mjs

@@ -13,7 +13,9 @@
 // there.
 import { readSource } from './sources.mjs';
 import { copiesOf, tsoCommands } from './utilities.mjs';
-import { statementCard } from './card.mjs';
+import { statementCard, parseOperands } from './cards.mjs';
+
+export { splitOperands, parseOperands } from './cards.mjs';
 
 // The operations a statement may carry. Anything else in the operation field is a statement this
 // reader does not know, which is a diagnostic rather than a silent skip.
@@ -38,53 +40,6 @@ const SYSTEM_SYMBOLS = new Set(`SYSUID SYSALVL SYSCLONE SYSNAME SYSOSLVL SYSPLEX
   YYMMDD LYYMMDD HHMMSS LHHMMSS DAY HR MIN SEC JDAY MON YR2 YR4 WDAY
   LDATE LDAY LHR LMIN LSEC LJDAY LMON LYR2 LYR4 LWDAY LTIME JOBNAME DS SEQ DATE TIME`.split(/\s+/));
 
-// Splits an operand field on commas that are not inside parentheses or quotes. JCL nests both, and
-// a naive split on comma turns DISP=(NEW,CATLG,DELETE) into three operands.
-export function splitOperands(s) {
-  const out = [];
-  let depth = 0, quoted = false, start = 0;
-  for (let i = 0; i < s.length; i++) {
-    const c = s[i];
-    if (quoted) { if (c === "'") { if (s[i + 1] === "'") i++; else quoted = false; } continue; }
-    if (c === "'") quoted = true;
-    else if (c === '(') depth++;
-    else if (c === ')') depth = Math.max(0, depth - 1);
-    else if (c === ',' && depth === 0) { out.push(s.slice(start, i)); start = i + 1; }
-  }
-  out.push(s.slice(start));
-  return out.filter((x, i, a) => x.length || i < a.length - 1);
-}
-
-// Operands are positional until the first KEY=VALUE, and keyword after it. Both forms matter:
-// EXEC takes its procedure name positionally, and everything interesting on DD is a keyword.
-export function parseOperands(field) {
-  const positional = [];
-  const keywords = new Map();
-  for (const raw of splitOperands(field)) {
-    const part = raw.trim();
-    if (!part) continue;
-    const eq = keywordSplit(part);
-    if (eq < 0) positional.push(part);
-    else keywords.set(part.slice(0, eq).toUpperCase(), part.slice(eq + 1));
-  }
-  return { positional, keywords };
-}
-
-// The first '=' that is not inside parentheses or quotes. DCB=(RECFM=FB,LRECL=80) is one keyword
-// whose value happens to contain more of them.
-function keywordSplit(part) {
-  let depth = 0, quoted = false;
-  for (let i = 0; i < part.length; i++) {
-    const c = part[i];
-    if (quoted) { if (c === "'") { if (part[i + 1] === "'") i++; else quoted = false; } continue; }
-    if (c === "'") quoted = true;
-    else if (c === '(') depth++;
-    else if (c === ')') depth = Math.max(0, depth - 1);
-    else if (c === '=' && depth === 0) return i;
-  }
-  return -1;
-}
-
 // Symbolic substitution. &NAME ends at a non-name character, and a trailing dot is a separator
 // that is consumed rather than kept: &PREFIX..DATA resolves to <prefix>.DATA.
 export function substitute(text, symbols) {

package/lib/kernel/git.mjs

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// Reading a revision out of a repository with git plumbing only.
+import { spawnSync } from 'node:child_process';
+
+// A reviewed repository's .git/config can name commands; with fsmonitor off and plumbing only, git runs none.
+const GIT_ENV = { ...process.env, GIT_OPTIONAL_LOCKS: '0', GIT_TERMINAL_PROMPT: '0' };
+export const git = (repo, args, opts = {}) => spawnSync('git', ['-c', 'core.fsmonitor=false', '-C', repo, ...args], { env: GIT_ENV, maxBuffer: 256 * 1024 * 1024, ...opts });
+export const refused = (what, ref, why) => Object.assign(new Error(`${what} ${ref} failed: ${why}`), { code: 'EDIFFREF' });
+const BATCH_BYTES = 64 * 1024 * 1024;
+
+export function treeOf(repo, ref) {
+  if (!ref || String(ref).startsWith('-')) throw refused('git rev-parse', ref, 'a revision cannot be empty or start with "-"');
+  const r = git(repo, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${ref}^{tree}`], { encoding: 'utf8' });
+  const oid = String(r.stdout || '').trim();
+  if (r.status !== 0 || !/^[0-9a-f]{40}([0-9a-f]{24})?$/.test(oid)) throw refused('git rev-parse', ref, String(r.stderr || '').trim() || 'not a revision');
+  return oid;
+}
+
+// The parts of a tree path, or null if a part is empty, climbs, names a drive or stream, or is .git.
+export function treePathParts(path) {
+  const parts = path.split('/');
+  return parts.some((p) => !p || p === '.' || p === '..' || /[\\:\0]/.test(p) || p.toLowerCase() === '.git') ? null : parts;
+}
+
+// Every file in the tree `oid` as committed, as [{ path, bytes }] in tree order: blobs as stored,
+// with no filter or line-ending conversion. Links and submodules are left out and counted.
+export function revisionBlobs(repo, oid, ref = oid) {
+  const listed = git(repo, ['ls-tree', '-r', '-z', '-l', '--full-tree', oid]);
+  if (listed.status !== 0) throw refused('git ls-tree', ref, String(listed.stderr || '').trim() || `exit ${listed.status}`);
+  const blobs = [];
+  let links = 0;
+  for (const entry of listed.stdout.toString('utf8').split('\0')) {
+    const m = /^(\d{6}) blob ([0-9a-f]+) +(\d+)\t(.+)$/s.exec(entry);
+    if (!m) continue;
+    if (m[1] === '120000') { links++; continue; }
+    blobs.push({ oid: m[2], size: Number(m[3]), path: m[4] });
+  }
+  // Batched by bytes as well as count, and each batch's buffer sized to hold it, so one large blob
+  // cannot overflow a buffer sized for the others.
+  const batches = [];
+  for (const b of blobs) {
+    const last = batches[batches.length - 1];
+    if (!last || last.length >= 500 || last.bytes + b.size > BATCH_BYTES) batches.push(Object.assign([b], { bytes: b.size }));
+    else { last.push(b); last.bytes += b.size; }
+  }
+  for (const batch of batches) {
+    const r = git(repo, ['cat-file', '--batch'], { input: batch.map((b) => b.oid).join('\n') + '\n', maxBuffer: batch.bytes + batch.length * 128 + 4096 });
+    if (r.status !== 0) throw refused('git cat-file', ref, String(r.stderr || '').trim() || (r.error ? r.error.code || r.error.message : `exit ${r.status}`));
+    let at = 0;
+    for (const b of batch) {
+      const nl = r.stdout.indexOf(0x0a, at);
+      const head = r.stdout.toString('utf8', at, nl).split(' ');
+      if (head[1] !== 'blob') throw refused('git cat-file', ref, `${b.oid} is ${head[1] || 'missing'}`);
+      const size = Number(head[2]);
+      b.bytes = r.stdout.subarray(nl + 1, nl + 1 + size);
+      at = nl + 1 + size + 1;
+    }
+  }
+  return { blobs, links };
+}

package/lib/kernel/identity.mjs

@@ -131,14 +131,15 @@ function codeText(line, kind, format) {
   return maskSecrets(s).replace(/\s+/g, ' ').trim();
 }
 
-// Reads each file once, however many findings it holds.
+// Reads each file once, however many findings it holds, from disk or through the reader the
+// caller's source tree supplies.
 function fileFacts(root, path, cache) {
   let facts = cache.get(path);
   if (facts) return facts;
   facts = { kind: 'other', lines: [], scopes: null, format: null };
   cache.set(path, facts);
   let src;
-  try { src = readSource(join(root, path)).text; } catch { return facts; }
+  try { src = cache.read ? cache.read(path) : readSource(join(root, path)).text; } catch { return facts; }
   facts.lines = src.split(/\r?\n/);
   if (isJcl(join(root, path))) {
     facts.kind = 'jcl';
@@ -171,8 +172,8 @@ const digest = (parts) => createHash('sha256').update(parts.join('\u0000')).dige
 // Stamps `fingerprint` on every finding and returns how many shared one with an earlier finding.
 // `repo` names the repository when one report holds several, where the same program id in two
 // repositories is two programs.
-export function stampFingerprints(findings, { root, repo = '' } = {}) {
-  const cache = new Map();
+export function stampFingerprints(findings, { root, repo = '', read = null } = {}) {
+  const cache = Object.assign(new Map(), { read });
   const seen = new Set();
   let shared = 0;
   for (const f of findings) {
@@ -188,8 +189,8 @@ export function stampFingerprints(findings, { root, repo = '' } = {}) {
 // Looser keys for one finding seen in two trees whose line was edited between them: `scope` is the
 // fingerprint without the line's text, and `route` names a data-flow finding by the statement its
 // trace starts at. Null where a finding has no route.
-export function pairingKeys(findings, { root, repo = '' } = {}) {
-  const cache = new Map();
+export function pairingKeys(findings, { root, repo = '', read = null } = {}) {
+  const cache = Object.assign(new Map(), { read });
   return findings.map((f) => {
     const { scope, subject } = partsOf(f, root, cache);
     const src = f.evidence === 'path' && f.related && f.related[0];

package/lib/kernel/source-tree.mjs

@@ -26,13 +26,16 @@
 //
 // An adapter that does not have a filesystem underneath it has to make that decision itself, and
 // `contains` is where it makes it. For a tree held in memory it is key membership. For a git
-// revision it is membership of the tree object, which cannot name a path outside itself. Anything
-// cleverer than those two deserves the six containment tests pointed at it before it ships:
-// test/sources.test.mjs:44, :63, :88, :101, :144 and test/review.test.mjs:114.
-import { readFileSync, realpathSync } from 'node:fs';
-import { dirname, resolve, sep } from 'node:path';
+// revision it is membership of the tree object, which cannot name a path outside itself. For a PDS
+// export it is membership of the members the walk found, each read with no link followed. Anything
+// cleverer than those deserves the six containment tests pointed at it before it ships:
+// test/sources.test.mjs:44, :63, :88, :101, :144 and test/review.test.mjs:114, as
+// test/git-tree.test.mjs and test/pds-export.test.mjs do.
+import { closeSync, constants, fstatSync, openSync, readFileSync, readSync, realpathSync } from 'node:fs';
+import { basename, dirname, relative, resolve, sep } from 'node:path';
 import { buildFileIndex, parseSource } from '../parser.mjs';
-import { readSource, relPath } from '../sources.mjs';
+import { classifyUnder, decodeSource, diskClassifier, heldClassifier, kindOfBytes, looksEbcdic, readSource, relPath } from '../sources.mjs';
+import { revisionBlobs, treeOf, treePathParts } from './git.mjs';
 
 // The shape every adapter answers to. Checked rather than documented, because an adapter missing a
 // method fails at the first rule set that happens to call it rather than at the boundary.
@@ -64,10 +67,14 @@ export function directoryTree(root, opts = {}) {
   const idx = buildFileIndex(root);
   const systemDirs = opts.systemDirs || [];
   const top = resolve(root);
+  const kindOf = diskClassifier();
+  classifyUnder(root, kindOf);
+  if (top !== root) classifyUnder(top, kindOf);
 
   return {
     kind: 'directory',
     root,
+    kindOf,
     // The index itself, for the two callers that need more than a list: the copybook set reads
     // copyDirs, and the inventory reports on unreadable directories and symlinks.
     index: idx,
@@ -92,40 +99,203 @@ export function directoryTree(root, opts = {}) {
   };
 }
 
-// A tree that was never on disk, for tests. `files` maps a path to its contents, as a string or a
-// Buffer. Paths are used exactly as given, so a test reads the way it writes.
-//
-// It cannot parse. parseSource resolves COPY statements against real directories and reads them
-// with readSource, so a program with copybooks needs a filesystem underneath it. A rule set that
-// only reads text - hidden, recon, vendor, jcl, build - works against this; one that parses does
-// not, and says so rather than parsing something unexpected.
-export function memoryTree(files, { root = '/memory' } = {}) {
-  const store = new Map(Object.entries(files).map(([p, v]) => [p, Buffer.isBuffer(v) ? v : Buffer.from(v, 'latin1')]));
+// A tree whose files are held rather than on disk. The parser finds copybooks in the index built
+// here and reads them through `readText`, so a COPY resolves to a held file or to a system copy
+// library outside the tree, never to a file on disk that happens to sit under this tree's root.
+function heldTree({ kind, root, store, rel, systemDirs = [], copyDirs = null, classify = heldClassifier,
+  symlinks = { followed: 0, outside: 0, broken: 0 }, unreadableDirs = [] }) {
+  const index = new Map();
+  const dirs = new Set(copyDirs || []);
+  for (const p of store.keys()) {
+    index.set(resolve(root, p).toLowerCase(), p);
+    if (!copyDirs && /\.(cpy|copy|inc|cbl|cob)$/i.test(p)) dirs.add(dirname(resolve(root, p)));
+  }
+  index.root = root;
+  const top = resolve(root);
+  const absent = (p) => Object.assign(new Error(`no such file in this tree: ${p}`), { code: 'ENOENT' });
+  const bytes = (p) => {
+    const b = store.get(p);
+    if (!b) throw absent(p);
+    return b;
+  };
+  const kindOf = classify(bytes);
+  classifyUnder(top, kindOf);
+  const text = (p) => decodeSource(bytes(p));
+  const readText = (p) => {
+    if (store.has(p)) return text(p).text;
+    const r = resolve(root, p);
+    if (r === top || r.startsWith(top + sep)) throw absent(p);
+    return readSource(p).text;
+  };
   return {
-    kind: 'memory',
+    kind,
     root,
-    index: { index: store, copyDirs: [], unreadableDirs: [], symlinks: { followed: 0, outside: 0, broken: 0 } },
+    kindOf,
+    index: { index, copyDirs: [...dirs].sort(), unreadableDirs, symlinks },
     list: () => [...store.keys()].sort(),
-    bytes: (p) => {
-      const b = store.get(p);
-      if (!b) throw Object.assign(new Error(`no such file in this tree: ${p}`), { code: 'ENOENT' });
-      return b;
-    },
-    text: (p) => {
-      const b = store.get(p);
-      if (!b) throw Object.assign(new Error(`no such file in this tree: ${p}`), { code: 'ENOENT' });
-      return { text: b.toString('latin1'), encoding: 'latin1' };
-    },
-    rel: (p) => String(p).replace(/\\/g, '/').replace(new RegExp('^' + root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '/?'), ''),
+    bytes,
+    text,
+    rel,
     // Containment by construction: a path this tree does not hold is a path outside it.
     contains: (p) => store.has(p),
-    parse: () => {
-      throw Object.assign(
-        new Error('a memory tree cannot parse: COPY resolution reads directories from disk'),
-        { code: 'ETREEPARSE' },
-      );
-    },
+    parse: (file, t) => parseSource(t ?? text(file).text, file, {
+      format: 'auto',
+      includeDirs: [...dirs].sort(),
+      fileIndex: index,
+      mainDir: dirname(resolve(root, file)),
+      copyFormat: 'auto',
+      systemDirs,
+      readText,
+    }),
+  };
+}
+
+// A tree that was never on disk, for tests. `files` maps a path to its contents, as a string or a
+// Buffer. Paths are used exactly as given, so a test reads the way it writes.
+export function memoryTree(files, { root = '/memory', systemDirs = [] } = {}) {
+  const store = new Map(Object.entries(files).map(([p, v]) => [p, Buffer.isBuffer(v) ? v : Buffer.from(v, 'latin1')]));
+  const rel = (p) => String(p).replace(/\\/g, '/').replace(new RegExp('^' + root.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '/?'), '');
+  return heldTree({ kind: 'memory', root, store, rel, systemDirs });
+}
+
+// A git revision, read with plumbing into memory: nothing is written to disk. Its root is a path
+// beside the repository that does not exist, `<repo>@<tree>`, so every path it answers for is
+// absolute and none of them can be found on disk. The tree object is the containment: it cannot
+// name a path outside itself, and a path that would climb, name a drive or stream, or enter .git
+// is left out, as it is when a revision is written to disk. Links and submodules are not read.
+export function gitTree(repo, ref, opts = {}) {
+  const oid = treeOf(repo, ref);
+  const { blobs, links } = revisionBlobs(repo, oid, ref);
+  const root = `${resolve(repo)}@${oid.slice(0, 12)}`;
+  const store = new Map();
+  for (const b of blobs) {
+    const parts = treePathParts(b.path);
+    if (parts) store.set(resolve(root, ...parts), b.bytes);
+  }
+  const tree = heldTree({ kind: 'git', root, store, rel: (p) => relPath(root, p), systemDirs: opts.systemDirs || [],
+    symlinks: { followed: 0, outside: 0, broken: 0, notRead: links } });
+  return Object.assign(tree, { ref, oid });
+}
+
+// z/OSMF's record mode (zowe --record) writes each record after its length, four bytes big-endian,
+// and nothing between records. Read as that only when the lengths chain exactly to the end of the
+// file, and given back as lines; anything else is the file as it is.
+const MAX_LRECL = 32760;
+export function unframeRecords(buf) {
+  let records = 0;
+  for (let at = 0; at < buf.length; records++) {
+    if (buf.length - at < 4) return buf;
+    const n = buf.readUInt32BE(at);
+    if (n > MAX_LRECL || buf.length - at - 4 < n) return buf;
+    at += 4 + n;
+  }
+  if (!records || records * 4 === buf.length) return buf;
+  const lines = (end) => {
+    const out = Buffer.alloc(buf.length - records * 3);
+    let w = 0;
+    for (let at = 0; at < buf.length;) {
+      const n = buf.readUInt32BE(at);
+      w += buf.copy(out, w, at + 4, at + 4 + n);
+      out[w++] = end;
+      at += 4 + n;
+    }
+    return out;
   };
+  const ebcdic = lines(0x15);
+  return looksEbcdic(ebcdic) ? ebcdic : lines(0x0A);
+}
+
+// A member is read from the file the walk found. A link put in its place since, or a file that is
+// not a regular one, is refused rather than followed or waited on.
+function readMember(file) {
+  const fd = openSync(file, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
+  try {
+    const st = fstatSync(fd);
+    if (!st.isFile()) throw Object.assign(new Error(`not a regular file: ${file}`), { code: 'ENOTFILE' });
+    const buf = Buffer.alloc(st.size);
+    let n = 0;
+    while (n < st.size) {
+      const r = readSync(fd, buf, n, st.size - n, n);
+      if (!r) break;
+      n += r;
+    }
+    return unframeRecords(buf.subarray(0, n));
+  } finally { closeSync(fd); }
+}
+
+const QUALIFIER = /^[A-Z@#$][A-Z0-9@#$-]{0,7}$/;
+const MEMBER = /^([A-Z@#$][A-Z0-9@#$]{0,7})(?:\.[^.]*)?$/i;
+const MAX_LISTED = 8;
+
+// Partitioned data sets exported to a directory, a member to a file: what
+// `zowe zos-files download all-members` writes (ibmuser/new/cntl/member.txt, lower-cased unless
+// --preserve-original-letter-case, with whatever extension -e gave), or a directory named for the
+// data set with its members copied in. Each member is held as `DATA.SET.NAME/MEMBER` under a root
+// beside the export that does not exist, so a finding names the member as z/OS does however it was
+// downloaded, and nothing under that root is ever read from disk.
+//
+// The extension is the download's choice, not the language's, so members are classified by their
+// contents. A file whose directory is not a data set name or whose name is not a member name is not
+// read and is counted, as is a second file for a member already held. Every data set that holds
+// anything but JCL and maps is a copy library, searched in name order: the export does not say
+// what SYSLIB concatenated.
+export function pdsExportTree(dir, opts = {}) {
+  const idx = buildFileIndex(dir);
+  const top = realpathSync(dir);
+  const root = `${resolve(dir)}@pds`;
+  const origins = new Map();
+  const found = new Map();
+  const notMembers = [];
+  const duplicates = [];
+  const dataSets = new Set();
+  for (const file of [...idx.index.values()].sort()) {
+    const where = relative(dir, dirname(file)).split(sep).filter(Boolean);
+    const dsn = where.join('.').toUpperCase();
+    const member = MEMBER.exec(basename(file));
+    if (!member || dsn.length > 44 || (dsn && !dsn.split('.').every((q) => QUALIFIER.test(q)))) {
+      notMembers.push(relPath(dir, file));
+      continue;
+    }
+    const at = resolve(root, ...(dsn ? [dsn] : []), member[1].toUpperCase());
+    if (found.has(at)) { duplicates.push(`${relPath(dir, file)}: ${relPath(root, at)} is ${found.get(at)}`); continue; }
+    let real;
+    try { real = realpathSync(file); } catch { real = file; }
+    if (real !== top && !real.startsWith(top + sep)) { notMembers.push(relPath(dir, file)); continue; }
+    origins.set(at, real);
+    found.set(at, relPath(dir, file));
+    dataSets.add(dirname(at));
+  }
+  const kinds = new Map();
+  const byKind = {};
+  const copyDirs = new Set();
+  for (const [at, file] of origins) {
+    let kind = null;
+    try { kind = kindOfBytes(readMember(file)); } catch { kind = null; }
+    kinds.set(at, kind);
+    byKind[kind || 'other'] = (byKind[kind || 'other'] || 0) + 1;
+    if (kind !== 'jcl' && kind !== 'bms') copyDirs.add(dirname(at));
+  }
+  const store = {
+    keys: () => origins.keys(),
+    has: (p) => origins.has(p),
+    get: (p) => (origins.has(p) ? readMember(origins.get(p)) : undefined),
+  };
+  const tree = heldTree({
+    kind: 'pds-export', root, store, rel: (p) => relPath(root, p), systemDirs: opts.systemDirs || [],
+    copyDirs: [...copyDirs], classify: () => (p) => kinds.get(p) ?? null, symlinks: idx.symlinks,
+    unreadableDirs: idx.unreadableDirs.map((d) => resolve(root, relative(dir, d))),
+  });
+  return Object.assign(tree, {
+    dir,
+    origin: (p) => found.get(p) ?? null,
+    export: {
+      dataSets: dataSets.size,
+      members: origins.size,
+      byKind,
+      ...(notMembers.length ? { notMembers: notMembers.length, notMembersListed: notMembers.slice(0, MAX_LISTED) } : {}),
+      ...(duplicates.length ? { duplicates: duplicates.length, duplicatesListed: duplicates.slice(0, MAX_LISTED) } : {}),
+    },
+  });
 }
 
 // What a rule set calls. Given a tree it uses it; given none it builds the directory one, so every

package/lib/parser.mjs

@@ -446,11 +446,18 @@ export function tokenize(norm, file) {
   return { tokens: out, diags };
 }
 
-// Every rule set walks the tree through here. A directory it may not read is recorded, not treated
-// as empty: only ENOENT means absent. Symlinks are followed while they stay inside the tree, so a
-// symlinked copy library is read; one pointing outside is counted and never followed, because a
-// scan reads the tree it was given and nothing else.
-export function buildFileIndex(root) {
+// A drive that stalls or drops for a moment answers a listing with ENOENT or an I/O error. Such a
+// listing is tried again after a pause; the pauses add up to about 1.3 seconds.
+const LISTING_RETRIED = new Set(['ENOENT', 'EIO', 'ETIMEDOUT', 'ENXIO', 'EBUSY', 'EAGAIN', 'EINTR', 'ESTALE', 'ENOTCONN']);
+const LISTING_PAUSES_MS = [50, 250, 1000];
+const pause = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+
+// Every rule set walks the tree through here. A directory that cannot be listed is recorded, not
+// treated as empty: a scan over it is incomplete, not a scan of a smaller tree. Only the root's own
+// ENOENT means absent; a directory the walk found in its parent was there. Symlinks are followed
+// while they stay inside the tree, so a symlinked copy library is read; one pointing outside is
+// counted and never followed, because a scan reads the tree it was given and nothing else.
+export function buildFileIndex(root, { readdir = readdirSync, wait = pause } = {}) {
   const index = new Map();
   const dirs = new Set();
   const unreadableDirs = [];
@@ -460,16 +467,20 @@ export function buildFileIndex(root) {
   const inside = (p) => p === top || p.startsWith(top + sep);
   const visited = new Set();
   const addFile = (p, name, d) => { index.set(p.toLowerCase(), p); if (/\.(cpy|copy|inc|cbl|cob)$/i.test(name)) dirs.add(d); };
+  const list = (d) => {
+    for (let i = 0; ; i++) {
+      try { return readdir(d, { withFileTypes: true }); } catch (e) {
+        if (i >= LISTING_PAUSES_MS.length || !LISTING_RETRIED.has(e.code)) throw e;
+        wait(LISTING_PAUSES_MS[i]);
+      }
+    }
+  };
   // Each real directory is walked once, however many links lead to it, or its programs count twice.
   const walk = (d, real) => {
     if (visited.has(real)) return;
     visited.add(real);
     let es;
-    try { es = readdirSync(d, { withFileTypes: true }); } catch (e) {
-      if (e.code === 'ENOENT') return;
-      if (e.code === 'EACCES' || e.code === 'EPERM') { unreadableDirs.push(d); return; }
-      throw e;
-    }
+    try { es = list(d); } catch { unreadableDirs.push(d); return; }
     for (const e of es.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0))) {
       if (e.name === '.git' || e.name.startsWith('._')) continue;
       const p = join(d, e.name);
@@ -634,7 +645,7 @@ function isProgramFile(p, ctx) {
   const seen = (ctx.programFiles ||= new Map());
   if (!seen.has(p)) {
     let text = '';
-    try { text = readSource(p).text; } catch { /* an unreadable candidate is not known to be a program */ }
+    try { text = ctx.readText(p); } catch { /* an unreadable candidate is not known to be a program */ }
     seen.set(p, /^[^*\n]{0,6}\s*PROGRAM-ID\s*\./im.test(text));
   }
   return seen.get(p);
@@ -787,7 +798,7 @@ function includeCopy(name, lib, pairs, at, via, ctx, stack, inheritedFormat) {
   if (!path) return [];
   if (stack.includes(path) || stack.length > 40) { record.status = 'recursive'; return []; }
   if (++ctx.inclusions > MAX_INCLUSIONS || ctx.copyTokens > MAX_COPY_TOKENS) { record.status = 'expansion-limit'; return []; }
-  const src = readSource(path).text;
+  const src = ctx.readText(path);
   // A copybook is read in the format in force where the COPY statement sits, which a >>SOURCE
   // directive earlier in the including file may have changed from the file's starting format.
   const fmt = detectFormat(src) === 'terminal' && ctx.copyFormat === 'auto' ? 'terminal' : (at.fmt || (ctx.copyFormat !== 'auto' ? ctx.copyFormat : (inheritedFormat || ctx.mainFormat)));
@@ -1513,6 +1524,9 @@ function indexTokens(seg) {
       const lone = (from, to) => (to - from === 1 && (seg[from].t === 'word' || seg[from].t === 'num') && /^\d+$/.test(seg[from].v) ? Number(seg[from].v) : null);
       const refLength = colonAt < 0 ? null : lone(colonAt + 1, close);
       const refStart = colonAt < 0 ? null : lone(k + 1, colonAt);
+      // A length is judged against where its reference starts: a name, a name moved by a constant, or neither.
+      const named = (from, to) => (seg[from].t === 'word' && !/^\d+$/.test(seg[from].v) && (to - from === 1 || (to - from === 3 && offsetOf(seg, from, k, close, colonAt))) ? { tok: seg[from], offset: offsetOf(seg, from, k, close, colonAt) } : null);
+      const refFrom = colonAt < 0 || refStart != null ? null : named(k + 1, colonAt) || { tok: null, offset: 0 };
       for (let j = k + 1; j < close; j++) {
         const t = seg[j];
         // A number is tokenized as a word, and no name is all digits.
@@ -1521,7 +1535,7 @@ function indexTokens(seg) {
         if (prev && prev.t === 'word' && (prev.u === 'OF' || prev.u === 'IN' || prev.u === 'FUNCTION')) continue;
         const kind = colonAt < 0 ? 'subscript' : j < colonAt ? 'refmod-offset' : 'refmod-length';
         const other = kind === 'refmod-offset' ? refLength : kind === 'refmod-length' ? refStart : null;
-        out.push({ host, tok: t, kind, offset: offsetOf(seg, j, k, close, colonAt), ...(other ? { span: other } : {}) });
+        out.push({ host, tok: t, kind, offset: offsetOf(seg, j, k, close, colonAt), ...(other ? { span: other } : {}), ...(kind === 'refmod-length' && refFrom ? { from: refFrom } : {}) });
       }
     }
     lastHost = host;
@@ -1848,7 +1862,7 @@ function collectCopybookDefines(src, format, ctx, depth) {
     const path = resolveCopy(name, null, ctx);
     if (!path || seen.has(path)) continue;
     seen.add(path);
-    const text = readSource(path).text;
+    const text = ctx.readText(path);
     const fmt = detectFormat(text) === 'terminal' && ctx.copyFormat === 'auto' ? 'terminal' : current;
     normalize(text, fmt, ctx.defines, ctx.std);
     collectCopybookDefines(text, fmt, ctx, depth + 1);
@@ -1880,7 +1894,9 @@ export function parseSource(src, file, opts = {}) {
   const scheme = BINARY_SIZE[opts.std || 'default'] || BINARY_SIZE.default;
   const ctx = { mainDir: opts.mainDir || dirname(file), includeDirs: opts.includeDirs || [], systemDirs: opts.systemDirs || [],
     fileIndex: opts.fileIndex || null, allowAbsoluteCopy: !!opts.allowAbsoluteCopy, cache: new Map(), copies: [], diags: [], copyFormat: opts.copyFormat || format, defines: new Map(), std: opts.std,
-    inclusions: 0, copyTokens: 0, replacedGrowth: 0, replacedChars: 0 };
+    inclusions: 0, copyTokens: 0, replacedGrowth: 0, replacedChars: 0,
+    // Where a copybook's text comes from: the disk, or the source tree the program was read from.
+    readText: opts.readText || ((p) => readSource(p).text) };
   collectCopybookDefines(src, format, ctx, 0);
   const norm = normalize(src, format, ctx.defines, opts.std);
   ctx.mainFormat = norm.finalFormat;

package/lib/revision.json

@@ -1 +1 @@
-{"commit":"54c9654b8c13bc22c8d5683b02551362d95f0d0a","tag":"v0.2.140"}
+{"commit":"b242a380e1b8b7d8bb8ff2d6e03e2c8a30242967","tag":"v0.3.0"}