@portll/cobolwork 0.5.0 → 0.7.0

package/lib/kernel/source-tree.mjs

@@ -36,6 +36,7 @@ import { basename, dirname, relative, resolve, sep } from 'node:path';
 import { buildFileIndex, parseSource } from '../parser.mjs';
 import { classifyUnder, decodeSource, diskClassifier, heldClassifier, kindOfBytes, looksEbcdic, readSource, relPath } from '../sources.mjs';
 import { revisionBlobs, treeOf, treePathParts } from './git.mjs';
+import { archiveKind, readArchive } from './pds-archive.mjs';
 
 // The shape every adapter answers to. Checked rather than documented, because an adapter missing a
 // method fails at the first rule set that happens to call it rather than at the boundary.
@@ -70,6 +71,7 @@ export function directoryTree(root, opts = {}) {
   const kindOf = diskClassifier();
   classifyUnder(root, kindOf);
   if (top !== root) classifyUnder(top, kindOf);
+  const parseSpec = { copyDirs: idx.copyDirs, fileIndex: idx.index, indexRoot: idx.index.root, systemDirs };
 
   return {
     kind: 'directory',
@@ -81,6 +83,7 @@ export function directoryTree(root, opts = {}) {
     list: () => [...idx.index.values()].sort(),
     bytes: (p) => readFileSync(p),
     text: (p) => readSource(p),
+    decode: decodeSource,
     rel: (p) => relPath(root, p),
     // The same prefix test buildFileIndex applies while walking. It is not the containment
     // mechanism - the walk is, and it resolves symlinks before testing - it is the answer to
@@ -88,17 +91,26 @@ export function directoryTree(root, opts = {}) {
     contains: (p) => { const r = resolve(p); return r === top || r.startsWith(top + sep); },
     // One parse configuration, in one place. `text` is optional: a caller that has already read
     // the source passes it rather than reading twice.
-    parse: (file, text) => parseSource(text ?? readSource(file).text, file, {
-      format: 'auto',
-      includeDirs: idx.copyDirs,
-      fileIndex: idx.index,
-      mainDir: dirname(file),
-      copyFormat: 'auto',
-      systemDirs,
-    }),
+    parse: (file, text) => parseInDirectory(parseSpec, file, text),
+    // The same configuration as data, for a worker thread to parse as this tree does.
+    parseSpec,
   };
 }
 
+// A directory tree's parse, from its parseSpec. A spec cloned into a worker loses the index's root,
+// which the parser reads to refuse a COPY that leaves the tree, so it is put back.
+export function parseInDirectory(spec, file, text) {
+  if (spec.fileIndex.root !== spec.indexRoot) spec.fileIndex.root = spec.indexRoot;
+  return parseSource(text ?? readSource(file).text, file, {
+    format: 'auto',
+    includeDirs: spec.copyDirs,
+    fileIndex: spec.fileIndex,
+    mainDir: dirname(file),
+    copyFormat: 'auto',
+    systemDirs: spec.systemDirs,
+  });
+}
+
 // A tree whose files are held rather than on disk. The parser finds copybooks in the index built
 // here and reads them through `readText`, so a COPY resolves to a held file or to a system copy
 // library outside the tree, never to a file on disk that happens to sit under this tree's root.
@@ -135,6 +147,7 @@ function heldTree({ kind, root, store, rel, systemDirs = [], copyDirs = null, cl
     list: () => [...store.keys()].sort(),
     bytes,
     text,
+    decode: decodeSource,
     rel,
     // Containment by construction: a path this tree does not hold is a path outside it.
     contains: (p) => store.has(p),
@@ -206,24 +219,26 @@ export function unframeRecords(buf) {
   return looksEbcdic(ebcdic) ? ebcdic : lines(0x0A);
 }
 
-// A member is read from the file the walk found. A link put in its place since, or a file that is
-// not a regular one, is refused rather than followed or waited on.
-function readMember(file) {
+// A file the walk found, read as it is or its first `limit` bytes. A link put in its place since, or
+// a file that is not a regular one, is refused rather than followed or waited on.
+function readFound(file, limit = Infinity) {
   const fd = openSync(file, constants.O_RDONLY | (constants.O_NOFOLLOW || 0) | (constants.O_NONBLOCK || 0));
   try {
     const st = fstatSync(fd);
     if (!st.isFile()) throw Object.assign(new Error(`not a regular file: ${file}`), { code: 'ENOTFILE' });
-    const buf = Buffer.alloc(st.size);
+    const size = Math.min(st.size, limit);
+    const buf = Buffer.alloc(size);
     let n = 0;
-    while (n < st.size) {
-      const r = readSync(fd, buf, n, st.size - n, n);
+    while (n < size) {
+      const r = readSync(fd, buf, n, size - n, n);
       if (!r) break;
       n += r;
     }
-    return unframeRecords(buf.subarray(0, n));
+    return buf.subarray(0, n);
   } finally { closeSync(fd); }
 }
 
+
 const QUALIFIER = /^[A-Z@#$][A-Z0-9@#$-]{0,7}$/;
 const MEMBER = /^([A-Z@#$][A-Z0-9@#$]{0,7})(?:\.[^.]*)?$/i;
 const MAX_LISTED = 8;
@@ -243,58 +258,111 @@ const MAX_LISTED = 8;
 export function pdsExportTree(dir, opts = {}) {
   const idx = buildFileIndex(dir);
   const top = realpathSync(dir);
-  const root = `${resolve(dir)}@pds`;
+  const files = [...idx.index.values()].sort().map((file) => {
+    let real;
+    try { real = realpathSync(file); } catch { real = file; }
+    return { rel: relPath(dir, file), escapes: real !== top && !real.startsWith(top + sep), read: (limit) => readFound(real, limit) };
+  });
+  return exportOf({ dir, root: `${resolve(dir)}@pds`, files, systemDirs: opts.systemDirs, symlinks: idx.symlinks,
+    unreadableDirs: idx.unreadableDirs.map((d) => relative(dir, d)) });
+}
+
+// The same export as a git revision holds it, read with plumbing into memory, as gitTree reads one.
+export function pdsExportRevision(repo, ref, opts = {}) {
+  const oid = treeOf(repo, ref);
+  const { blobs } = revisionBlobs(repo, oid, ref);
+  const files = blobs.filter((b) => treePathParts(b.path)).map((b) => ({ rel: b.path, read: (limit = Infinity) => b.bytes.subarray(0, limit) }));
+  const at = `${resolve(repo)}@${oid.slice(0, 12)}`;
+  return Object.assign(exportOf({ dir: at, root: `${at}@pds`, files, systemDirs: opts.systemDirs }), { ref, oid });
+}
+
+// An export's files, each { rel, read(limit), escapes }, held as members: a file whose directory is a
+// data set name and whose name a member name, or an XMIT file or IEBCOPY unload read into its data
+// set. `rel` is the file's path from the export's top, with forward slashes.
+function exportOf({ dir, root, files, systemDirs = [], symlinks = { followed: 0, outside: 0, broken: 0 }, unreadableDirs = [] }) {
   const origins = new Map();
   const found = new Map();
   const notMembers = [];
   const duplicates = [];
   const dataSets = new Set();
-  for (const file of [...idx.index.values()].sort()) {
-    const where = relative(dir, dirname(file)).split(sep).filter(Boolean);
-    const dsn = where.join('.').toUpperCase();
-    const member = MEMBER.exec(basename(file));
+  const archived = new Map();
+  const archives = { files: 0, members: 0, aliases: 0, notRead: [] };
+  for (const { rel, read, escapes } of files) {
+    const parts = rel.split('/');
+    const file = parts.pop();
+    const dsn = parts.join('.').toUpperCase();
+    let head = null;
+    try { head = archiveKind(read(16)); } catch { head = null; }
+    if (head) {
+      if (escapes) { notMembers.push(rel); continue; }
+      let a;
+      try { a = readArchive(read()); } catch (e) { a = { why: reasonOf(e) }; }
+      const name = a.dataSet || file.replace(/\.[^.]*$/, '').toUpperCase();
+      if (!a.why && !(name.length <= 44 && name.split('.').every((q) => QUALIFIER.test(q)))) a.why = `${name} is not a data set name`;
+      if (a.why) { archives.notRead.push(`${rel}: ${a.why}`); continue; }
+      archives.files++;
+      archives.aliases += a.aliases.length;
+      if (a.unmatched) archives.notRead.push(`${rel}: ${a.unmatched} member(s) whose blocks no directory entry names`);
+      if (a.missing) archives.notRead.push(`${rel}: no data for ${a.missing.join(', ')}`);
+      for (const m of a.members) {
+        const at = m.name === null ? resolve(root, name) : resolve(root, name, m.name);
+        const origin = m.name === null ? rel : `${rel}(${m.name})`;
+        if (found.has(at)) { duplicates.push(`${origin}: ${relPath(root, at)} is ${found.get(at)}`); continue; }
+        archived.set(at, m.bytes);
+        found.set(at, origin);
+        dataSets.add(m.name === null ? at : dirname(at));
+        archives.members++;
+      }
+      continue;
+    }
+    const member = MEMBER.exec(file);
     if (!member || dsn.length > 44 || (dsn && !dsn.split('.').every((q) => QUALIFIER.test(q)))) {
-      notMembers.push(relPath(dir, file));
+      notMembers.push(rel);
       continue;
     }
     const at = resolve(root, ...(dsn ? [dsn] : []), member[1].toUpperCase());
-    if (found.has(at)) { duplicates.push(`${relPath(dir, file)}: ${relPath(root, at)} is ${found.get(at)}`); continue; }
-    let real;
-    try { real = realpathSync(file); } catch { real = file; }
-    if (real !== top && !real.startsWith(top + sep)) { notMembers.push(relPath(dir, file)); continue; }
-    origins.set(at, real);
-    found.set(at, relPath(dir, file));
+    if (found.has(at)) { duplicates.push(`${rel}: ${relPath(root, at)} is ${found.get(at)}`); continue; }
+    if (escapes) { notMembers.push(rel); continue; }
+    origins.set(at, read);
+    found.set(at, rel);
     dataSets.add(dirname(at));
   }
   const kinds = new Map();
   const byKind = {};
   const copyDirs = new Set();
-  for (const [at, file] of origins) {
+  const bytesOf = (at) => archived.get(at) ?? unframeRecords(origins.get(at)());
+  for (const at of [...origins.keys(), ...archived.keys()]) {
     let kind = null;
-    try { kind = kindOfBytes(readMember(file)); } catch { kind = null; }
+    try { kind = kindOfBytes(bytesOf(at)); } catch { kind = null; }
     kinds.set(at, kind);
     byKind[kind || 'other'] = (byKind[kind || 'other'] || 0) + 1;
     if (kind !== 'jcl' && kind !== 'bms') copyDirs.add(dirname(at));
   }
   const store = {
-    keys: () => origins.keys(),
-    has: (p) => origins.has(p),
-    get: (p) => (origins.has(p) ? readMember(origins.get(p)) : undefined),
+    keys: () => [...origins.keys(), ...archived.keys()].values(),
+    has: (p) => origins.has(p) || archived.has(p),
+    get: (p) => (origins.has(p) || archived.has(p) ? bytesOf(p) : undefined),
   };
   const tree = heldTree({
-    kind: 'pds-export', root, store, rel: (p) => relPath(root, p), systemDirs: opts.systemDirs || [],
-    copyDirs: [...copyDirs], classify: () => (p) => kinds.get(p) ?? null, symlinks: idx.symlinks,
-    unreadableDirs: idx.unreadableDirs.map((d) => resolve(root, relative(dir, d))),
+    kind: 'pds-export', root, store, rel: (p) => relPath(root, p), systemDirs,
+    copyDirs: [...copyDirs], classify: () => (p) => kinds.get(p) ?? null, symlinks,
+    unreadableDirs: unreadableDirs.map((d) => resolve(root, d)),
   });
   return Object.assign(tree, {
     dir,
     origin: (p) => found.get(p) ?? null,
     export: {
       dataSets: dataSets.size,
-      members: origins.size,
+      members: origins.size + archived.size,
       byKind,
       ...(notMembers.length ? { notMembers: notMembers.length, notMembersListed: notMembers.slice(0, MAX_LISTED) } : {}),
       ...(duplicates.length ? { duplicates: duplicates.length, duplicatesListed: duplicates.slice(0, MAX_LISTED) } : {}),
+      ...(archives.files || archives.notRead.length ? {
+        archives: {
+          files: archives.files, members: archives.members, aliases: archives.aliases,
+          ...(archives.notRead.length ? { notRead: archives.notRead.length, notReadListed: archives.notRead.slice(0, MAX_LISTED) } : {}),
+        },
+      } : {}),
     },
   });
 }

package/lib/kernel/version-key.mjs

@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: AGPL-3.0-or-later
+// The version key and unknown keys of a document cobolwork reads. A document without the key
+// predates versioning and is read as version 0, which holds the same keys as version 1.
+import { printable } from './printable.mjs';
+
+// Why the document cannot be read, as a clause to follow its name, or null when it can.
+export function versionProblem(raw, current) {
+  const version = raw.version === undefined ? 0 : raw.version;
+  if (!Number.isInteger(version) || version < 0) return `has version ${printable(JSON.stringify(version), 40)}, which is not a whole number from 0`;
+  if (version > current) return `is version ${version}, and this cobolwork reads up to version ${current}; upgrade cobolwork`;
+  return null;
+}
+
+// A key opening with an underscore is a note for people, as diag/propose-site.mjs writes them.
+export const unknownKeys = (raw, known) => Object.keys(raw).filter((k) => !known.has(k) && !k.startsWith('_')).sort();
+
+export const ignoredKey = (where, key) => `${where}: ${printable(JSON.stringify(key), 80)} is not a key this cobolwork reads, so it was ignored`;

package/lib/layout.mjs

@@ -22,6 +22,8 @@ function picInfo(pic, constants) {
     if (ch === 'V' || ch === 'P') continue;
     if (ch === '9') { info.digits += n; info.display += n; continue; }
     if (ch === 'N' || ch === 'G') { info.national = true; info.display += 2 * n; continue; }
+    // A U is one UTF-8 character, for which IBM reserves four bytes (Defining UTF-8 data items).
+    if (ch === 'U') { info.utf8 = true; info.display += 4 * n; continue; }
     if (ch === 'X' || ch === 'A') info.alphanumeric = true;
     info.display += n;
   }
@@ -45,9 +47,13 @@ function binaryBytes(digits, scheme) {
 }
 
 function elementarySize(item, scheme, constants) {
+  if (item.byteLength) return item.byteLength;
   if (!item.picture && !item.usage) {
     const constBytes = constants && constants.textBytes;
-    const fromValue = item.values.reduce((n, v) => n + (v.t === 'lit' ? literalBytes(v) : v.t === 'word' && constBytes && constBytes.has(v.u) ? constBytes.get(v.u) : 0), 0);
+    const bytes = (v) => (v.t === 'lit' ? literalBytes(v) : v.t === 'word' && constBytes && constBytes.has(v.u) ? constBytes.get(v.u) : 0);
+    // A screen item shows a numeric VALUE as written, one position per character.
+    const digits = (v) => v.t === 'num' || (v.t === 'word' && /^\d+$/.test(v.v));
+    const fromValue = item.values.reduce((n, v) => n + (item.section === 'SCREEN' && digits(v) ? v.v.length : bytes(v)), 0);
     if (item.section === 'SCREEN' && !fromValue && item.screenRefItem) return item.screenRefItem.size || 0;
     if (item.section === 'SCREEN') return Math.max(1, fromValue);
     if (fromValue) return fromValue;
@@ -147,47 +153,36 @@ export function computeSizes(roots, scheme, constants) {
   for (const r of roots) place(r, 0);
 }
 
-// A report group is laid out by column, not by adding up its items: a line is as wide as the column
-// its rightmost item ends in, COLUMN PLUS counting on from the end of the item before. A group holds
-// its lines one after another, and every 01 group of a report, like the file the report is written
-// to, is as large as the largest group.
+// A report group is as large as the larger of its items laid end to end and the column its
+// rightmost elementary item ends in, COLUMN PLUS counting on from the end of the item before; an
+// elementary item keeps its picture's size. Every 01 group of a report, like the file the report is
+// written to, is as large as the largest group. These are GnuCOBOL's sizes: IBM's Report Writer is a
+// precompiler that writes its own records.
 export function layoutReport(rd) {
   const structural = (x) => x.children.filter(c => c.level !== 88 && c.level !== 66 && c.level !== 78);
   const mark = (x) => { x.rd = rd; for (const c of x.children) mark(c); };
   for (const g of rd.groups) mark(g);
-  let width = 0;
-  // Items printed at the same column under PRESENT WHEN each keep their own storage, so a line is
-  // never smaller than its items laid end to end.
-  const lineWidth = (line) => {
+  const sizeOf = (x) => {
+    const kids = structural(x);
+    if (!kids.length) return x.size || 0;
     let last = 0;
     let right = 0;
     let total = 0;
-    const place = (c) => {
-      const len = c.contributes || c.size || 0;
+    for (const c of kids) {
+      const len = sizeOf(c) * (c.occurs || 1);
+      total += len;
+      if (structural(c).length) continue;
       const start = c.rwColumn ? (c.rwColumn.at != null ? c.rwColumn.at : last + c.rwColumn.plus) : last + 1;
       last = start + len - 1;
       right = Math.max(right, last);
-      total += len;
-    };
-    const walk = (x) => { for (const c of structural(x)) { if (structural(c).length) walk(c); else place(c); } };
-    if (structural(line).length) walk(line);
-    else place(line);
-    return Math.max(right, total);
-  };
-  // The entries that open a line; a group with no LINE clause anywhere in it is one line.
-  for (const g of rd.groups) {
-    const lines = [];
-    const collect = (x) => { if (x.rwLine) { lines.push(x); return; } for (const c of structural(x)) collect(c); };
-    collect(g);
-    if (!lines.length) lines.push(g);
-    let total = 0;
-    for (const line of lines) {
-      const w = lineWidth(line);
-      if (line !== g) { line.size = w; line.contributes = w * (line.occurs || 1); }
-      total += w;
     }
-    width = Math.max(width, total);
-  }
+    const size = Math.max(right, total);
+    x.size = size;
+    x.contributes = size * (x.occurs || 1);
+    return size;
+  };
+  let width = 0;
+  for (const g of rd.groups) width = Math.max(width, sizeOf(g));
   for (const g of rd.groups) { g.size = width; g.contributes = width; }
   rd.width = width;
 }

package/lib/options.mjs

@@ -485,17 +485,27 @@ export function sourceOperands(args, compiler = 'gnucobol') {
   return out;
 }
 
+// The first GnuCOBOL release with the -fec option, and with each condition added after it.
+const FEC_SINCE = '3.1';
+const CONDITION_SINCE = { 'EC-PROGRAM-ARG-MISMATCH': '3.2' };
+export const olderThan = (version, since) => {
+  const [a, b] = [version, since].map((v) => String(v).split('.').map(Number));
+  for (let i = 0; i < Math.max(a.length, b.length); i++) if ((a[i] || 0) !== (b[i] || 0)) return (a[i] || 0) < (b[i] || 0);
+  return false;
+};
+
 // A cobc argument vector. Returns { checks, forbidden, add }: `add` holds the -fec options that
 // supply a required check nothing turned on, and `forbidden` the options the policy's forbid list
 // names. A check turned off is a failed check, not overridden: appending past it would hide a build
-// configured against the policy.
-export function cobcChecks(args, { required, forbid = [] }) {
+// configured against the policy. With `pinned` ({ version, where }), a check that GnuCOBOL version
+// cannot generate fails, however the command asks for it.
+export function cobcChecks(args, { required, forbid = [], pinned = null }) {
   const state = new Map();
   const turnedOff = new Map();
   const set = (name, on, arg) => {
     for (const check of required) {
       if (!covers(name, EXCEPTION_OF_CHECK[check])) continue;
-      state.set(check, on);
+      state.set(check, on ? arg : false);
       if (on) turnedOff.delete(check); else turnedOff.set(check, arg);
     }
   };
@@ -509,11 +519,23 @@ export function cobcChecks(args, { required, forbid = [] }) {
     if (value === undefined) continue;
     set(exceptionName(value), !m[1], shown);
   }
+  const tooOld = (check, by) => {
+    if (!pinned) return null;
+    const condition = EXCEPTION_OF_CHECK[check];
+    const since = CONDITION_SINCE[condition];
+    const pin = `${pinned.where} pins ${pinned.version}`;
+    if (since && olderThan(pinned.version, since)) return `${check} needs ${condition}, which is in GnuCOBOL ${since} and later; ${pin}`;
+    if (!/^-(debug|d)$/.test(by) && olderThan(pinned.version, FEC_SINCE)) return `${check} needs -fec, which is in GnuCOBOL ${FEC_SINCE} and later; ${pin}`;
+    return null;
+  };
   const add = [];
   const checks = required.map((check) => {
-    if (state.get(check) === true) return { check, ok: true, why: 'turned on by the command line' };
     if (turnedOff.has(check)) return { check, ok: false, why: `${turnedOff.get(check)} turns off the ${check} check` };
     const flag = `-fec=${EXCEPTION_OF_CHECK[check]}`;
+    const by = state.get(check) || flag;
+    const old = tooOld(check, by);
+    if (old) return { check, ok: false, why: old };
+    if (state.get(check)) return { check, ok: true, why: 'turned on by the command line' };
     if (!add.includes(flag)) add.push(flag);
     return { check, ok: true, added: true, why: `added as ${flag}` };
   });