@portll/cobolwork 0.2.76 → 0.2.117

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +66 -14
  2. package/bin/cobolwork.mjs +126 -9
  3. package/lib/arith.mjs +235 -0
  4. package/lib/baseline.mjs +5 -0
  5. package/lib/build.mjs +112 -30
  6. package/lib/capabilities.mjs +12 -7
  7. package/lib/consequence.mjs +29 -0
  8. package/lib/control.mjs +1052 -91
  9. package/lib/csd.mjs +17 -2
  10. package/lib/dataflow.mjs +245 -78
  11. package/lib/enterprise-options.mjs +255 -0
  12. package/lib/equivalence.mjs +124 -0
  13. package/lib/evidence/cli.mjs +65 -0
  14. package/lib/evidence/journal.mjs +85 -0
  15. package/lib/evidence/record.mjs +110 -0
  16. package/lib/evidence/run.mjs +111 -0
  17. package/lib/evidence/seal.mjs +177 -0
  18. package/lib/evidence/slsa.mjs +64 -0
  19. package/lib/evidence/sshsig.mjs +172 -0
  20. package/lib/evidence/store.mjs +170 -0
  21. package/lib/evidence/verify.mjs +167 -0
  22. package/lib/explain.mjs +4 -1
  23. package/lib/exploitability.mjs +263 -0
  24. package/lib/ironwork.mjs +111 -0
  25. package/lib/kernel/findings.mjs +11 -0
  26. package/lib/kernel/registry.mjs +4 -0
  27. package/lib/option-diff.mjs +49 -0
  28. package/lib/options.mjs +109 -46
  29. package/lib/parser.mjs +29 -7
  30. package/lib/policy.mjs +8 -3
  31. package/lib/precompile.mjs +4 -0
  32. package/lib/reach.mjs +29 -12
  33. package/lib/revision.json +1 -1
  34. package/lib/sarif.mjs +1 -0
  35. package/lib/sbom.mjs +147 -0
  36. package/lib/scan.mjs +93 -27
  37. package/lib/sets/cics.mjs +114 -0
  38. package/lib/sets/flow.mjs +82 -2
  39. package/lib/sets/jcl.mjs +67 -0
  40. package/lib/sets/semantics.mjs +377 -0
  41. package/lib/sets/web.mjs +34 -10
  42. package/lib/sets/zowe.mjs +220 -0
  43. package/lib/sources.mjs +10 -0
  44. package/lib/tui/app.mjs +31 -13
  45. package/lib/tui/model.mjs +17 -2
  46. package/lib/utilities.mjs +85 -13
  47. package/lib/verify.mjs +128 -0
  48. package/package.json +1 -1
  49. package/rules/compliance-dora.json +507 -0
  50. package/rules/compliance-ffiec.json +507 -0
  51. package/rules/compliance-nist80053.json +507 -0
  52. package/schema/cobolwork.policy.schema.json +2 -1
package/lib/sets/flow.mjs CHANGED
@@ -3,6 +3,9 @@ import { analyze } from '../dataflow.mjs';
3
3
  import { FLOW_MODEL, SCHEMA_VERSION, TOOL_VERSION } from '../version.mjs';
4
4
  import { finish, withMeta, sortFindings } from '../kernel/findings.mjs';
5
5
 
6
+ // Other uses of the same index listed on a finding; the count of the rest is alsoUses.
7
+ const ALSO_LISTED = 25;
8
+
6
9
  export const RULES = {
7
10
  'argv-or-env-to-os-command': { sev: 'crit', evidence: 'path', cwe: 'CWE-78', text: 'Command-line or environment input reaches an operating-system command routine' },
8
11
  'cics-terminal-to-os-command': { sev: 'crit', evidence: 'path', cwe: 'CWE-78', text: 'Terminal input reaches an operating-system command routine' },
@@ -108,10 +111,42 @@ export const RULES = {
108
111
  // misleads whoever reads the log rather than changing what the program does.
109
112
  'cics-terminal-to-log': { sev: 'low', evidence: 'path', cwe: 'CWE-117', text: 'Terminal input is written to a log unchecked, where control characters in it can start a line the program did not write' },
110
113
  'cics-web-to-log': { sev: 'low', evidence: 'path', cwe: 'CWE-117', text: 'Web request data is written to a log unchecked, where control characters in it can start a line the program did not write' },
114
+ // Storage acquired with a length the caller chose: in a CICS region the storage is shared by every
115
+ // task, where a job's own PARM or command line exhausts only its own address space.
116
+ 'cics-terminal-to-storage-length': { sev: 'med', evidence: 'path', cwe: 'CWE-770', text: 'Terminal input decides how much storage the program acquires' },
117
+ 'cics-web-to-storage-length': { sev: 'med', evidence: 'path', cwe: 'CWE-770', text: 'Web input decides how much storage the program acquires' },
118
+ 'argv-or-env-to-storage-length': { sev: 'low', evidence: 'path', cwe: 'CWE-770', text: 'Command-line or environment input decides how much storage the program acquires' },
119
+ 'jcl-parm-to-storage-length': { sev: 'low', evidence: 'path', cwe: 'CWE-770', text: 'A JCL PARM decides how much storage the program acquires' },
120
+ 'jcl-instream-to-storage-length': { sev: 'low', evidence: 'path', cwe: 'CWE-770', text: 'In-stream job data decides how much storage the program acquires' },
121
+ // The database or queue manager a program connects to, chosen by its caller: CVE-2024-52899 is the
122
+ // JDBC shape of this in an IBM product. A job's own PARM reaches only what its submitter could.
123
+ 'cics-terminal-to-connection-target': { sev: 'med', evidence: 'path', cwe: 'CWE-99', text: 'Terminal input names the database or queue manager the program connects to' },
124
+ 'cics-web-to-connection-target': { sev: 'med', evidence: 'path', cwe: 'CWE-99', text: 'Web input names the database or queue manager the program connects to' },
125
+ 'argv-or-env-to-connection-target': { sev: 'low', evidence: 'path', cwe: 'CWE-99', text: 'Command-line or environment input names the database or queue manager the program connects to' },
126
+ 'jcl-parm-to-connection-target': { sev: 'low', evidence: 'path', cwe: 'CWE-99', text: 'A JCL PARM names the database or queue manager the program connects to' },
127
+ 'jcl-instream-to-connection-target': { sev: 'low', evidence: 'path', cwe: 'CWE-99', text: 'In-stream job data names the database or queue manager the program connects to' },
128
+ // Routing by input. A SYSID ships the request to another region, which applies its own link
129
+ // security; an EXEC CICS SET changes a region's resources with system-programming authority.
130
+ 'cics-terminal-to-cics-sysid': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'Terminal input decides which region a CICS command is shipped to' },
131
+ 'cics-web-to-cics-sysid': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'Web input decides which region a CICS command is shipped to' },
132
+ 'file-record-to-cics-sysid': { sev: 'low', evidence: 'path', cwe: 'CWE-15', text: 'A file record decides which region a CICS command is shipped to' },
133
+ 'database-to-cics-sysid': { sev: 'low', evidence: 'path', cwe: 'CWE-15', text: 'A database value decides which region a CICS command is shipped to' },
134
+ 'cics-terminal-to-cics-system-resource': { sev: 'high', evidence: 'path', cwe: 'CWE-15', text: 'Terminal input names the resource an EXEC CICS SET changes' },
135
+ 'cics-web-to-cics-system-resource': { sev: 'high', evidence: 'path', cwe: 'CWE-15', text: 'Web input names the resource an EXEC CICS SET changes' },
136
+ 'file-record-to-cics-system-resource': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'A file record names the resource an EXEC CICS SET changes' },
137
+ 'database-to-cics-system-resource': { sev: 'med', evidence: 'path', cwe: 'CWE-15', text: 'A database value names the resource an EXEC CICS SET changes' },
138
+ // A document from outside parsed as XML. Whether a DTD in it is honoured - an external entity
139
+ // fetched, an internal one expanded - is the parser's, chosen by the XMLPARSE compiler option, and
140
+ // no public repository parses XML from outside, so it is low until graded against the compiler.
141
+ 'cics-web-to-xml-document': { sev: 'low', evidence: 'path', cwe: 'CWE-611', text: 'Web input is parsed as an XML document' },
142
+ 'cics-terminal-to-xml-document': { sev: 'low', evidence: 'path', cwe: 'CWE-611', text: 'Terminal input is parsed as an XML document' },
111
143
  // A response code in a reply tells the client which failure it caused and something of the system
112
144
  // behind the program, which is what an attacker probing it wants.
113
145
  'system-response-to-web-response': { sev: 'low', evidence: 'path', cwe: 'CWE-209', text: 'A response code or SQL error the system set is sent in a web response' },
114
146
  'system-response-to-http-header': { sev: 'low', evidence: 'path', cwe: 'CWE-209', text: 'A response code or SQL error the system set is sent in a response header' },
147
+ // Db2's message text shown on a terminal names the tables, columns and constraints behind the
148
+ // program. A RESP or SQLCODE on an error line is ordinary and is not followed there.
149
+ 'system-response-to-screen': { sev: 'low', evidence: 'path', cwe: 'CWE-209', text: 'Db2\'s error message text is shown on the terminal' },
115
150
  // A field the program filled and the map protects, read back and used to choose a record: the 3270
116
151
  // hidden form field. The terminal enforces PROT and ASKIP, CICS does not (NetSPI 'Conquering CICS',
117
152
  // ways 3 and 4). No check lowers it: a key that is well formed is still someone else's key.
@@ -150,7 +185,7 @@ export const RULES = {
150
185
  // from one to note; `remedy` is the standard fix, one per sink. Both ride with every finding as
151
186
  // ruleImpact/ruleRemedy. Composed from the source's actor and the sink's guidance rather than
152
187
  // written out for each path rule.
153
- const WHO = {
188
+ export const WHO = {
154
189
  'argv-or-env': "Whoever sets the program's command line or environment",
155
190
  'cics-terminal': 'A terminal user',
156
191
  'cics-web': 'A web caller',
@@ -247,6 +282,26 @@ const SINK = {
247
282
  clause: 'updates or deletes a record chosen by a value a modified 3270 client controls',
248
283
  remedy: 'Re-derive the key from server state and authorize the change; do not trust a protected or hidden screen field as a key',
249
284
  },
285
+ 'cics-sysid': {
286
+ clause: 'chooses the region a CICS command is shipped to',
287
+ remedy: 'Ship to a region chosen from a fixed table, or name the SYSID as a literal; never take it from input or a record',
288
+ },
289
+ 'cics-system-resource': {
290
+ clause: 'chooses the program, transaction, file or queue a system-programming command enables, disables or changes',
291
+ remedy: 'Name the resource an EXEC CICS SET changes from a fixed table, and keep SP commands out of programs input reaches',
292
+ },
293
+ 'connection-target': {
294
+ clause: 'chooses the database or queue manager the program connects to, with the program\'s own credentials',
295
+ remedy: 'Connect only to a location or queue manager chosen from a fixed table, never one named by input',
296
+ },
297
+ 'xml-document': {
298
+ clause: 'supplies the document XML PARSE reads, where a DTD it carries can name entities for the parser to fetch or expand, if the parser the program is compiled with honours them',
299
+ remedy: 'Refuse a document that carries a DTD (<!DOCTYPE) before XML PARSE, and bound its length; check what the XMLPARSE option the program is compiled with does with entities',
300
+ },
301
+ 'storage-length': {
302
+ clause: 'chooses how much storage the program acquires, so a large enough value fails the request or uses up storage it shares',
303
+ remedy: 'Test the length against the most the program means to acquire before GETMAIN or CEEGTST, and refuse anything larger',
304
+ },
250
305
  };
251
306
  // Exfiltration over HTTP reads the same as the host case.
252
307
  SINK['outbound-http'] = SINK['outbound-host'];
@@ -294,7 +349,6 @@ const DEFAULT = { sev: 'med', evidence: 'path', cwe: null, text: 'Untrusted inpu
294
349
  // is safe for the sink - one of a list of literals, digits where digits are needed, a bound where an
295
350
  // index is - is not a finding at all, and is listed under `checked` with the check that stops it.
296
351
  const LOWER = { crit: 'high', high: 'med', med: 'low', low: 'info', info: 'info' };
297
-
298
352
  // One graph per repository, for the reason scanAll gives: names are only unique within one.
299
353
  function analyzeEach(root, opts) {
300
354
  if (!opts.repos || opts.repos.length < 2) return analyze(root, opts);
@@ -349,6 +403,7 @@ export function scan(root, opts = {}) {
349
403
  // how many hops are missing cannot tell a truncated trace from one that mentions hops.
350
404
  trace: f.path.map(p => ({ program: p.program, item: p.item, file: p.file, via: p.via, ...(p.dir ? { dir: p.dir } : {}), ...(p.elided ? { elided: p.elided } : {}) })),
351
405
  related: [{ path: f.source.file, line: f.source.line, detail: f.source.detail }],
406
+ ...(f.also ? { also: f.also } : {}),
352
407
  });
353
408
  }
354
409
  // One sink reached by many sources is one finding, not many: a report where a single statement
@@ -361,8 +416,33 @@ export function scan(root, opts = {}) {
361
416
  held.sources++;
362
417
  // A sink is credited only as far as the least-credited source that reaches it.
363
418
  const level = Math.min(held.level, f.level);
419
+ const also = [...(held.also || []), ...(f.also || [])];
364
420
  if (f.hops < held.hops) Object.assign(held, f, { sources: held.sources });
365
421
  held.level = level;
422
+ if (also.length) held.also = also;
423
+ }
424
+ // A use one source's route reaches unchecked, and the report leaves out for another use of the same
425
+ // index, still limits how far a finding at that use is credited.
426
+ for (const f of findings) {
427
+ for (const a of f.also || []) {
428
+ const held = a.unreached ? null : perSink.get(`${f.rule}|${a.file}:${a.line}|${a.program || ''}`);
429
+ if (held && a.level < held.level) held.level = a.level;
430
+ }
431
+ }
432
+ for (const held of perSink.values()) {
433
+ if (!held.also) continue;
434
+ const seenAt = new Set();
435
+ const left = held.also.filter((a) => {
436
+ const at = `${a.file}:${a.line}`;
437
+ if (seenAt.has(at) || perSink.has(`${held.rule}|${at}|${a.program || ''}`)) return false;
438
+ seenAt.add(at);
439
+ return true;
440
+ }).sort((a, b) => (a.file < b.file ? -1 : a.file > b.file ? 1 : a.line - b.line));
441
+ delete held.also;
442
+ if (!left.length) continue;
443
+ const how = (a) => (a.unreached ? 'no route from the entries reaches it' : a.level === 2 ? `stopped by the check at line ${a.guardLine}` : a.level === 1 ? `checked at line ${a.guardLine}, not on every route` : 'not checked');
444
+ held.related = [...held.related, ...left.slice(0, ALSO_LISTED).map((a) => ({ path: a.file, line: a.line, detail: `also ${a.detail}: ${how(a)}` }))];
445
+ held.alsoUses = left.length;
366
446
  }
367
447
  findings.length = 0;
368
448
  const checked = [];
package/lib/sets/jcl.mjs CHANGED
@@ -59,8 +59,74 @@ export const JCL_RULES = {
59
59
  impact: 'A dataset the estate calls production leaves the system to the FTP partner',
60
60
  remedy: 'Confirm the transfer and its partner are authorized for production data, and send over a TLS session',
61
61
  },
62
+ 'control-cards-from-dataset': {
63
+ sev: 'low', evidence: 'construct', cwe: 'CWE-829',
64
+ text: 'A utility step reads the commands it runs from a cataloged data set, so what it runs is not in the reviewed JCL',
65
+ impact: 'Whoever can update that data set decides what the step runs the next time the job does, with the job\'s authority',
66
+ remedy: 'Keep the commands in-stream where they are reviewed with the job, or put the data set under the same change control and write access as the job',
67
+ },
68
+ 'sort-exit-named': {
69
+ sev: 'med', evidence: 'construct', cwe: 'CWE-829',
70
+ text: 'A DFSORT or ICETOOL step names an exit routine on a MODS statement, a load module the sort loads and runs',
71
+ impact: 'The exit runs inside the sort with the step\'s authority and sees every record; whoever can replace that load module runs code in the job',
72
+ remedy: 'Confirm the exit\'s load library is under change control with restricted write access, or remove the exit',
73
+ },
74
+ 'tso-batch-runs-program': {
75
+ sev: 'low', evidence: 'construct', cwe: 'CWE-78',
76
+ text: 'A TSO batch step runs another program, CLIST, REXX exec or job from its in-stream commands',
77
+ impact: 'What the step runs is decided by the commands, not the PGM= the job names, and runs with the job\'s TSO authority',
78
+ remedy: 'Review what the commands run; name a program with PGM= where the step exists to run one',
79
+ },
62
80
  };
63
81
 
82
+ const TSO = new Set(['IKJEFT01', 'IKJEFT1A', 'IKJEFT1B']);
83
+ const SORTS = new Set(['SORT', 'ICEMAN', 'SYNCSORT']);
84
+ const CONTROL_DDS = [
85
+ [TSO, ['SYSTSIN']], [new Set(['BPXBATCH']), ['STDPARM']], [new Set(['IDCAMS']), ['SYSIN']],
86
+ [SORTS, ['SYSIN', 'DFSPARM']], [new Set(['ICETOOL']), ['TOOLIN', 'DFSPARM']],
87
+ [new Set(['DSNTEP2', 'DSNTEP4', 'DSNTIAD']), ['SYSIN']],
88
+ ];
89
+ const TSO_RUNS = /^\s*(CALL|EXEC|EX|SUBMIT|ISPSTART|RUN)\b/i;
90
+ // RUN PROGRAM(DSNTEP2) under the DSN command processor makes that step's SYSIN a program's SQL.
91
+ const RUNS_SQL_PROGRAM = /\bRUN\s+PROGRAM\s*\(\s*(DSNTEP2|DSNTEP4|DSNTIAD)\s*\)/i;
92
+
93
+ // Utilities whose commands come from a DD: a cataloged data set there is commands nobody reviewed
94
+ // with the job; a MODS exit is a load module the sort runs; TSO commands say what else runs.
95
+ function controlCardFindings(r, path, findings) {
96
+ for (const step of r.steps) {
97
+ const pgm = step.pgm ? step.pgm.toUpperCase() : null;
98
+ if (!pgm) continue;
99
+ const groups = ddGroups(step);
100
+ const cards = (name) => (groups.get(name) || []).flatMap((dd) => dd.inStream || []);
101
+ const ddNames = new Set(CONTROL_DDS.filter(([pgms]) => pgms.has(pgm)).flatMap(([, dds]) => dds));
102
+ if (TSO.has(pgm) && cards('SYSTSIN').some((l) => RUNS_SQL_PROGRAM.test(l.text))) ddNames.add('SYSIN');
103
+ for (const name of ddNames) {
104
+ for (const dd of groups.get(name) || []) {
105
+ if (dd.dsn && !dd.inStream && !dd.temporary) {
106
+ findings.push({ rule: 'control-cards-from-dataset', path, line: dd.line, step: step.name,
107
+ detail: `step ${step.name || '(unnamed)'} runs ${pgm} with its ${name} commands in ${dd.dsn}, which is not part of this job` });
108
+ }
109
+ }
110
+ }
111
+ if (SORTS.has(pgm) || pgm === 'ICETOOL') {
112
+ for (const l of [...cards('SYSIN'), ...cards('DFSPARM')]) {
113
+ const mods = /^\s*MODS\s+(.*)$/i.exec(l.text);
114
+ if (!mods) continue;
115
+ for (const ex of mods[1].matchAll(/\b(E\d\d)\s*=\s*\(\s*([A-Z$#@][A-Z0-9$#@]{0,7})/gi)) {
116
+ findings.push({ rule: 'sort-exit-named', path, line: l.line, step: step.name, detail: `step ${step.name || '(unnamed)'} names ${ex[1].toUpperCase()} exit ${ex[2].toUpperCase()} on a MODS statement` });
117
+ }
118
+ }
119
+ }
120
+ if (TSO.has(pgm)) {
121
+ const runs = cards('SYSTSIN').map((l) => TSO_RUNS.exec(l.text)).filter(Boolean).map((m) => m[1].toUpperCase());
122
+ if (runs.length) {
123
+ findings.push({ rule: 'tso-batch-runs-program', path, line: step.line, step: step.name,
124
+ detail: `step ${step.name || '(unnamed)'} runs ${runs.length} TSO command(s) that start other work: ${[...new Set(runs)].join(', ')}` });
125
+ }
126
+ }
127
+ }
128
+ }
129
+
64
130
  // IBM's batch FTP client takes the host and its options on PARM, or the host as the first line of
65
131
  // its input, reads subcommands from INPUT, and logs on from NETRC or from its input. CardDemo's job
66
132
  // gives the same lines on SYSIN, which is read here when there is no INPUT, though the page below
@@ -414,6 +480,7 @@ export function scanJcl(root, opts = {}) {
414
480
  }
415
481
  }
416
482
  }
483
+ controlCardFindings(r, path, findings);
417
484
  return src.length;
418
485
  }, { label: 'jcl', maxBytes: opts.maxSourceBytes ?? Infinity });
419
486
 
@@ -0,0 +1,377 @@
1
+ // SPDX-License-Identifier: AGPL-3.0-or-later
2
+ // What Enterprise COBOL's generated code does that the source does not say, read from ironwork's
3
+ // model of the compiler: a binary store under TRUNC(OPT) that can exceed its PICTURE, an intermediate
4
+ // result wider than ARITH lets the compiler carry, and a range of characters whose order EBCDIC and
5
+ // ASCII disagree on. The first two rest on entries in ironwork's assumption register
6
+ // (crates/numeric/src/assumptions.rs), C2 and C1, which no Enterprise COBOL compile has settled yet,
7
+ // and each finding says so. cobolwork links nothing of ironwork; the rules are its model written down.
8
+ import { basename, extname } from 'node:path';
9
+ import { inScope, isProgram, isJcl, relPath, ebcdicByte } from '../sources.mjs';
10
+ import { report } from '../kernel/ruleset.mjs';
11
+ import { treeFor, noteUnread, noteUnparsed } from '../kernel/source-tree.mjs';
12
+ import { eachWithinMemory } from '../kernel/memory.mjs';
13
+ import { loadSite } from '../site.mjs';
14
+ import { parseJcl } from '../jcl.mjs';
15
+ import { optionCards, optionTokens, compileStepOptions } from '../options.mjs';
16
+
17
+ export const SEMANTICS_RULES = {
18
+ 'binary-store-exceeds-picture-under-trunc-opt': {
19
+ sev: 'low', evidence: 'construct', cwe: 'CWE-758',
20
+ text: 'Under TRUNC(OPT) a binary field receives a value that can have more digits than its PICTURE',
21
+ impact: 'TRUNC(OPT) lets the compiler assume every value fits the PICTURE, so when one does not, whether the field keeps the excess digits or loses them depends on the code generated for that statement and can change with the compiler level or its optimisation',
22
+ remedy: 'Make the receiver COMP-5 or wide enough for the value, add ON SIZE ERROR, or compile the program with TRUNC(STD) or TRUNC(BIN)',
23
+ },
24
+ 'intermediate-result-loses-high-order-digits': {
25
+ sev: 'low', evidence: 'construct', cwe: 'CWE-197',
26
+ text: 'A COMPUTE has an intermediate result wider than the compiler carries, so high-order digits can be lost',
27
+ impact: 'In ironwork\'s model of Enterprise COBOL a fixed-point intermediate carries at most 30 digits (31 under ARITH(EXTEND)) and, beyond that, keeps the decimal places the statement needs and drops high-order integer digits without raising SIZE ERROR, so a large enough value computes a wrong result silently',
28
+ remedy: 'Split the expression so each step fits, carry fewer decimal places in the operands and receivers, or compile with ARITH(EXTEND) where one more digit is enough',
29
+ },
30
+ 'character-range-reverses-in-ascii': {
31
+ sev: 'low', evidence: 'construct', cwe: 'CWE-474',
32
+ text: 'A range of characters is in order in EBCDIC and reversed in ASCII, or the other way round, so it holds values in one and none in the other',
33
+ impact: 'The same source tests a different set of characters when it is compiled with an ASCII collating sequence - GnuCOBOL, Micro Focus, a migration target - so a test on one platform passes or fails for a reason the source does not show',
34
+ remedy: 'Test the class the range means (IS NUMERIC, IS ALPHABETIC, a CLASS condition) or list the values, rather than relying on the order of the code page',
35
+ },
36
+ };
37
+
38
+ const MAX_SHOWN = 3;
39
+ const memberName = (file) => basename(file, extname(file)).toUpperCase();
40
+
41
+ // The last setting of an option family across the estate's defaults, the JCL step that compiles the
42
+ // member and its own CBL and PROCESS cards, with where it was set. null where no level sets it.
43
+ function lastSetting(names, { site, steps, cards }) {
44
+ const settings = [
45
+ ...site.flatMap((s) => optionTokens(s).map((token) => ({ token, where: 'compilerOptions in cobolwork.site.json' }))),
46
+ ...steps.flatMap((s) => s.options.map((token) => ({ token, where: `the compile step at ${s.file}:${s.line}` }))),
47
+ ...cards.flatMap((c) => c.options.map((token) => ({ token, where: `the ${c.level} statement at line ${c.line}` }))),
48
+ ];
49
+ for (let k = settings.length - 1; k >= 0; k--) {
50
+ const m = /^([A-Z0-9-]+)(?:\((.*)\))?$/.exec(settings[k].token);
51
+ if (m && names.includes(m[1])) return { sub: (m[2] || '').trim(), token: settings[k].token, where: settings[k].where };
52
+ }
53
+ return null;
54
+ }
55
+
56
+ // Integer and decimal places of a numeric PICTURE, with P scaling: P before the 9s adds decimal
57
+ // places, after them integer places. null for anything that is not a fixed-point number.
58
+ export function placesOf(picture) {
59
+ if (!picture) return null;
60
+ const pic = String(picture).toUpperCase().replace(/(.)\((\d+)\)/g, (_, c, n) => c.repeat(Number(n)));
61
+ if (!/^S?[9PV]+$/.test(pic) || !pic.includes('9')) return null;
62
+ const body = pic.replace(/^S/, '');
63
+ const v = body.indexOf('V');
64
+ const [left, right] = v < 0 ? [body, ''] : [body.slice(0, v), body.slice(v + 1)];
65
+ const leadingP = /^P+/.test(left) ? left.match(/^P+/)[0].length : 0;
66
+ const nines = (s) => (s.match(/9/g) || []).length;
67
+ const ps = (s) => (s.match(/P/g) || []).length;
68
+ if (leadingP || (v >= 0 && /^P/.test(right))) return { int: 0, dec: nines(left) + nines(right) + ps(left) + ps(right) };
69
+ return { int: nines(left) + ps(left), dec: nines(right) };
70
+ }
71
+
72
+ const BINARY_USAGE = new Set(['COMP', 'COMP-4', 'BINARY', 'COMPUTATIONAL', 'COMPUTATIONAL-4']);
73
+ const FLOAT_USAGE = new Set(['COMP-1', 'COMP-2', 'COMPUTATIONAL-1', 'COMPUTATIONAL-2', 'FLOAT-SHORT', 'FLOAT-LONG']);
74
+ const numberPlaces = (text) => {
75
+ const m = /^[+-]?(\d*)(?:[.,](\d*))?$/.exec(String(text));
76
+ if (!m || !(m[1] || m[2])) return null;
77
+ return { int: m[1].replace(/^0+(?=\d)/, '').length, dec: (m[2] || '').length };
78
+ };
79
+
80
+ // IBM's places for an intermediate, as ironwork's numeric/src/precision.rs computes them.
81
+ const sum = (a, b) => ({ int: Math.max(a.int, b.int) + 1, dec: Math.max(a.dec, b.dec) });
82
+ const product = (a, b) => ({ int: a.int + b.int, dec: a.dec + b.dec });
83
+ const quotient = (a, b, dmax) => ({ int: a.int + b.dec, dec: Math.max(a.dec, dmax) });
84
+ function carried(ir, dmax, n) {
85
+ if (ir.int + ir.dec <= n) return ir;
86
+ if (ir.dec <= dmax) return { int: Math.max(0, n - ir.dec), dec: ir.dec };
87
+ if (ir.int + dmax <= n) return { int: ir.int, dec: n - ir.int };
88
+ return { int: Math.max(0, n - dmax), dec: dmax };
89
+ }
90
+
91
+ // A COMPUTE's expression as operands and operators: an identifier (with its subscript skipped), a
92
+ // numeric literal, parentheses, unary and binary + - * /. Exponentiation and functions are left
93
+ // unread: IBM computes them in floating point or by rules of their own.
94
+ function expression(toks) {
95
+ const out = [];
96
+ for (let k = 0; k < toks.length; k++) {
97
+ const t = toks[k];
98
+ if (t.t === 'op' && ['+', '-', '*', '/'].includes(t.v)) { out.push({ op: t.v }); continue; }
99
+ if (t.t === 'op' && t.v === '**') return null;
100
+ if (t.t === 'sep' && (t.v === '(' || t.v === ')')) { out.push({ paren: t.v }); continue; }
101
+ if (t.t === 'num' || (t.t === 'word' && /^[+-]?\d+([.,]\d+)?$/.test(t.v))) { out.push({ literal: String(t.v) }); continue; }
102
+ if (t.t === 'word' && t.u === 'FUNCTION') return null;
103
+ if (t.t === 'word') {
104
+ const name = [t.u];
105
+ while (toks[k + 1] && toks[k + 1].t === 'word' && (toks[k + 1].u === 'OF' || toks[k + 1].u === 'IN') && toks[k + 2]) { name.push(toks[k + 2].u); k += 2; }
106
+ out.push({ name: name[0], qualified: name.length > 1 });
107
+ if (toks[k + 1] && toks[k + 1].t === 'sep' && toks[k + 1].v === '(') {
108
+ let depth = 0;
109
+ for (k++; k < toks.length; k++) {
110
+ if (toks[k].t === 'sep' && toks[k].v === '(') depth++;
111
+ if (toks[k].t === 'sep' && toks[k].v === ')' && --depth === 0) break;
112
+ }
113
+ }
114
+ continue;
115
+ }
116
+ return null;
117
+ }
118
+ return out;
119
+ }
120
+
121
+ // Walks the expression by precedence, carrying each intermediate as IBM would, and returns the first
122
+ // intermediate whose integer places the carry cut, or null. `lookup` gives an operand's places.
123
+ function firstLoss(items, { lookup, dmax, n }) {
124
+ let at = 0;
125
+ let loss = null;
126
+ const combine = (a, op, b) => {
127
+ const ir = op === '+' || op === '-' ? sum(a, b) : op === '*' ? product(a, b) : quotient(a, b, dmax);
128
+ const kept = carried(ir, dmax, n);
129
+ if (!loss && kept.int < ir.int) loss = { ir, kept, op };
130
+ return kept;
131
+ };
132
+ const primary = () => {
133
+ const x = items[at++];
134
+ if (!x) throw new Error('end');
135
+ if (x.op === '+' || x.op === '-') return primary();
136
+ if (x.paren === '(') { const v = additive(); if (!items[at] || items[at].paren !== ')') throw new Error('paren'); at++; return v; }
137
+ const p = x.literal !== undefined ? numberPlaces(x.literal) : lookup(x);
138
+ if (!p) throw new Error('operand');
139
+ return p;
140
+ };
141
+ const multiplicative = () => {
142
+ let v = primary();
143
+ while (items[at] && (items[at].op === '*' || items[at].op === '/')) { const op = items[at++].op; v = combine(v, op, primary()); }
144
+ return v;
145
+ };
146
+ const additive = () => {
147
+ let v = multiplicative();
148
+ while (items[at] && (items[at].op === '+' || items[at].op === '-')) { const op = items[at++].op; v = combine(v, op, multiplicative()); }
149
+ return v;
150
+ };
151
+ try { additive(); } catch { return { unread: true }; }
152
+ return at === items.length ? loss : { unread: true };
153
+ }
154
+
155
+ // The integer places a value from `items` can have: no carry for + and -, as a counter's increment
156
+ // is not a value the program lets grow past its field; a product or quotient as IBM places it.
157
+ function naturalInt(items, lookup) {
158
+ let at = 0;
159
+ const primary = () => {
160
+ const x = items[at++];
161
+ if (!x) throw new Error('end');
162
+ if (x.op === '+' || x.op === '-') return primary();
163
+ if (x.paren === '(') { const v = additive(); at++; return v; }
164
+ const p = x.literal !== undefined ? numberPlaces(x.literal) : lookup(x);
165
+ if (!p) throw new Error('operand');
166
+ return p;
167
+ };
168
+ const multiplicative = () => {
169
+ let v = primary();
170
+ while (items[at] && (items[at].op === '*' || items[at].op === '/')) {
171
+ const op = items[at++].op;
172
+ const b = primary();
173
+ v = op === '*' ? product(v, b) : { int: v.int + b.dec, dec: v.dec };
174
+ }
175
+ return v;
176
+ };
177
+ const additive = () => {
178
+ let v = multiplicative();
179
+ while (items[at] && (items[at].op === '+' || items[at].op === '-')) { at++; const b = multiplicative(); v = { int: Math.max(v.int, b.int), dec: Math.max(v.dec, b.dec) }; }
180
+ return v;
181
+ };
182
+ try { return additive().int; } catch { return null; }
183
+ }
184
+
185
+ // Two literals compared as the collating sequence would: shorter padded with spaces, by code.
186
+ function order(a, b, code) {
187
+ const len = Math.max(a.length, b.length);
188
+ for (let i = 0; i < len; i++) {
189
+ const x = code(a[i] ?? ' ');
190
+ const y = code(b[i] ?? ' ');
191
+ if (x === null || y === null) return null;
192
+ if (x !== y) return x < y ? -1 : 1;
193
+ }
194
+ return 0;
195
+ }
196
+ const asciiCode = (ch) => { const c = ch.codePointAt(0); return c >= 0x20 && c <= 0x7E ? c : null; };
197
+
198
+ // The literal pairs `lo THRU hi` in a run of VALUE or WHEN tokens.
199
+ function thruPairs(toks) {
200
+ const out = [];
201
+ for (let k = 0; k + 2 < toks.length; k++) {
202
+ const [lo, thru, hi] = [toks[k], toks[k + 1], toks[k + 2]];
203
+ if (lo.t === 'lit' && thru.t === 'word' && (thru.u === 'THRU' || thru.u === 'THROUGH') && hi.t === 'lit') out.push({ lo: String(lo.v), hi: String(hi.v), line: lo.line });
204
+ }
205
+ return out;
206
+ }
207
+ const WHEN_WORDS = new Set(['THRU', 'THROUGH', 'ALSO', 'OR', 'NOT', 'ANY', 'OTHER', 'TRUE', 'FALSE']);
208
+
209
+ export function scanSemantics(root, opts = {}) {
210
+ const tree = treeFor(root, opts);
211
+ const all = tree.list().filter(inScope(opts));
212
+ const files = all.filter(isProgram);
213
+ const findings = [];
214
+ const stats = {
215
+ filesScanned: 0, filesUnreadable: 0, filesUnparsed: 0,
216
+ truncOptPrograms: 0, truncUnknownPrograms: 0, arithExtendPrograms: 0, beyondEnterprisePrograms: 0, computesRead: 0, computesUnread: 0,
217
+ rangesRead: 0, collatingDeclaredPrograms: 0,
218
+ };
219
+ const site = loadSite(root, opts.site || null).compilerOptions || [];
220
+ const parsedJcl = [];
221
+ for (const f of all.filter(isJcl)) {
222
+ try { parsedJcl.push({ ...parseJcl(tree.text(f).text, f), file: relPath(root, f) }); } catch { /* the jcl set reports it */ }
223
+ }
224
+ const stepsOf = new Map();
225
+ for (const s of compileStepOptions(parsedJcl)) {
226
+ if (!stepsOf.has(s.member)) stepsOf.set(s.member, []);
227
+ stepsOf.get(s.member).push(s);
228
+ }
229
+
230
+ function judge(p, path, levels, collatingDeclared) {
231
+ const byName = new Map();
232
+ for (const it of p.items) {
233
+ const k = String(it.name).toUpperCase();
234
+ byName.set(k, byName.has(k) ? null : it);
235
+ }
236
+ const numeric = (x) => {
237
+ const it = byName.get(x.name);
238
+ if (!it || FLOAT_USAGE.has(String(it.effectiveUsage || '').toUpperCase())) return null;
239
+ return placesOf(it.picture);
240
+ };
241
+ const trunc = lastSetting(['TRUNC'], levels);
242
+ const arith = lastSetting(['ARITH', 'AR'], levels);
243
+ // ARITH(COMPAT) allows 18 digits in an item and ARITH(EXTEND) 31 (ironwork's
244
+ // Arith::max_picture_digits): a wider item says which one the program needs, or that it is not
245
+ // Enterprise COBOL at all.
246
+ const widest = Math.max(0, ...p.items.map((it) => { const x = placesOf(it.picture); return x ? x.int + x.dec : 0; }));
247
+ const extendByItem = !arith && widest > 18;
248
+ const n = (arith && /^(EXTEND|E)$/.test(arith.sub)) || extendByItem ? 31 : 30;
249
+ const beyondIbm = widest > 31;
250
+ if (beyondIbm) stats.beyondEnterprisePrograms++;
251
+ const truncOpt = trunc && trunc.sub === 'OPT';
252
+ if (truncOpt) stats.truncOptPrograms++;
253
+ if (!trunc) stats.truncUnknownPrograms++;
254
+ if (n === 31) stats.arithExtendPrograms++;
255
+ const toks = p.proc ? p.proc.tokens : [];
256
+ const narrowed = [];
257
+
258
+ for (const st of p.statements) {
259
+ const seg = toks.slice(st.at + 1, st.end ?? toks.length);
260
+ const sizeError = seg.some((t, k) => t.t === 'word' && t.u === 'SIZE' && seg[k + 1] && seg[k + 1].u === 'ERROR');
261
+
262
+ if (st.verb === 'COMPUTE') {
263
+ const eq = seg.findIndex((t) => (t.t === 'op' && t.v === '=') || (t.t === 'word' && (t.u === 'EQUAL' || t.u === 'EQUALS')));
264
+ if (eq < 0) continue;
265
+ let stop = seg.findIndex((t, k) => k > eq && t.t === 'word' && (t.u === 'ON' || t.u === 'NOT' || t.u === 'SIZE' || t.u === 'END-COMPUTE'));
266
+ if (stop < 0) stop = seg.length;
267
+ const items = expression(seg.slice(eq + 1, stop));
268
+ const receivers = seg.slice(0, eq).filter((t) => t.t === 'word' && t.u !== 'ROUNDED').map((t) => byName.get(t.u)).filter(Boolean);
269
+ if (!items || beyondIbm) { stats.computesUnread++; continue; }
270
+ const dmax = Math.max(0, ...receivers.map((r) => (placesOf(r.picture) || { dec: 0 }).dec),
271
+ ...items.filter((x, k) => !(items[k - 1] && items[k - 1].op === '/')).map((x) => (x.name ? (numeric(x) || { dec: 0 }).dec : x.literal ? (numberPlaces(x.literal) || { dec: 0 }).dec : 0)));
272
+ const loss = firstLoss(items, { lookup: numeric, dmax, n });
273
+ if (loss && loss.unread) { stats.computesUnread++; continue; }
274
+ stats.computesRead++;
275
+ if (loss) {
276
+ findings.push({
277
+ rule: 'intermediate-result-loses-high-order-digits', path, line: st.line, program: p.id,
278
+ detail: `${p.id}: a ${loss.op === '*' ? 'product' : loss.op === '/' ? 'quotient' : 'sum'} in this COMPUTE has ${loss.ir.int} integer and ${loss.ir.dec} decimal places; under ARITH(${n === 31 ? 'EXTEND' : 'COMPAT'}) (${arith ? `set by ${arith.where}` : extendByItem ? `which its ${widest}-digit item needs, since no level this set reads sets ARITH` : 'IBM\'s default: no level this set reads sets ARITH'}) the compiler carries ${n} digits and keeps ${loss.kept.int} integer places. This follows ironwork's intermediate table, assumption C1, which no Enterprise COBOL compile has settled yet`,
279
+ });
280
+ }
281
+ if (truncOpt && !sizeError) {
282
+ const int = naturalInt(items, numeric);
283
+ for (const r of receivers) {
284
+ const rp = placesOf(r.picture);
285
+ if (rp && BINARY_USAGE.has(String(r.effectiveUsage || '').toUpperCase()) && int !== null && int > rp.int) narrowed.push({ st, r, rp, int, via: 'COMPUTE' });
286
+ }
287
+ }
288
+ continue;
289
+ }
290
+
291
+ if (!truncOpt || sizeError) continue;
292
+ if (st.verb === 'MOVE' && !st.corresponding) {
293
+ const to = seg.findIndex((t) => t.t === 'word' && t.u === 'TO');
294
+ if (to !== 1) continue;
295
+ const src = seg[0];
296
+ const sp = src.t === 'num' || (src.t === 'word' && /^[+-]?\d/.test(src.v)) ? numberPlaces(src.v) : src.t === 'word' ? numeric({ name: src.u }) : null;
297
+ if (!sp) continue;
298
+ for (const t of seg.slice(to + 1)) {
299
+ if (t.t !== 'word') continue;
300
+ const r = byName.get(t.u);
301
+ const rp = r && placesOf(r.picture);
302
+ if (rp && BINARY_USAGE.has(String(r.effectiveUsage || '').toUpperCase()) && sp.int > rp.int) narrowed.push({ st, r, rp, int: sp.int, via: 'MOVE' });
303
+ }
304
+ continue;
305
+ }
306
+ if (['ADD', 'SUBTRACT', 'MULTIPLY', 'DIVIDE'].includes(st.verb)) {
307
+ const operands = st.sources.map((t) => numeric({ name: t.u })).filter(Boolean);
308
+ const literal = st.literals.map((l) => numberPlaces(l.v)).filter(Boolean);
309
+ const all = [...operands, ...literal];
310
+ if (!all.length) continue;
311
+ for (const t of st.targets) {
312
+ const r = byName.get(t.u);
313
+ const rp = r && placesOf(r.picture);
314
+ if (!rp || !BINARY_USAGE.has(String(r.effectiveUsage || '').toUpperCase())) continue;
315
+ const giving = seg.some((x) => x.t === 'word' && x.u === 'GIVING');
316
+ const int = st.verb === 'MULTIPLY' ? all.reduce((a, b) => a + b.int, giving ? 0 : rp.int)
317
+ : st.verb === 'DIVIDE' ? Math.max(...all.map((x) => x.int))
318
+ : Math.max(...all.map((x) => x.int));
319
+ if (int > rp.int) narrowed.push({ st, r, rp, int, via: st.verb });
320
+ }
321
+ }
322
+ }
323
+
324
+ // One finding per statement: the receivers it narrows, and where TRUNC(OPT) was set.
325
+ const byStatement = new Map();
326
+ for (const x of narrowed) {
327
+ if (!byStatement.has(x.st)) byStatement.set(x.st, []);
328
+ byStatement.get(x.st).push(x);
329
+ }
330
+ for (const [st, xs] of byStatement) {
331
+ const shown = xs.slice(0, MAX_SHOWN).map((x) => `${x.r.name} (${x.r.picture} ${String(x.r.effectiveUsage).toUpperCase()}, ${x.rp.int} integer digits) a value with up to ${x.int} integer digits`).join('; ');
332
+ findings.push({
333
+ rule: 'binary-store-exceeds-picture-under-trunc-opt', path, line: st.line, program: p.id,
334
+ detail: `${p.id}: this ${xs[0].via} gives ${shown}, under ${trunc.token} set by ${trunc.where}; what the field then holds depends on the generated code (ironwork's model keeps the binary width, assumption C2, which no Enterprise COBOL compile has settled yet)`,
335
+ });
336
+ }
337
+
338
+ if (collatingDeclared) { stats.collatingDeclaredPrograms++; return; }
339
+ const ranges = [];
340
+ for (const it of p.items) if (it.level === 88) for (const r of thruPairs(it.values || [])) ranges.push({ ...r, where: `88 ${it.name}` });
341
+ for (const st of p.statements) {
342
+ if (st.verb !== 'WHEN') continue;
343
+ const run = [];
344
+ for (let k = st.at + 1; k < toks.length; k++) {
345
+ const t = toks[k];
346
+ if (t.t === 'lit' || (t.t === 'word' && WHEN_WORDS.has(t.u))) run.push(t); else break;
347
+ }
348
+ for (const r of thruPairs(run)) ranges.push({ ...r, line: st.line, where: 'WHEN' });
349
+ }
350
+ for (const r of ranges) {
351
+ stats.rangesRead++;
352
+ const e = order(r.lo, r.hi, ebcdicByte);
353
+ const a = order(r.lo, r.hi, asciiCode);
354
+ if (e === null || a === null || e === 0 || a === 0 || e === a) continue;
355
+ findings.push({
356
+ rule: 'character-range-reverses-in-ascii', path, line: r.line, program: p.id,
357
+ detail: `${p.id}: ${r.where} '${r.lo}' THRU '${r.hi}' is ${e < 0 ? 'in order' : 'reversed'} in EBCDIC and ${a < 0 ? 'in order' : 'reversed'} in ASCII, so it is empty ${e < 0 ? 'under an ASCII collating sequence' : 'in EBCDIC, on the mainframe'}`,
358
+ });
359
+ }
360
+ }
361
+
362
+ const run = eachWithinMemory(files, (f) => {
363
+ let src;
364
+ try { src = tree.text(f).text; } catch (e) { noteUnread(stats, tree, f, e); return 0; }
365
+ let r;
366
+ try { r = tree.parse(f, src); } catch (e) { noteUnparsed(stats, tree, f, e); return src.length; }
367
+ stats.filesScanned++;
368
+ const path = relPath(root, f);
369
+ const levels = { site, steps: stepsOf.get(memberName(f)) || [], cards: optionCards(src) };
370
+ const collatingDeclared = /\bPROGRAM\s+COLLATING\s+SEQUENCE\b/i.test(src);
371
+ for (const p of r.programs) judge(p, path, levels, collatingDeclared);
372
+ r = null;
373
+ return src.length;
374
+ }, { label: 'semantics', maxBytes: opts.maxSourceBytes ?? Infinity });
375
+
376
+ return report('semantics', { rules: SEMANTICS_RULES, findings, stats, run });
377
+ }