@scientific-method/standard-checker 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/README.md +6 -0
  2. package/dist/checks/base.d.ts +7 -0
  3. package/dist/checks/base.js +72 -0
  4. package/dist/checks/claims.d.ts +4 -0
  5. package/dist/checks/claims.js +83 -0
  6. package/dist/checks/comment-addresses.d.ts +3 -0
  7. package/dist/checks/comment-addresses.js +111 -0
  8. package/dist/checks/cross-entry.d.ts +4 -0
  9. package/dist/checks/cross-entry.js +177 -0
  10. package/dist/checks/deviations.d.ts +12 -0
  11. package/dist/checks/deviations.js +101 -0
  12. package/dist/checks/entries.d.ts +7 -0
  13. package/dist/checks/entries.js +130 -0
  14. package/dist/checks/evidence-entries.d.ts +6 -0
  15. package/dist/checks/evidence-entries.js +167 -0
  16. package/dist/checks/formats.d.ts +13 -0
  17. package/dist/checks/formats.js +194 -0
  18. package/dist/checks/kaitai.d.ts +6 -0
  19. package/dist/checks/kaitai.js +103 -0
  20. package/dist/checks/parity.d.ts +30 -0
  21. package/dist/checks/parity.js +189 -0
  22. package/dist/checks/references.d.ts +4 -0
  23. package/dist/checks/references.js +37 -0
  24. package/dist/checks/rules.d.ts +4 -0
  25. package/dist/checks/rules.js +259 -0
  26. package/dist/checks/screens.d.ts +4 -0
  27. package/dist/checks/screens.js +44 -0
  28. package/dist/checks/validation.d.ts +7 -0
  29. package/dist/checks/validation.js +116 -0
  30. package/dist/code-comments.d.ts +3 -0
  31. package/dist/code-comments.js +150 -0
  32. package/dist/code-files.d.ts +11 -0
  33. package/dist/code-files.js +41 -0
  34. package/dist/context.d.ts +34 -0
  35. package/dist/context.js +2 -0
  36. package/dist/evidence.d.ts +37 -0
  37. package/dist/evidence.js +91 -0
  38. package/dist/files.d.ts +11 -0
  39. package/dist/files.js +35 -0
  40. package/dist/generate/indexes.d.ts +3 -0
  41. package/dist/generate/indexes.js +135 -0
  42. package/dist/generate/layout.d.ts +18 -0
  43. package/dist/generate/layout.js +49 -0
  44. package/dist/generate/parity-md.d.ts +7 -0
  45. package/dist/generate/parity-md.js +44 -0
  46. package/dist/generate/write.d.ts +8 -0
  47. package/dist/generate/write.js +75 -0
  48. package/dist/ids.d.ts +13 -0
  49. package/dist/ids.js +22 -0
  50. package/dist/load/builds.d.ts +9 -0
  51. package/dist/load/builds.js +124 -0
  52. package/dist/load/code-ranges.d.ts +4 -0
  53. package/dist/load/code-ranges.js +79 -0
  54. package/dist/load/entries.d.ts +7 -0
  55. package/dist/load/entries.js +57 -0
  56. package/dist/load/glossary.d.ts +12 -0
  57. package/dist/load/glossary.js +70 -0
  58. package/dist/load/readme.d.ts +3 -0
  59. package/dist/load/readme.js +36 -0
  60. package/dist/load/spec.d.ts +6 -0
  61. package/dist/load/spec.js +25 -0
  62. package/dist/locations.d.ts +19 -0
  63. package/dist/locations.js +57 -0
  64. package/dist/markdown.d.ts +27 -0
  65. package/dist/markdown.js +162 -0
  66. package/dist/options.d.ts +34 -0
  67. package/dist/options.js +87 -0
  68. package/dist/problems.d.ts +27 -0
  69. package/dist/problems.js +27 -0
  70. package/dist/standard-checker.js +51 -3033
  71. package/dist/standard.d.ts +65 -0
  72. package/dist/standard.js +173 -0
  73. package/dist/types.d.ts +91 -0
  74. package/dist/types.js +2 -0
  75. package/dist/yaml.d.ts +6 -0
  76. package/dist/yaml.js +173 -0
  77. package/package.json +1 -1
@@ -31,17 +31,41 @@
31
31
  // used. Run it only after every test in those files passed, with none
32
32
  // skipped, against the original's files
33
33
  //
34
+ // Each problem is one line that starts with the path it concerns, or spec for the spec as a whole.
35
+ // A problem that breaks a numbered rule of the standard ends with the rule's label, such as
36
+ // [STATUS-4] for the rule whose heading is anchored at #status-4.
37
+ //
34
38
  // The KSC environment variable names the Kaitai Struct compiler. Without it, the check looks for
35
39
  // kaitai-struct-compiler or ksc on PATH, and warns when it finds neither.
36
40
  //
37
41
  // No dependencies. The YAML reader understands the subset the standard's front matter uses:
38
42
  // scalars, flow lists, and block lists of flat maps.
39
- import { existsSync, readFileSync, writeFileSync, readdirSync, mkdirSync, mkdtempSync, rmSync, statSync, } from "node:fs";
40
- import { join, dirname, relative, basename, resolve, sep } from "node:path";
41
- import { tmpdir } from "node:os";
43
+ //
44
+ // This file reads the options and runs the phases in order: load the spec, check the entries, the
45
+ // rules and what crosses entries, compile the Kaitai definitions, check the deviations, parity,
46
+ // VALIDATION.md, the code's references and comments and the base ref, then write or check the
47
+ // generated files. Every phase reports into one collector, which prints the problems at the end in
48
+ // the order they were found.
49
+ import { readFileSync } from "node:fs";
50
+ import { dirname } from "node:path";
42
51
  import { fileURLToPath } from "node:url";
43
- import { execFileSync } from "node:child_process";
44
- import { createHash } from "node:crypto";
52
+ import { checkBase } from "./checks/base.js";
53
+ import { checkCommentAddresses } from "./checks/comment-addresses.js";
54
+ import { checkAcrossEntries } from "./checks/cross-entry.js";
55
+ import { checkDeviations } from "./checks/deviations.js";
56
+ import { checkEntries } from "./checks/entries.js";
57
+ import { compileKaitai } from "./checks/kaitai.js";
58
+ import { checkParity } from "./checks/parity.js";
59
+ import { checkReferences } from "./checks/references.js";
60
+ import { checkRules } from "./checks/rules.js";
61
+ import { checkValidation } from "./checks/validation.js";
62
+ import { createCodeFiles } from "./code-files.js";
63
+ import { generateIndexes } from "./generate/indexes.js";
64
+ import { generateParity } from "./generate/parity-md.js";
65
+ import { checkLineLimits, writeGenerated } from "./generate/write.js";
66
+ import { loadSpec } from "./load/spec.js";
67
+ import { parseOptions } from "./options.js";
68
+ import { createProblems } from "./problems.js";
45
69
  const selfPath = fileURLToPath(import.meta.url);
46
70
  const argv = process.argv.slice(2);
47
71
  if (argv.includes("--help") || argv.includes("-h")) {
@@ -52,3031 +76,25 @@ if (argv.includes("--help") || argv.includes("-h")) {
52
76
  .trim());
53
77
  process.exit(0);
54
78
  }
55
- const FLAGS = ["--check", "--no-ksy"];
56
- const VALUED = [
57
- "--root",
58
- "--base",
59
- "--glossary",
60
- "--code",
61
- "--references",
62
- "--images",
63
- "--max-range",
64
- "--data-dirs",
65
- "--record-validation",
66
- ];
67
- const options = { glossary: [] };
68
- for (let k = 0; k < argv.length; k++) {
69
- const arg = argv[k];
70
- if (FLAGS.includes(arg))
71
- options[arg.slice(2)] = true;
72
- else if (VALUED.includes(arg)) {
73
- if (k + 1 >= argv.length) {
74
- console.error(`${arg} needs a value; see --help`);
75
- process.exit(2);
76
- }
77
- if (arg === "--glossary")
78
- options.glossary.push(argv[++k]);
79
- else
80
- options[arg.slice(2)] = argv[++k];
81
- }
82
- else {
83
- console.error(`unknown option ${arg}; see --help`);
84
- process.exit(2);
85
- }
86
- }
87
- const dirList = (value, fallback) => value === undefined
88
- ? fallback
89
- : value
90
- .split(",")
91
- .map((x) => x.trim())
92
- .filter(Boolean);
93
- const repoDir = resolve(options.root ?? ".");
94
- const specDir = join(repoDir, "spec");
95
- const checkOnly = options.check === true;
96
- if (checkOnly && options["record-validation"] !== undefined) {
97
- console.error("--record-validation writes VALIDATION.md, so it cannot be combined with --check");
98
- process.exit(2);
99
- }
100
- const skipKsy = options["no-ksy"] === true;
101
- const baseArg = options.base ?? null;
102
- const codeRoots = dirList(options.code, ["src", "tests", "tools"]);
103
- const referenceRoots = dirList(options.references, []);
104
- // Half-open [low, high) ranges, as numbers: flat 32-bit addresses fit exactly.
105
- const images = dirList(options.images, []).map((range) => {
106
- const m = /^0x([0-9A-Fa-f]{8})\.\.0x([0-9A-Fa-f]{8})$/.exec(range);
107
- const [low, high] = m ? [parseInt(m[1], 16), parseInt(m[2], 16)] : [NaN, NaN];
108
- if (!m || high <= low) {
109
- console.error(`--images takes half-open ranges such as 0x00400000..0x004C9000, not ${range}`);
110
- process.exit(2);
111
- }
112
- return [low, high];
113
- });
114
- const maxRange = options["max-range"] === undefined ? 0x10000 : Number(options["max-range"]);
115
- if (!Number.isSafeInteger(maxRange) || maxRange < 1) {
116
- console.error(`--max-range takes a positive number of bytes, such as 0x10000, not ${options["max-range"]}`);
117
- process.exit(2);
118
- }
119
- const problems = [];
120
- let codeFilesCache; // see codeFiles()
121
- const problem = (file, message) => problems.push(`${file ? relative(repoDir, file).replaceAll("\\", "/") : "spec"}: ${message}`);
122
- // ---------------------------------------------------------------------------------------------
123
- // Kinds, statuses and sections
124
- const KINDS = {
125
- BLD: { dir: "builds", statuses: null },
126
- SRC: { dir: "sources", statuses: null },
127
- FMT: { dir: "formats", statuses: "claim" },
128
- RULE: { dir: "rules", statuses: "claim" },
129
- FND: { dir: "findings", statuses: "evidence" },
130
- EXP: { dir: "experiments", statuses: "evidence" },
131
- BUG: { dir: "bugs", statuses: "claim" },
132
- SCR: { dir: "screens", statuses: "claim" },
133
- };
134
- const CLAIM_STATUSES = ["unknown", "sourced", "supported", "established", "disputed", "superseded"];
135
- const SCALE = ["unknown", "sourced", "supported", "established"];
136
- const EVIDENCE_STATUSES = ["recorded", "reproduced", "superseded"];
137
- const ROW_STATUSES = ["unknown", "sourced", "supported", "established", "disputed"];
138
- const SECTIONS = {
139
- BLD: ["Obtaining", "Compared with other builds", "Other files", "Code ranges"],
140
- SRC: ["Use", "Known errors"],
141
- FND: ["Observation", "Interpretation", "Alternatives", "How to reproduce"],
142
- EXP: ["Question", "Setup", "Procedure", "Observations", "Results", "Conclusion"],
143
- FMT: ["Layout", "Enumerations and flags", "Differences between builds", "Coverage", "Open questions"],
144
- RULE: [
145
- "Summary",
146
- "When it runs",
147
- "Parameters",
148
- "Inputs",
149
- "Procedure",
150
- "Outputs",
151
- "Edge cases",
152
- "What the sources say",
153
- "Differences between builds",
154
- "Open questions",
155
- ],
156
- BUG: [
157
- "Symptom",
158
- "Trigger conditions",
159
- "Mechanism",
160
- "Frequency",
161
- "Player reliance",
162
- "Fixes elsewhere",
163
- "Differences between builds",
164
- "Open questions",
165
- ],
166
- SCR: [
167
- "Drawn elements",
168
- "Mouse input",
169
- "Keyboard input",
170
- "Other input",
171
- "Sounds",
172
- "States",
173
- "Timing",
174
- "Differences between builds",
175
- "Open questions",
176
- ],
177
- };
178
- const COMMON = ["id", "title", "status", "builds", "superseded_by"];
179
- const CLAIM_LINKS = ["evidence", "conflicting", "split_with", "related"];
180
- const FIELDS = {
181
- BLD: {
182
- required: [
183
- "id",
184
- "title",
185
- "superseded_by",
186
- "developer",
187
- "publisher",
188
- "publisher_version",
189
- "distribution",
190
- "languages",
191
- "int_width",
192
- "manifest",
193
- ],
194
- },
195
- SRC: { required: ["id", "title", "superseded_by", "author", "date", "location", "xxh3", "licence"] },
196
- FND: { required: [...COMMON, "recorded_by", "reproduced_by", "method", "locations", "tool", "environment"] },
197
- EXP: {
198
- required: [
199
- ...COMMON,
200
- "recorded_by",
201
- "reproduced_by",
202
- "environment",
203
- "starting_state",
204
- "recording",
205
- "repetitions",
206
- "fixture",
207
- ],
208
- },
209
- FMT: { required: [...COMMON, "files", "byte_order", "size", "text", "definition", ...CLAIM_LINKS] },
210
- RULE: { required: [...COMMON, ...CLAIM_LINKS] },
211
- BUG: { required: [...COMMON, "impact", "intent", "player_reliance", ...CLAIM_LINKS] },
212
- SCR: { required: [...COMMON, "resolution", ...CLAIM_LINKS] },
213
- };
214
- const RELATED_KINDS = {
215
- RULE: ["RULE", "FMT", "SCR"],
216
- FMT: ["RULE"],
217
- BUG: ["RULE", "FMT", "SCR"],
218
- SCR: ["RULE", "SCR"],
219
- };
220
- const SCREEN_TABLES = {
221
- "Drawn elements": ["Element", "Resource", "Shows", "Position", "Shown when", "Evidence"],
222
- "Mouse input": ["Region", "Rectangle", "Enabled when", "Effect", "Evidence"],
223
- "Keyboard input": ["Key", "Enabled when", "Effect", "Evidence"],
224
- "Other input": ["Device", "Input", "Enabled when", "Effect", "Evidence"],
225
- Sounds: ["Sound", "Resource", "Played when", "Evidence"],
226
- States: ["State", "Entered when", "Left when", "Evidence"],
227
- };
228
- const BINARY_LAYOUT = ["Offset", "Size", "Type", "Name", "Meaning", "Status", "Evidence"];
229
- const TEXT_LAYOUT = ["Key", "Type", "Name", "Meaning", "Status", "Evidence"];
230
- const ENUM_TABLE = ["Value", "Name", "Meaning", "Status", "Evidence"];
231
- const ID_RE = /\b(?:(?:FMT|RULE|FND|EXP|BUG|SCR)-[A-Z][A-Z0-9]*-\d{3,}|(?:BLD|SRC)-[A-Z][A-Z0-9.-]*[A-Z0-9])\b/g;
232
- const DEV_RE = /\bDEV-[A-Z][A-Z0-9]*-\d{3,}\b/g;
233
- // Every Markdown file the standard defines, generated or not, is at most this many lines long.
234
- const LINE_LIMIT = 1000;
235
- // A list written out in a procedure or a table definition holds at most this many values. A longer
236
- // one takes them from a value file.
237
- const LIST_LIMIT = 64;
238
- // How a location in each file format is given, per Standard v1. `address` is the notation of a
239
- // loaded address; `offset` allows a range of the shipped file. Both are judged by the unpacked
240
- // format when the file is packed. A format not listed here has no rule: the Standard must first decide and document how it
241
- // is located, and only then is it added here.
242
- const SEG = /^[0-9A-F]{4}:[0-9A-F]{4}$/;
243
- const FLAT32 = /^0x[0-9A-F]{8}$/;
244
- const LOCATIONS = {
245
- MZ: { address: SEG, offset: true }, // offset only for overlay code outside the load image
246
- COM: { address: SEG },
247
- NE: { address: SEG },
248
- PE: { address: FLAT32 },
249
- LE: { address: FLAT32 },
250
- LX: { address: FLAT32 },
251
- ELF: { address: /^0x(?:[0-9A-F]{8}|[0-9A-F]{16})$/ },
252
- data: { offset: true },
253
- cdda: { offset: true },
254
- };
255
- const FORMATS = Object.keys(LOCATIONS);
256
- const locationRule = (format) => (Object.hasOwn(LOCATIONS, format) ? LOCATIONS[format] : undefined);
257
- const unlistedFormat = (format) => `format ${format} has no location rule in Standard v1 (known: ${FORMATS.join(", ")}); the Standard must document how it is located before it is used`;
258
- // Names Windows cannot give a file, whatever the extension, so no glossary term may be one.
259
- const RESERVED_NAMES = /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
260
- const lineCount = (text) => (text === "" ? 0 : text.split("\n").length - (text.endsWith("\n") ? 1 : 0));
261
- const readText = (file) => readFileSync(file, "utf8").replace(/\r\n/g, "\n");
262
- // ---------------------------------------------------------------------------------------------
263
- // Front matter reader
264
- function parseScalar(raw) {
265
- let v = raw.trim();
266
- if (v === "")
267
- return "";
268
- if (v.startsWith('"')) {
269
- const end = v.indexOf('"', 1);
270
- return v.slice(1, end);
271
- }
272
- if (v.startsWith("'")) {
273
- const end = v.indexOf("'", 1);
274
- return v.slice(1, end);
275
- }
276
- const hash = v.search(/\s#/);
277
- if (hash >= 0)
278
- v = v.slice(0, hash).trim();
279
- if (v.startsWith("[")) {
280
- const inner = v.slice(1, v.lastIndexOf("]")).trim();
281
- if (inner === "")
282
- return [];
283
- return splitFlow(inner).map((x) => parseScalar(x));
284
- }
285
- if (v === "null" || v === "~")
286
- return null;
287
- if (v === "true")
288
- return true;
289
- if (v === "false")
290
- return false;
291
- if (/^-?\d+$/.test(v))
292
- return Number(v);
293
- if (/^-?\d+\.\d+$/.test(v))
294
- return Number(v);
295
- return v;
296
- }
297
- function splitFlow(s) {
298
- const out = [];
299
- let cur = "";
300
- let quote = null;
301
- for (const ch of s) {
302
- if (quote) {
303
- cur += ch;
304
- if (ch === quote)
305
- quote = null;
306
- }
307
- else if (ch === '"' || ch === "'") {
308
- quote = ch;
309
- cur += ch;
310
- }
311
- else if (ch === ",") {
312
- out.push(cur.trim());
313
- cur = "";
314
- }
315
- else
316
- cur += ch;
317
- }
318
- if (cur.trim() !== "")
319
- out.push(cur.trim());
320
- return out;
321
- }
322
- function stripComment(line) {
323
- let quote = null;
324
- for (let i = 0; i < line.length; i++) {
325
- const ch = line[i];
326
- if (quote) {
327
- if (ch === quote)
328
- quote = null;
329
- }
330
- else if (ch === '"' || ch === "'")
331
- quote = ch;
332
- else if (ch === "#" && (i === 0 || /\s/.test(line[i - 1])))
333
- return line.slice(0, i).replace(/\s+$/, "");
334
- }
335
- return line.replace(/\s+$/, "");
336
- }
337
- function parseYaml(text, file) {
338
- const lines = text.split(/\r?\n/).map(stripComment);
339
- const obj = {};
340
- let i = 0;
341
- while (i < lines.length) {
342
- const line = lines[i];
343
- if (line.trim() === "") {
344
- i++;
345
- continue;
346
- }
347
- const m = /^([A-Za-z_][A-Za-z0-9_]*):(.*)$/.exec(line);
348
- if (!m) {
349
- problem(file, `unreadable front matter line: ${line}`);
350
- i++;
351
- continue;
352
- }
353
- const key = m[1];
354
- const rest = m[2];
355
- if (key in obj)
356
- problem(file, `front matter repeats the field ${key}`);
357
- if (rest.trim() !== "") {
358
- obj[key] = parseScalar(rest);
359
- i++;
360
- continue;
361
- }
362
- // block list
363
- const items = [];
364
- i++;
365
- let current = null;
366
- let itemIndent = null;
367
- while (i < lines.length && (lines[i].trim() === "" || /^\s/.test(lines[i]))) {
368
- const l = lines[i];
369
- if (l.trim() === "") {
370
- i++;
371
- continue;
372
- }
373
- const dash = /^(\s*)- (.*)$/.exec(l);
374
- if (dash && (itemIndent === null || dash[1].length === itemIndent)) {
375
- itemIndent = dash[1].length;
376
- const body = dash[2];
377
- const kv = /^([A-Za-z_][A-Za-z0-9_]*):(.*)$/.exec(body);
378
- if (kv) {
379
- current = {};
380
- items.push(current);
381
- if (kv[2].trim() !== "")
382
- current[kv[1]] = parseScalar(kv[2]);
383
- else
384
- current[kv[1]] = parseNested();
385
- }
386
- else {
387
- current = null;
388
- items.push(parseScalar(body));
389
- }
390
- i++;
391
- continue;
392
- }
393
- const kv = /^\s+([A-Za-z_][A-Za-z0-9_]*):(.*)$/.exec(l);
394
- if (kv && current) {
395
- if (kv[2].trim() !== "")
396
- current[kv[1]] = parseScalar(kv[2]);
397
- else {
398
- i++;
399
- current[kv[1]] = parseNestedFrom();
400
- continue;
401
- }
402
- i++;
403
- continue;
404
- }
405
- problem(file, `unreadable front matter line: ${l}`);
406
- i++;
407
- }
408
- obj[key] = items;
409
- }
410
- return obj;
411
- // A nested map under a list item (such as `unpacked:` of a packed file).
412
- function parseNestedFrom() {
413
- const nested = {};
414
- const indent = /^(\s*)/.exec(lines[i])[1].length;
415
- while (i < lines.length &&
416
- lines[i].trim() !== "" &&
417
- /^(\s*)/.exec(lines[i])[1].length >= indent &&
418
- !/^\s*- /.test(lines[i])) {
419
- const kv = /^\s+([A-Za-z_][A-Za-z0-9_]*):(.*)$/.exec(lines[i]);
420
- if (kv)
421
- nested[kv[1]] = parseScalar(kv[2]);
422
- i++;
423
- }
424
- return nested;
425
- }
426
- function parseNested() {
427
- i++;
428
- const r = parseNestedFrom();
429
- i--;
430
- return r;
431
- }
432
- }
433
- function readEntry(file) {
434
- const text = readFileSync(file, "utf8").replace(/\r\n/g, "\n");
435
- if (!text.startsWith("---\n")) {
436
- problem(file, "has no front matter");
437
- return null;
438
- }
439
- const end = text.indexOf("\n---\n", 4);
440
- if (end < 0) {
441
- problem(file, "front matter is not closed");
442
- return null;
443
- }
444
- const meta = parseYaml(text.slice(4, end), file);
445
- const body = text.slice(end + 5);
446
- return { file, meta, body, sections: splitSections(body) };
447
- }
448
- function splitSections(body) {
449
- const sections = [];
450
- let current = null;
451
- let fence = false;
452
- for (const line of body.split("\n")) {
453
- if (/^(```|~~~)/.test(line))
454
- fence = !fence;
455
- const m = !fence && /^## (.+)$/.exec(line);
456
- if (m) {
457
- current = { title: m[1].trim(), lines: [] };
458
- sections.push(current);
459
- }
460
- else if (current)
461
- current.lines.push(line);
462
- }
463
- return sections.map((s) => ({ title: s.title, text: s.lines.join("\n") }));
464
- }
465
- function tables(text) {
466
- // Every Markdown table in a section, with the `###` heading above it.
467
- const out = [];
468
- const lines = text.split("\n");
469
- let heading = null;
470
- let fence = false;
471
- for (let i = 0; i < lines.length; i++) {
472
- if (/^(```|~~~)/.test(lines[i]))
473
- fence = !fence;
474
- if (fence)
475
- continue;
476
- const h = /^### (.+)$/.exec(lines[i]);
477
- if (h)
478
- heading = h[1].trim();
479
- if (lines[i].startsWith("|") && i + 1 < lines.length && /^\|[\s:|-]+\|\s*$/.test(lines[i + 1])) {
480
- const header = cells(lines[i]);
481
- const rows = [];
482
- let j = i + 2;
483
- while (j < lines.length && lines[j].startsWith("|")) {
484
- rows.push(cells(lines[j]));
485
- j++;
486
- }
487
- out.push({ heading, header, rows, line: i });
488
- i = j - 1;
489
- }
490
- }
491
- return out;
492
- }
493
- function cells(line) {
494
- const parts = [];
495
- let cur = "";
496
- let code = false;
497
- const s = line.trim().replace(/^\|/, "").replace(/\|$/, "");
498
- for (let k = 0; k < s.length; k++) {
499
- const ch = s[k];
500
- if (ch === "`")
501
- code = !code;
502
- if (ch === "\\" && s[k + 1] === "|") {
503
- cur += "|";
504
- k++;
505
- continue;
506
- }
507
- if (ch === "|" && !code) {
508
- parts.push(cur.trim());
509
- cur = "";
510
- continue;
511
- }
512
- cur += ch;
513
- }
514
- parts.push(cur.trim());
515
- return parts;
516
- }
517
- // A value file: CSV as RFC 4180 defines it, with a header row. Returns { header, rows }, or null
518
- // after reporting a problem.
519
- function readCsv(file) {
520
- // Spreadsheet programs start a UTF-8 CSV with a byte order mark, which would join the first header.
521
- const text = readFileSync(file, "utf8").replace(/^/, "");
522
- const records = [];
523
- let record = [];
524
- let field = "";
525
- let quoted = false;
526
- let wasQuoted = false;
527
- for (let i = 0; i < text.length; i++) {
528
- const ch = text[i];
529
- if (quoted) {
530
- if (ch === '"' && text[i + 1] === '"') {
531
- field += '"';
532
- i++;
533
- }
534
- else if (ch === '"')
535
- quoted = false;
536
- else
537
- field += ch;
538
- }
539
- else if (ch === '"' && field === "" && !wasQuoted) {
540
- quoted = true;
541
- wasQuoted = true;
542
- }
543
- else if (ch === ",") {
544
- record.push(field);
545
- field = "";
546
- wasQuoted = false;
547
- }
548
- else if (ch === "\n" || ch === "\r") {
549
- if (ch === "\r" && text[i + 1] === "\n")
550
- i++;
551
- record.push(field);
552
- records.push(record);
553
- record = [];
554
- field = "";
555
- wasQuoted = false;
556
- }
557
- else
558
- field += ch;
559
- }
560
- if (quoted) {
561
- problem(file, "has a quoted field that is never closed");
562
- return null;
563
- }
564
- if (field !== "" || record.length > 0) {
565
- record.push(field);
566
- records.push(record);
567
- }
568
- // Blank lines at the end of the file are not rows.
569
- while (records.length > 0 && records.at(-1).length === 1 && records.at(-1)[0] === "")
570
- records.pop();
571
- if (records.length === 0) {
572
- problem(file, "has no header row");
573
- return null;
574
- }
575
- const [header, ...rows] = records;
576
- for (const row of rows)
577
- if (row.length !== header.length) {
578
- problem(file, `the row ${row.join(",")} has ${row.length} fields, not ${header.length}`);
579
- return null;
580
- }
581
- return { header, rows };
582
- }
583
- const idsIn = (text) => [...new Set(String(text ?? "").match(ID_RE) ?? [])];
584
- const kindOf = (id) => id.split("-")[0];
585
- const areaOf = (id) => id.split("-")[1];
586
- // Orders IDs of one kind and area by number, so RULE-A-999 comes before RULE-A-1000. Anything else
587
- // compares by UTF-16 code unit, as Array.prototype.sort does, so the order does not depend on the
588
- // machine's locale.
589
- const idSortKey = (id) => id.replace(/^((?:FMT|RULE|FND|EXP|BUG|SCR|DEV)-[A-Z][A-Z0-9]*-)(\d+)$/, (_, head, n) => head + n.padStart(12, "0"));
590
- const compareIds = (a, b) => {
591
- const x = idSortKey(a);
592
- const y = idSortKey(b);
593
- return x < y ? -1 : x > y ? 1 : 0;
594
- };
595
- const asList = (v) => (Array.isArray(v) ? v : v === null || v === undefined || v === "" ? [] : [v]);
596
- // ---------------------------------------------------------------------------------------------
597
- // Load
598
- if (!existsSync(specDir)) {
599
- console.error(`No spec/ directory in ${repoDir}.`);
600
- process.exit(1);
601
- }
602
- const readme = existsSync(join(specDir, "README.md"))
603
- ? readFileSync(join(specDir, "README.md"), "utf8").replace(/\r\n/g, "\n")
604
- : "";
605
- if (!readme)
606
- problem(null, "spec/README.md is missing");
607
- const areas = [];
608
- {
609
- const readmeSections = splitSections(readme);
610
- const titles = readmeSections.map((s) => s.title);
611
- const expected = ["Scope", "Standard version", "Areas"];
612
- if (titles.join("|") !== expected.join("|"))
613
- problem(join(specDir, "README.md"), `sections must be ${expected.join(", ")} in that order, found ${titles.join(", ")}`);
614
- const areaSection = readmeSections.find((s) => s.title === "Areas");
615
- const areaTable = areaSection && tables(areaSection.text)[0];
616
- if (!areaTable || areaTable.header.join("|") !== "Area|Covers")
617
- problem(join(specDir, "README.md"), "the area list must be a table with the columns Area | Covers");
618
- else
619
- for (const row of areaTable.rows) {
620
- const a = row[0].replaceAll("`", "");
621
- if (!/^[A-Z][A-Z0-9]*$/.test(a))
622
- problem(join(specDir, "README.md"), `area ${a} must be upper-case letters and digits starting with a letter`);
623
- if (areas.includes(a))
624
- problem(join(specDir, "README.md"), `area ${a} is listed twice`);
625
- areas.push(a);
626
- }
627
- const version = readmeSections.find((s) => s.title === "Standard version");
628
- if (version && !/version 1 of the/.test(version.text))
629
- problem(join(specDir, "README.md"), "the Standard version section must say which version it follows (version 1)");
630
- }
631
- const entries = new Map();
632
- for (const [kind, { dir }] of Object.entries(KINDS)) {
633
- const d = join(specDir, dir);
634
- if (!existsSync(d))
635
- continue;
636
- for (const name of readdirSync(d)) {
637
- const file = join(d, name);
638
- if (statSync(file).isDirectory() || !name.endsWith(".md"))
639
- continue;
640
- const read = readEntry(file);
641
- if (!read)
642
- continue;
643
- const entry = Object.assign(read, { kind });
644
- const id = entry.meta.id;
645
- if (typeof id !== "string") {
646
- problem(file, "has no id");
647
- continue;
648
- }
649
- if (name !== `${id}.md`)
650
- problem(file, `file name must be ${id}.md`);
651
- // Later checks look the kind up in KINDS, so an entry of an unknown kind is reported and dropped.
652
- if (!KINDS[kindOf(id)]) {
653
- problem(file, `${id} is not an ID of a known kind`);
654
- continue;
655
- }
656
- if (kindOf(id) !== kind)
657
- problem(file, `a ${kindOf(id)} entry does not belong in spec/${dir}/`);
658
- if (entries.has(id))
659
- problem(file, `ID ${id} is used twice`);
660
- entries.set(id, entry);
661
- }
662
- }
663
- // Stray Markdown anywhere else in spec/ that looks like an entry.
664
- for (const dir of readdirSync(specDir)) {
665
- const d = join(specDir, dir);
666
- if (!statSync(d).isDirectory() ||
667
- Object.values(KINDS).some((k) => k.dir === dir) ||
668
- dir === "index" ||
669
- dir === "glossary")
670
- continue;
671
- problem(d, "is not a directory the standard defines");
672
- }
673
- // The glossary: one file per term in spec/glossary/, named after the term and opening with it as a
674
- // # heading. A glossary file is not an entry, so it has no front matter.
675
- const glossaryDir = join(specDir, "glossary");
676
- const glossary = new Map(); // term -> text after the heading
677
- const glossaryFiles = new Map(); // term -> file
678
- const glossaryFile = (term) => glossaryFiles.get(term) ?? glossaryDir;
679
- if (existsSync(join(specDir, "glossary.md")))
680
- problem(join(specDir, "glossary.md"), "the glossary is the directory spec/glossary/; move each ## term to spec/glossary/<term>.md with the term as its # heading");
681
- function readTerm(file, report = true) {
682
- const text = readText(file);
683
- const say = (message) => {
684
- if (report)
685
- problem(file, message);
686
- };
687
- if (text.startsWith("---\n"))
688
- say("a glossary file has no front matter");
689
- const m = /^# (.+)\n?([\s\S]*)$/.exec(text.replace(/^---\n[\s\S]*?\n---\n/, "").replace(/^\s+/, ""));
690
- if (!m) {
691
- say("opens with the term as a # heading");
692
- return null;
693
- }
694
- const term = m[1].trim().replaceAll("`", "");
695
- if (basename(file) !== `${term}.md`)
696
- say(`is named after its term, ${term}.md`);
697
- if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(term))
698
- say(`${term} is not a name the pseudocode can use`);
699
- if (RESERVED_NAMES.test(term))
700
- say(`${term} is a name Windows reserves for a device, so no file can have it`);
701
- return { term, text: m[2] };
702
- }
703
- const termFiles = (dir) => readdirSync(dir)
704
- .filter((name) => name !== ".gitkeep")
705
- .map((name) => join(dir, name));
706
- if (!existsSync(glossaryDir))
707
- problem(null, "spec/glossary/ is missing");
708
- else {
709
- const folded = new Map();
710
- for (const file of termFiles(glossaryDir)) {
711
- if (statSync(file).isDirectory() || !file.endsWith(".md")) {
712
- problem(file, "is not a glossary file; spec/glossary/ holds one <term>.md per term");
713
- continue;
714
- }
715
- const t = readTerm(file);
716
- if (!t)
717
- continue;
718
- const key = t.term.toLowerCase();
719
- if (folded.has(key))
720
- problem(file, `${t.term} differs only in case from ${folded.get(key)}`);
721
- else
722
- folded.set(key, t.term);
723
- glossary.set(t.term, t.text);
724
- glossaryFiles.set(t.term, file);
725
- }
726
- }
727
- // --glossary <path> adds the terms of a draft term file, or of a directory of them, for checking
728
- // entries before their terms are merged into spec/glossary/.
729
- for (const draft of options.glossary) {
730
- const files = statSync(draft).isDirectory() ? termFiles(draft).filter((f) => f.endsWith(".md")) : [draft];
731
- for (const file of files) {
732
- const t = readTerm(file, false);
733
- if (t && !glossary.has(t.term))
734
- glossary.set(t.term, t.text);
735
- }
736
- }
737
- // Build manifests: builds/<ID>.files.yaml holds a build's files list and nothing else.
738
- const buildFiles = new Map();
739
- for (const [id, e] of entries) {
740
- if (e.kind !== "BLD")
741
- continue;
742
- const expected = `${id}.files.yaml`;
743
- if ("files" in e.meta)
744
- problem(e.file, `the files list belongs in the manifest ${expected}, not in the entry`);
745
- if (e.meta.manifest !== expected) {
746
- problem(e.file, `manifest must be ${expected}`);
747
- continue;
748
- }
749
- const path = join(dirname(e.file), expected);
750
- if (!existsSync(path)) {
751
- problem(e.file, `manifest ${expected} does not exist`);
752
- continue;
753
- }
754
- const manifest = parseYaml(readText(path), path);
755
- for (const key of Object.keys(manifest))
756
- if (key !== "files")
757
- problem(path, `a manifest has only the key files, not ${key}`);
758
- if (!Array.isArray(manifest.files)) {
759
- problem(path, "files must be a list");
760
- continue;
761
- }
762
- // Later checks read f.path, so an item that is not a map is reported and left out.
763
- if (manifest.files.some((f) => !f || typeof f !== "object"))
764
- problem(path, "every item of files is a map of path, format, size and xxh3");
765
- const files = manifest.files.filter((f) => f && typeof f === "object");
766
- buildFiles.set(id, files);
767
- for (const f of files) {
768
- if (!f.path)
769
- problem(path, "every file has a path");
770
- if (f.format === undefined || f.format === null || f.format === "")
771
- problem(path, `${f.path}: every file has a format`);
772
- else if (!locationRule(f.format))
773
- problem(path, `${f.path}: ${unlistedFormat(f.format)}`);
774
- const unpackedFormat = f.unpacked?.format;
775
- if (unpackedFormat !== undefined &&
776
- unpackedFormat !== null &&
777
- unpackedFormat !== "" &&
778
- !locationRule(unpackedFormat))
779
- problem(path, `${f.path}: unpacked ${unlistedFormat(unpackedFormat)}`);
780
- if (!/^[0-9a-f]{32}$/.test(String(f.xxh3)))
781
- problem(path, `${f.path}: xxh3 must be 32 lower-case hex digits`);
782
- if (typeof f.size !== "number")
783
- problem(path, `${f.path}: size must be a number`);
784
- if (f.packer && !(f.unpacked && f.unpacked.size && f.unpacked.xxh3 && f.unpacked.format && f.unpacked.tool))
785
- problem(path, `${f.path}: a packed file gives the size, xxh3, format and tool of its unpacked form`);
786
- if (String(f.path).includes("\\"))
787
- problem(path, `${f.path}: paths use forward slashes`);
788
- }
789
- }
790
- if (existsSync(join(specDir, "builds")))
791
- for (const name of readdirSync(join(specDir, "builds"))) {
792
- const m = /^(.+)\.(?:other-)?files\.yaml$/.exec(name);
793
- if (m && entries.get(m[1])?.kind !== "BLD")
794
- problem(join(specDir, "builds", name), `belongs to no build entry (${m[1]})`);
795
- }
796
- // Other files: every path of the installation's listing that the manifest leaves out, each with
797
- // its reason, in the build entry's Other files section or, for a long list, in
798
- // builds/<ID>.other-files.yaml, which that section names.
799
- for (const [id, e] of entries) {
800
- if (e.kind !== "BLD")
801
- continue;
802
- const name = `${id}.other-files.yaml`;
803
- const path = join(dirname(e.file), name);
804
- const named = (e.sections.find((s) => s.title === "Other files")?.text ?? "").includes(name);
805
- if (!existsSync(path)) {
806
- if (named)
807
- problem(e.file, `Other files names ${name}, which does not exist`);
808
- continue;
809
- }
810
- if (!named)
811
- problem(e.file, `the Other files section names ${name}, which lists the paths the manifest leaves out`);
812
- const list = parseYaml(readText(path), path);
813
- for (const key of Object.keys(list))
814
- if (key !== "other_files")
815
- problem(path, `a list of other files has only the key other_files, not ${key}`);
816
- if (!Array.isArray(list.other_files)) {
817
- problem(path, "other_files must be a list");
818
- continue;
819
- }
820
- // The front matter reader turns a bare name such as 1990 into a number, so paths compare as text.
821
- const inManifest = new Set((buildFiles.get(id) ?? []).map((f) => String(f.path)));
822
- const seen = new Set();
823
- for (const item of list.other_files) {
824
- if (!item || typeof item !== "object" || Object.keys(item).sort().join(",") !== "path,reason") {
825
- problem(path, "every item of other_files is a map of path and reason");
826
- continue;
827
- }
828
- if (item.path === null || item.path === undefined || item.path === "") {
829
- problem(path, "every other file has a path");
830
- continue;
831
- }
832
- const other = String(item.path);
833
- if (item.reason === null || String(item.reason).trim() === "")
834
- problem(path, `${other}: every other file gives the reason the manifest leaves it out`);
835
- if (other.includes("\\"))
836
- problem(path, `${other}: paths use forward slashes`);
837
- if (inManifest.has(other))
838
- problem(path, `${other} is in the manifest, so it is not one of the other files`);
839
- if (seen.has(other))
840
- problem(path, `${other} is listed twice`);
841
- seen.add(other);
842
- }
843
- }
844
- // Code ranges: the half-open ranges of each file that hold code located by offset, each with the
845
- // finding that shows it. A table File | Range | Overlay | Finding, or None.
846
- const CODE_RANGES = ["File", "Range", "Overlay", "Finding"];
847
- const codeRanges = new Map(); // build ID -> [{ file, start, end }]
848
- const unticked = (cell) => cell.replace(/^`(.*)`$/, "$1").trim();
849
- for (const [id, e] of entries) {
850
- if (e.kind !== "BLD")
851
- continue;
852
- const section = e.sections.find((s) => s.title === "Code ranges");
853
- // A missing section is reported with the other sections.
854
- if (!section)
855
- continue;
856
- const ranges = [];
857
- if (/^\s*None\.\s*$/.test(section.text)) {
858
- codeRanges.set(id, ranges);
859
- continue;
860
- }
861
- const found = tables(section.text);
862
- // A malformed section is reported once here; offsets into the build are then not measured
863
- // against it, as with a missing section, rather than each failing again.
864
- if (found.length !== 1 || found[0].header.join("|") !== CODE_RANGES.join("|") || found[0].rows.length === 0) {
865
- problem(e.file, `the Code ranges section is one table with the columns ${CODE_RANGES.join(" | ")}, or None.`);
866
- continue;
867
- }
868
- codeRanges.set(id, ranges);
869
- const files = buildFiles.get(id) ?? [];
870
- for (const row of found[0].rows) {
871
- if (row.length !== CODE_RANGES.length) {
872
- problem(e.file, `Code ranges row ${row.join(" | ")}: a row has ${CODE_RANGES.length} cells, not ${row.length}`);
873
- continue;
874
- }
875
- const [path, range, overlay, finding] = row.map(unticked);
876
- const at = `Code ranges row ${path} ${range}`;
877
- const bf = files.find((f) => f.path === path);
878
- if (!bf) {
879
- problem(e.file, `${at}: ${path} is not in the manifest`);
880
- continue;
881
- }
882
- // Only overlay code is located by offset, so a row is for a file whose unpacked format takes
883
- // both addresses and offsets (MZ). An unlisted format is already reported against its manifest.
884
- const format = bf.unpacked?.format ?? bf.format;
885
- const rule = locationRule(format);
886
- if (rule && !(rule.offset && rule.address))
887
- problem(e.file, `${at}: ${path} is a ${format} file, which holds no code located by offset`);
888
- // The notation is parseOffset's; a row additionally needs both ends of the range.
889
- if (!range.includes("..") || !parseOffset(range)) {
890
- problem(e.file, `${at}: the range is one half-open offset range, 0x followed by upper-case hex digits on each side of ..`);
891
- continue;
892
- }
893
- if (!/^(?:-|\d+|0x[0-9A-F]+)$/.test(overlay))
894
- problem(e.file, `${at}: the overlay is its number, or - where there is none`);
895
- // A finding that does not exist is reported with the other unresolved IDs of the body.
896
- const ids = idsIn(finding);
897
- const cited = entries.get(ids[0]);
898
- if (ids.length !== 1 || kindOf(ids[0]) !== "FND" || finding !== ids[0])
899
- problem(e.file, `${at}: the finding column holds the ID of one finding`);
900
- else if (cited && !asList(cited.meta.builds).includes(id))
901
- problem(e.file, `${at}: ${ids[0]} does not list ${id}`);
902
- else if (cited?.meta.status === "superseded")
903
- problem(e.file, `${at}: cites ${ids[0]}, which is superseded`);
904
- // The finding shows code in this file of this build, so it has a location there that is not
905
- // file data. One with no locations there, or only file data there, shows no code in the row.
906
- else if (cited &&
907
- !asList(cited.meta.locations).some((loc) => loc?.build === id && loc?.file === path && loc?.kind !== "file-data")) {
908
- problem(e.file, `${at}: ${ids[0]} has no code location in ${path} of ${id}, so it cannot establish a code range there`);
909
- }
910
- const parsed = checkOffset(e.file, range, bf);
911
- if (parsed)
912
- ranges.push({ file: path, start: parsed[0], end: parsed[1] });
913
- }
914
- }
915
- // ---------------------------------------------------------------------------------------------
916
- // Per-entry checks
917
- const statusIndex = (s) => SCALE.indexOf(s);
918
- const isSuperseded = (id) => entries.get(id)?.meta.status === "superseded" ||
919
- (entries.get(id) &&
920
- asList(entries.get(id).meta.superseded_by).length > 0 &&
921
- ["BLD", "SRC"].includes(entries.get(id).kind));
922
- function checkIdForm(file, id) {
923
- const kind = kindOf(id);
924
- if (!KINDS[kind]) {
925
- problem(file, `${id} is not an ID of a known kind`);
926
- return;
927
- }
928
- if (kind === "BLD" || kind === "SRC") {
929
- if (!/^(BLD|SRC)-[A-Z][A-Z0-9.-]*$/.test(id))
930
- problem(file, `${id}: an alias starts with an upper-case letter and holds only upper-case letters, digits, dots and hyphens`);
931
- return;
932
- }
933
- const m = /^[A-Z]+-([A-Z][A-Z0-9]*)-(\d+)$/.exec(id);
934
- if (!m) {
935
- problem(file, `${id} does not have the form KIND-AREA-NNN`);
936
- return;
937
- }
938
- if (!areas.includes(m[1]))
939
- problem(file, `${id}: area ${m[1]} is not in the area list`);
940
- if (m[2].length < 3 || (m[2].length > 3 && m[2].startsWith("0")))
941
- problem(file, `${id}: the number is zero-padded to exactly three digits until it passes 999`);
942
- }
943
- function checkResolves(file, ids, what) {
944
- for (const id of ids)
945
- if (!entries.has(id))
946
- problem(file, `${what} cites ${id}, which does not exist`);
947
- }
948
- // What the evidence of a claim covers for the first build.
949
- // A whole entry counts as read completely when complete_reading holds any valid finding; a row of
950
- // its tables only when every static finding the row cites is part of that reading.
951
- const evidenceFacts = (e) => {
952
- const reading = completeReading(e);
953
- return {
954
- ...rowFacts(asList(e.meta.evidence), asList(e.meta.builds)[0], reading),
955
- completeReading: reading.length > 0,
956
- };
957
- };
958
- // The static findings of a complete reading: those in complete_reading that the entry cites in
959
- // evidence and that list its first build. Anything else there is reported where the field is checked.
960
- function completeReading(e) {
961
- const first = asList(e.meta.builds)[0];
962
- const evidence = asList(e.meta.evidence);
963
- return asList(e.meta.complete_reading).filter((x) => {
964
- const f = entries.get(x);
965
- return (f?.kind === "FND" && f.meta.method === "static" && evidence.includes(x) && asList(f.meta.builds).includes(first));
966
- });
967
- }
968
- // A run's draws from the generator, in order. Each is named by the rule it was made under, which
969
- // the rebuild cites too, and never by the address of the call in the original's code. A live
970
- // experiment's draws cite living rules, as its other links do.
971
- const DRAW_KEYS = ["rule", "bound", "result"];
972
- function checkDraws(fixture, draws, live) {
973
- if (draws === undefined)
974
- return;
975
- if (!Array.isArray(draws))
976
- return problem(fixture, "draws is a list");
977
- draws.forEach((draw, i) => {
978
- if (draw === null || typeof draw !== "object" || Array.isArray(draw))
979
- return problem(fixture, `draw ${i} is an object with rule, bound and result`);
980
- const extra = Object.keys(draw).filter((k) => !DRAW_KEYS.includes(k));
981
- if (extra.length)
982
- problem(fixture, `draw ${i} has ${extra.join(", ")}; a draw gives only rule, bound and result`);
983
- if (entries.get(draw.rule)?.kind !== "RULE")
984
- problem(fixture, `draw ${i} names ${draw.rule}, which is not a rule entry`);
985
- else if (live && isSuperseded(draw.rule))
986
- problem(fixture, `draw ${i} names ${draw.rule}, which is superseded`);
987
- if (!Number.isInteger(draw.bound) || !Number.isInteger(draw.result))
988
- problem(fixture, `draw ${i} gives bound and result as integers`);
989
- });
990
- }
991
- // True when all of an entry's evidence from the original running is emulated calls of single
992
- // functions, which model neither interrupts nor timing.
993
- function onlyEmulatedRuns(e) {
994
- const runs = asList(e.meta.evidence)
995
- .map((x) => entries.get(x))
996
- .filter((x) => x !== undefined && (x.kind === "EXP" || (x.kind === "FND" && x.meta.method === "dynamic")));
997
- return runs.length > 0 && runs.every((x) => x.kind === "EXP" && x.meta.starting_state === "emulated-call");
998
- }
999
- const mayBeInterrupted = (e) => e.kind === "RULE" && /# may run: RULE-/.test(e.code ?? "");
1000
- function checkStatusCitations(file, status, facts, conflicting, label = "status") {
1001
- if (status === "sourced" && facts.sources === 0)
1002
- problem(file, `${label} sourced needs at least one source`);
1003
- if (status === "supported" && facts.staticF + facts.dynamic === 0)
1004
- problem(file, `${label} supported needs at least one finding or experiment that lists the first build`);
1005
- if (status === "established" && (facts.staticF === 0 || (facts.dynamic === 0 && !facts.completeReading)))
1006
- problem(file, `${label} established needs a static finding and either a dynamic finding or experiment that list the first build, or a complete reading in complete_reading`);
1007
- if (status === "disputed" && conflicting.length === 0)
1008
- problem(file, `${label} disputed needs at least one finding or experiment in conflicting`);
1009
- }
1010
- // complete is the entry's complete reading, and the cited evidence counts as part of it when it
1011
- // holds static findings and every one of them is in it.
1012
- function rowFacts(ids, first, complete = []) {
1013
- let sources = 0, staticF = 0, dynamic = 0, outside = 0;
1014
- for (const id of ids) {
1015
- const ev = entries.get(id);
1016
- if (!ev)
1017
- continue;
1018
- if (ev.kind === "SRC")
1019
- sources++;
1020
- if (!["FND", "EXP"].includes(ev.kind) || !asList(ev.meta.builds).includes(first))
1021
- continue;
1022
- if (ev.kind === "EXP" || ev.meta.method === "dynamic")
1023
- dynamic++;
1024
- else if (ev.meta.method === "static") {
1025
- staticF++;
1026
- if (!complete.includes(id))
1027
- outside++;
1028
- }
1029
- }
1030
- return { sources, staticF, dynamic, completeReading: complete.length > 0 && staticF > 0 && outside === 0 };
1031
- }
1032
- // An unlisted format is already reported against its manifest, so it is skipped here.
1033
- function checkAddress(file, value, format) {
1034
- const rule = locationRule(format);
1035
- if (!rule)
1036
- return;
1037
- const re = rule.address;
1038
- if (!re) {
1039
- problem(file, `an address cannot be given in a file of format ${format}; use offset`);
1040
- return;
1041
- }
1042
- const parts = String(value).split("..");
1043
- if (parts.length > 2 || parts.some((p) => !re.test(p)))
1044
- problem(file, `address ${value} is not in the notation for a ${format} file`);
1045
- }
1046
- // An offset names one byte, or a half-open range of two: 0x20..0x3C covers 0x20 up to but not
1047
- // including 0x3C. Returns [start, end) as BigInts, or null when the notation is wrong.
1048
- function parseOffset(value) {
1049
- const parts = String(value).split("..");
1050
- if (parts.length > 2 || parts.some((p) => !/^0x[0-9A-F]{2,}$/.test(p)))
1051
- return null;
1052
- const [start, end] = parts.map((p) => BigInt(p));
1053
- return [start, end ?? start + 1n];
1054
- }
1055
- // An offset is into the shipped file bf, so the bytes it covers lie within bf.size. Returns the
1056
- // parsed range when it is well formed.
1057
- // `bf` is the file the offset is into, given as { path, size }: the shipped file, or the unpacked
1058
- // form of a packed one.
1059
- function checkOffset(file, value, bf, what = "shipped file") {
1060
- const range = parseOffset(value);
1061
- if (!range) {
1062
- problem(file, `offset ${value} must be 0x followed by at least two upper-case hex digits, or a range of two`);
1063
- return null;
1064
- }
1065
- const [start, end] = range;
1066
- if (start > end) {
1067
- problem(file, "offset range is reversed");
1068
- return null;
1069
- }
1070
- if (start === end) {
1071
- problem(file, `offset range ${value} is empty; a range is half-open`);
1072
- return null;
1073
- }
1074
- if (Number.isSafeInteger(bf.size) && bf.size >= 0 && end > BigInt(bf.size)) {
1075
- problem(file, `offset ${value} is outside the ${what} ${bf.path} (${bf.size} bytes)`);
1076
- return null;
1077
- }
1078
- return range;
1079
- }
1080
- const enumNames = new Map(); // name -> format IDs
1081
- const fieldNames = new Map(); // format ID -> Set of names
1082
- for (const [id, e] of entries) {
1083
- const { file, meta, kind } = e;
1084
- checkIdForm(file, id);
1085
- for (const f of FIELDS[kind].required)
1086
- if (!(f in meta))
1087
- problem(file, `front matter lacks ${f}`);
1088
- if (!Array.isArray(meta.superseded_by))
1089
- problem(file, "superseded_by must be a list");
1090
- const expectedSections = SECTIONS[kind];
1091
- const got = e.sections.map((s) => s.title);
1092
- if (got.join("|") !== expectedSections.join("|"))
1093
- problem(file, `sections must be ${expectedSections.join(", ")} in that order; found ${got.join(", ") || "none"}`);
1094
- for (const s of e.sections)
1095
- if (s.text.trim() === "")
1096
- problem(file, `section ${s.title} is empty; write None known. or None.`);
1097
- const superseded = asList(meta.superseded_by);
1098
- const status = meta.status;
1099
- if (KINDS[kind].statuses === "claim" && !CLAIM_STATUSES.includes(status))
1100
- problem(file, `status ${status} is not one of ${CLAIM_STATUSES.join(", ")}`);
1101
- if (KINDS[kind].statuses === "evidence" && !EVIDENCE_STATUSES.includes(status))
1102
- problem(file, `status ${status} is not one of ${EVIDENCE_STATUSES.join(", ")}`);
1103
- const isSup = status === "superseded" || ((kind === "BLD" || kind === "SRC") && superseded.length > 0);
1104
- if (status === "superseded" && superseded.length === 0)
1105
- problem(file, "a superseded entry names what replaced or disproved it in superseded_by");
1106
- if (status && status !== "superseded" && superseded.length > 0)
1107
- problem(file, "superseded_by must be empty unless the status is superseded");
1108
- checkResolves(file, superseded, "superseded_by");
1109
- for (const s of superseded) {
1110
- const k = kindOf(s);
1111
- const ok = ["FND", "EXP"].includes(kind)
1112
- ? ["FND", "EXP"].includes(k)
1113
- : kind === "BLD"
1114
- ? k === "BLD"
1115
- : kind === "SRC"
1116
- ? k === "SRC"
1117
- : kind === "BUG"
1118
- ? true
1119
- : k !== "SRC" && k !== "BLD";
1120
- if (!ok)
1121
- problem(file, `superseded_by may not name ${s}`);
1122
- }
1123
- if (kind !== "BLD" && kind !== "SRC") {
1124
- const builds = asList(meta.builds);
1125
- if (builds.length === 0)
1126
- problem(file, "builds must list at least one build");
1127
- checkResolves(file, builds, "builds");
1128
- for (const b of builds)
1129
- if (entries.has(b) && kindOf(b) !== "BLD")
1130
- problem(file, `builds lists ${b}, which is not a build`);
1131
- }
1132
- // Links that must not point at superseded entries
1133
- if (!isSup) {
1134
- const linkFields = ["builds", "evidence", "conflicting", "related"];
1135
- for (const f of linkFields)
1136
- for (const t of asList(meta[f]))
1137
- if (entries.has(t) && isSuperseded(t))
1138
- problem(file, `${f} cites ${t}, which is superseded`);
1139
- for (const loc of asList(meta.locations))
1140
- if (loc && entries.has(loc.build) && isSuperseded(loc.build))
1141
- problem(file, `a location names ${loc.build}, which is superseded`);
1142
- }
1143
- if (["FND", "EXP"].includes(kind)) {
1144
- const rep = asList(meta.reproduced_by);
1145
- if (status === "reproduced" && rep.every((p) => p === meta.recorded_by))
1146
- problem(file, "a reproduced entry names someone other than recorded_by in reproduced_by");
1147
- if (status !== "reproduced" && rep.length > 0)
1148
- problem(file, "reproduced_by must be empty unless the status is reproduced");
1149
- if (typeof meta.recorded_by !== "string" || !meta.recorded_by)
1150
- problem(file, "recorded_by must be a GitHub username");
1151
- }
1152
- if (kind === "FND") {
1153
- if (!["static", "dynamic"].includes(meta.method))
1154
- problem(file, "method must be static or dynamic");
1155
- if (meta.method === "static" && meta.environment !== null)
1156
- problem(file, "a static finding has environment: null");
1157
- if (meta.method === "dynamic" && (meta.environment === null || meta.environment === ""))
1158
- problem(file, "a dynamic finding gives its environment");
1159
- const locations = asList(meta.locations);
1160
- const builds = asList(meta.builds);
1161
- if (meta.method === "static")
1162
- for (const b of builds)
1163
- if (!locations.some((l) => l && l.build === b))
1164
- problem(file, `a static finding has at least one location in ${b}`);
1165
- for (const loc of locations) {
1166
- if (!loc || typeof loc !== "object") {
1167
- problem(file, "a location must be a map of build, file and address or offset");
1168
- continue;
1169
- }
1170
- if (!builds.includes(loc.build))
1171
- problem(file, `location build ${loc.build} is not in builds`);
1172
- const files = buildFiles.get(loc.build) ?? [];
1173
- const bf = files.find((f) => f.path === loc.file);
1174
- if (!bf) {
1175
- problem(file, `location file ${loc.file} is not in the files of ${loc.build}`);
1176
- continue;
1177
- }
1178
- const format = bf.unpacked?.format ?? bf.format;
1179
- // kind tells code from data within an executable; a data file holds no code to tell apart.
1180
- if (loc.kind !== undefined && !locationRule(format)?.address)
1181
- problem(file, `location kind ${loc.kind} in ${loc.file}: a ${format} file is not an executable, so its locations give no kind`);
1182
- else if (loc.kind !== undefined && loc.kind !== "code" && loc.kind !== "file-data")
1183
- problem(file, `location kind ${loc.kind} in ${loc.file}: kind must be code or file-data when given`);
1184
- const fileData = loc.kind === "file-data";
1185
- if (fileData && "address" in loc)
1186
- problem(file, `a file-data location in ${loc.file} gives a file offset, not an address`);
1187
- // A file-data location in a packed file may name bytes that exist only once it is unpacked,
1188
- // such as the relocation table an unpacker writes, by an offset into the unpacked form.
1189
- const intoUnpacked = loc.unpacked === true;
1190
- if ("unpacked" in loc && !intoUnpacked)
1191
- problem(file, `a location in ${loc.file} gives unpacked: true or leaves it out`);
1192
- else if (intoUnpacked && !fileData)
1193
- problem(file, `a location in ${loc.file} gives unpacked: true only with kind: file-data`);
1194
- else if (intoUnpacked && !bf.packer)
1195
- problem(file, `a location in ${loc.file} gives unpacked: true, but ${loc.file} is not packed`);
1196
- if ("address" in loc && "offset" in loc)
1197
- problem(file, "a location gives address or offset, not both");
1198
- if ("address" in loc) {
1199
- if (!fileData)
1200
- checkAddress(file, loc.address, format);
1201
- }
1202
- else if ("offset" in loc) {
1203
- // Explicit file-data offsets name shipped container metadata or data, never code.
1204
- // Other offsets name bytes of the shipped file: data, CD audio, or MZ overlay code
1205
- // outside the load image. The finding must establish the overlay mapping. Like an
1206
- // address, an offset is judged by the unpacked format, so a packed MZ stub around LE
1207
- // or PE code cannot use offsets for code the loader maps.
1208
- const rule = locationRule(format);
1209
- if (!fileData && rule && !rule.offset)
1210
- problem(file, `location in ${loc.file} gives an offset; a ${format} executable is located by address (only MZ overlay code uses offsets)`);
1211
- const range = intoUnpacked && bf.packer
1212
- ? checkOffset(file, loc.offset, { path: bf.path, size: Number(bf.unpacked?.size) }, "unpacked form of")
1213
- : checkOffset(file, loc.offset, bf);
1214
- // An offset into an executable locates overlay code, so it lies wholly inside one row of
1215
- // the build's Code ranges. Adjacent rows are not joined: a range that crosses from one
1216
- // into the next, such as into another bank, fails.
1217
- if (!fileData && range && rule?.offset && rule.address && codeRanges.has(loc.build)) {
1218
- const inside = codeRanges
1219
- .get(loc.build)
1220
- .some((r) => r.file === loc.file && r.start <= range[0] && range[1] <= r.end);
1221
- if (!inside)
1222
- problem(file, `offset ${loc.offset} in ${loc.file} does not lie wholly inside one of the rows the Code ranges section of ${loc.build} gives for that file`);
1223
- }
1224
- }
1225
- else
1226
- problem(file, "a location gives an address or an offset");
1227
- }
1228
- }
1229
- if (kind === "EXP") {
1230
- const builds = asList(meta.builds);
1231
- if (builds.length !== 1)
1232
- problem(file, "an experiment lists exactly one build");
1233
- const fixture = meta.fixture && join(specDir, "experiments", meta.fixture);
1234
- if (!fixture || !existsSync(fixture))
1235
- problem(file, `fixture ${meta.fixture} does not exist`);
1236
- else {
1237
- try {
1238
- const fx = JSON.parse(readFileSync(fixture, "utf8"));
1239
- if (fx.experiment !== id)
1240
- problem(fixture, `experiment must be ${id}`);
1241
- if (!["new-game", "emulated-call"].includes(meta.starting_state) &&
1242
- !(fx.starting_state && fx.starting_state.xxh3))
1243
- problem(fixture, "gives the hash of the save its runs started from");
1244
- if (typeof meta.starting_state === "string" &&
1245
- meta.starting_state.endsWith(".patch.json") &&
1246
- !fx.starting_state?.base_xxh3)
1247
- problem(fixture, "a patch fixture gives the base save's hash as well");
1248
- for (const run of asList(fx.runs)) {
1249
- for (const ev of asList(run?.events))
1250
- if (!glossary.has(ev?.event))
1251
- problem(fixture, `event ${ev?.event} has no glossary entry`);
1252
- checkDraws(fixture, run?.draws, !isSup);
1253
- }
1254
- if (typeof meta.recording === "string" && meta.recording !== "" && !fx.recording_xxh3)
1255
- problem(fixture, "an experiment with a recording gives the recording's hash in recording_xxh3");
1256
- }
1257
- catch (err) {
1258
- problem(fixture, `is not valid JSON: ${err.message}`);
1259
- }
1260
- }
1261
- if (typeof meta.starting_state === "string" &&
1262
- meta.starting_state.startsWith("saves/") &&
1263
- !existsSync(join(specDir, "experiments", meta.starting_state)))
1264
- problem(file, `starting_state ${meta.starting_state} does not exist`);
1265
- // A recording is committed in recordings/, kept with the captures, or one of the build's files.
1266
- if (typeof meta.recording === "string" && meta.recording !== "") {
1267
- const rec = meta.recording;
1268
- if (rec.startsWith("recordings/")) {
1269
- if (!existsSync(join(specDir, "experiments", rec)))
1270
- problem(file, `recording ${rec} does not exist`);
1271
- }
1272
- else if (!rec.startsWith("captures/") &&
1273
- !builds.some((b) => (buildFiles.get(b) ?? []).some((f) => f.path === rec)))
1274
- problem(file, `recording ${rec} is neither in recordings/ or captures/ nor a file of ${builds.join(", ")}`);
1275
- }
1276
- }
1277
- if (KINDS[kind].statuses === "claim") {
1278
- for (const f of CLAIM_LINKS)
1279
- if (!Array.isArray(meta[f]))
1280
- problem(file, `${f} must be a list`);
1281
- const evidence = asList(meta.evidence);
1282
- const conflicting = asList(meta.conflicting);
1283
- const related = asList(meta.related);
1284
- const split = asList(meta.split_with);
1285
- checkResolves(file, evidence, "evidence");
1286
- checkResolves(file, conflicting, "conflicting");
1287
- checkResolves(file, related, "related");
1288
- checkResolves(file, split, "split_with");
1289
- if ("complete_reading" in meta) {
1290
- if (!Array.isArray(meta.complete_reading))
1291
- problem(file, "complete_reading must be a list");
1292
- const reading = asList(meta.complete_reading);
1293
- checkResolves(file, reading, "complete_reading");
1294
- const first = asList(meta.builds)[0];
1295
- for (const x of reading) {
1296
- const f = entries.get(x);
1297
- if (!f)
1298
- continue;
1299
- if (f.kind !== "FND" || f.meta.method !== "static")
1300
- problem(file, `complete_reading may hold only static findings, not ${x}`);
1301
- else if (!evidence.includes(x))
1302
- problem(file, `complete_reading names ${x}; list it in evidence as well`);
1303
- else if (!asList(f.meta.builds).includes(first))
1304
- problem(file, `complete_reading names ${x}, which does not list the first build ${first}`);
1305
- }
1306
- }
1307
- for (const x of evidence)
1308
- if (!["FND", "EXP", "SRC"].includes(kindOf(x)))
1309
- problem(file, `evidence may hold only findings, experiments and sources, not ${x}`);
1310
- for (const x of conflicting)
1311
- if (!["FND", "EXP"].includes(kindOf(x)))
1312
- problem(file, `conflicting may hold only findings and experiments, not ${x}`);
1313
- if (conflicting.length > 0 && status !== "disputed")
1314
- problem(file, "conflicting must be empty unless the status is disputed");
1315
- for (const x of related)
1316
- if (!RELATED_KINDS[kind].includes(kindOf(x)))
1317
- problem(file, `related may not link to ${x}`);
1318
- for (const s of split) {
1319
- const other = entries.get(s);
1320
- if (!other)
1321
- continue;
1322
- if (!asList(other.meta.split_with).includes(id))
1323
- problem(file, `${s} does not name ${id} back in split_with`);
1324
- for (const b of asList(meta.builds))
1325
- if (asList(other.meta.builds).includes(b))
1326
- problem(file, `${s} is split from this entry but also lists ${b}`);
1327
- }
1328
- if (status !== "superseded") {
1329
- const facts = evidenceFacts(e);
1330
- checkStatusCitations(file, status, facts, conflicting);
1331
- if (["supported", "established"].includes(status)) {
1332
- for (const b of asList(meta.builds)) {
1333
- const covered = evidence.some((x) => ["FND", "EXP"].includes(kindOf(x)) && asList(entries.get(x)?.meta.builds).includes(b));
1334
- if (!covered)
1335
- problem(file, `lists ${b}, but no finding or experiment it cites lists that build`);
1336
- }
1337
- }
1338
- }
1339
- if (kind === "BUG") {
1340
- if (!["crash", "hang", "save-corruption", "rules", "presentation", "performance"].includes(meta.impact))
1341
- problem(file, "impact must be crash, hang, save-corruption, rules, presentation or performance");
1342
- if (!["unintended", "unclear"].includes(meta.intent))
1343
- problem(file, "intent must be unintended or unclear");
1344
- if (!["relied-on", "not-relied-on", "unknown"].includes(meta.player_reliance))
1345
- problem(file, "player_reliance must be relied-on, not-relied-on or unknown");
1346
- if (!related.some((x) => ["RULE", "FMT", "SCR"].includes(kindOf(x))))
1347
- problem(file, "a bug names at least one rule, format or screen in related");
1348
- }
1349
- }
1350
- if (kind === "BLD" && ![16, 32].includes(meta.int_width))
1351
- problem(file, "int_width must be 16 or 32");
1352
- if (kind === "SRC" && meta.xxh3 !== null && !/^[0-9a-f]{32}$/.test(String(meta.xxh3)))
1353
- problem(file, "xxh3 must be null or 32 lower-case hex digits");
1354
- if (kind === "FMT")
1355
- checkFormat(e);
1356
- if (kind === "SCR")
1357
- checkScreen(e);
1358
- }
1359
- function tableIds(e, sectionTitles) {
1360
- const ids = new Set();
1361
- for (const s of e.sections)
1362
- if (!sectionTitles || sectionTitles.includes(s.title))
1363
- for (const t of tables(s.text))
1364
- for (const r of t.rows)
1365
- for (const x of idsIn(r[t.header.length - 1] ?? ""))
1366
- ids.add(x);
1367
- return ids;
1368
- }
1369
- function checkFormat(e) {
1370
- const { file, meta } = e;
1371
- const id = meta.id;
1372
- const first = asList(meta.builds)[0];
1373
- if (meta.text === true) {
1374
- if (meta.definition !== null || meta.size !== null || meta.byte_order !== null)
1375
- problem(file, "a text format has definition, size and byte_order null");
1376
- }
1377
- else {
1378
- if (!["little", "big"].includes(meta.byte_order))
1379
- problem(file, "byte_order must be little or big for a binary format");
1380
- if (meta.status !== "unknown" && meta.status !== "superseded") {
1381
- const expected = `${id.toLowerCase().replaceAll("-", "_")}.ksy`;
1382
- if (meta.definition !== expected)
1383
- problem(file, `definition must be ${expected}`);
1384
- else if (!existsSync(join(dirname(file), expected)))
1385
- problem(file, `definition ${expected} does not exist`);
1386
- }
1387
- }
1388
- for (const pattern of asList(meta.files)) {
1389
- const re = new RegExp("^" +
1390
- String(pattern)
1391
- .replace(/[.+^${}()|[\]\\]/g, "\\$&")
1392
- .replaceAll("*", "[^/]*")
1393
- .replaceAll("?", "[^/]") +
1394
- "$");
1395
- for (const b of asList(meta.builds))
1396
- if (!(buildFiles.get(b) ?? []).some((f) => re.test(f.path)))
1397
- problem(file, `files pattern ${pattern} matches no file of ${b}`);
1398
- }
1399
- const layout = e.sections.find((s) => s.title === "Layout");
1400
- const enums = e.sections.find((s) => s.title === "Enumerations and flags");
1401
- const names = new Set();
1402
- fieldNames.set(id, names);
1403
- let lowest = null;
1404
- let disputed = false;
1405
- const visit = (t, kindLabel) => {
1406
- const statusCol = t.header.indexOf("Status");
1407
- const evCol = t.header.indexOf("Evidence");
1408
- const nameCol = t.header.indexOf("Name");
1409
- for (const row of t.rows) {
1410
- if (row.length !== t.header.length) {
1411
- problem(file, `${kindLabel} row ${row.join(" | ")} has ${row.length} cells, not ${t.header.length}`);
1412
- continue;
1413
- }
1414
- const isTotal = (row[t.header.indexOf("Meaning")] ?? "").startsWith("Total") && (row[statusCol] ?? "") === "";
1415
- if (isTotal)
1416
- continue;
1417
- const st = row[statusCol];
1418
- if (!ROW_STATUSES.includes(st)) {
1419
- problem(file, `${kindLabel} row ${row[nameCol] ?? row[0]}: status ${st} is not allowed in a row`);
1420
- continue;
1421
- }
1422
- if (st === "disputed")
1423
- disputed = true;
1424
- else if (lowest === null || statusIndex(st) < statusIndex(lowest))
1425
- lowest = st;
1426
- const ids = idsIn(row[evCol]);
1427
- checkResolves(file, ids, `${kindLabel} row ${row[nameCol] ?? row[0]}`);
1428
- const conflicting = ids.filter((x) => asList(meta.conflicting).includes(x));
1429
- checkStatusCitations(file, st, rowFacts(ids, first, completeReading(e)), conflicting, `${kindLabel} row ${(row[nameCol] ?? row[0]).replaceAll("`", "")}: status`);
1430
- if (nameCol >= 0 && row[nameCol])
1431
- names.add(row[nameCol].replaceAll("`", ""));
1432
- }
1433
- };
1434
- if (layout) {
1435
- const ts = tables(layout.text);
1436
- const wanted = meta.text === true ? TEXT_LAYOUT : BINARY_LAYOUT;
1437
- if (meta.status !== "unknown" && ts.length === 0)
1438
- problem(file, "Layout has no table");
1439
- for (const t of ts) {
1440
- if (t.header.join("|") !== wanted.join("|"))
1441
- problem(file, `a layout table has the columns ${wanted.join(" | ")}`);
1442
- else
1443
- visit(t, "layout");
1444
- }
1445
- }
1446
- // An enumeration table kept in a value file counts as one of the entry's tables.
1447
- const enumTables = enums ? [...tables(enums.text), ...valueFileTables(e, enums.text)] : [];
1448
- e.valueTables = enumTables.filter((t) => t.file);
1449
- for (const t of enumTables) {
1450
- if (t.header.join("|") !== ENUM_TABLE.join("|")) {
1451
- problem(t.file ?? file, `an enumeration table has the columns ${ENUM_TABLE.join(" | ")}`);
1452
- continue;
1453
- }
1454
- if (!t.heading)
1455
- problem(file, "an enumeration table sits under a ### heading naming its fields");
1456
- else
1457
- for (const f of (t.heading.match(/`([^`]+)`/g) ?? t.heading.split(/\s*,\s*|\s+and\s+/))
1458
- .map((x) => x.replaceAll("`", "").trim())
1459
- .filter(Boolean))
1460
- if (!names.has(f))
1461
- problem(file, `enumeration heading names ${f}, which is not a field of the layout`);
1462
- visit(t, "enumeration");
1463
- for (const row of t.rows) {
1464
- if (row.length !== t.header.length)
1465
- continue; // visit reported it
1466
- const n = row[1].replaceAll("`", "");
1467
- if (!/^[A-Z][A-Z0-9_]*$/.test(n))
1468
- problem(file, `enumeration name ${n} must be upper-case letters, digits and underscores`);
1469
- if (!enumNames.has(n))
1470
- enumNames.set(n, []);
1471
- enumNames.get(n).push(id);
1472
- }
1473
- }
1474
- if (meta.status !== "superseded" && meta.status !== "unknown") {
1475
- const expected = disputed ? "disputed" : lowest;
1476
- if (expected && meta.status !== expected)
1477
- problem(file, `status must be ${expected}, the lowest status among its rows`);
1478
- }
1479
- const cited = tableIds(e, ["Layout", "Enumerations and flags"]);
1480
- for (const t of e.valueTables)
1481
- for (const row of t.rows)
1482
- for (const x of idsIn(row[t.header.length - 1]))
1483
- cited.add(x);
1484
- const listed = new Set([...asList(meta.evidence), ...asList(meta.conflicting)]);
1485
- for (const x of cited)
1486
- if (!listed.has(x) && ["FND", "EXP", "SRC"].includes(kindOf(x)))
1487
- problem(file, `${x} is cited in a table but not in evidence or conflicting`);
1488
- for (const t of tables(layout?.text ?? ""))
1489
- for (const row of t.rows)
1490
- for (const x of idsIn(row[t.header.indexOf("Meaning")]))
1491
- if (kindOf(x) === "RULE" && !asList(meta.related).includes(x))
1492
- problem(file, `layout names ${x}; add it to related`);
1493
- // Kaitai definition
1494
- if (meta.definition && existsSync(join(dirname(file), meta.definition))) {
1495
- const ksy = readFileSync(join(dirname(file), meta.definition), "utf8");
1496
- const expectedId = id.toLowerCase().replaceAll("-", "_");
1497
- if (!new RegExp(`^\\s*id:\\s*${expectedId}\\s*$`, "m").test(ksy))
1498
- problem(join(dirname(file), meta.definition), `meta/id must be ${expectedId}`);
1499
- if (!/^\s*license:\s*\S+/m.test(ksy))
1500
- problem(join(dirname(file), meta.definition), "meta/license must name the licence");
1501
- }
1502
- }
1503
- // The enumeration tables an entry keeps in value files: each ### heading stays in the entry, followed
1504
- // by a sentence naming its file, formats/<ID>.<table>.csv.
1505
- function valueFileTables(e, text) {
1506
- const out = [];
1507
- let heading = null;
1508
- let fence = false;
1509
- for (const line of text.split("\n")) {
1510
- if (/^(```|~~~)/.test(line))
1511
- fence = !fence;
1512
- if (fence)
1513
- continue;
1514
- const h = /^### (.+)$/.exec(line);
1515
- if (h) {
1516
- heading = h[1].trim();
1517
- continue;
1518
- }
1519
- for (const m of line.matchAll(/\b([A-Z]+-[A-Z0-9]+-\d{3,}\.[A-Za-z0-9_]+\.csv)\b/g)) {
1520
- // Another entry's value file holds that entry's rows, so it is not read as one of these.
1521
- if (!m[1].startsWith(`${e.meta.id}.`)) {
1522
- problem(e.file, `value file ${m[1]} belongs to another entry; an entry's value files are named ${e.meta.id}.<table>.csv`);
1523
- continue;
1524
- }
1525
- const path = join(dirname(e.file), m[1]);
1526
- if (!existsSync(path)) {
1527
- problem(e.file, `value file ${m[1]} does not exist`);
1528
- continue;
1529
- }
1530
- const csv = readCsv(path);
1531
- if (csv)
1532
- out.push({ heading, header: csv.header, rows: csv.rows, file: path });
1533
- }
1534
- }
1535
- return out;
1536
- }
1537
- function checkScreen(e) {
1538
- const { file, meta } = e;
1539
- if (!/^\d+x\d+$/.test(String(meta.resolution)))
1540
- problem(file, "resolution must be written WIDTHxHEIGHT");
1541
- for (const s of e.sections) {
1542
- const want = SCREEN_TABLES[s.title];
1543
- if (!want)
1544
- continue;
1545
- const ts = tables(s.text);
1546
- if (ts.length === 0 && !/^\s*None( known)?\.\s*$/.test(s.text))
1547
- problem(file, `${s.title} has neither a table nor None known.`);
1548
- for (const t of ts)
1549
- if (t.header.join("|") !== want.join("|"))
1550
- problem(file, `${s.title} table has the columns ${want.join(" | ")}`);
1551
- }
1552
- const cited = tableIds(e, Object.keys(SCREEN_TABLES));
1553
- const listed = new Set([...asList(meta.evidence), ...asList(meta.conflicting)]);
1554
- for (const x of cited)
1555
- if (!listed.has(x) && ["FND", "EXP", "SRC"].includes(kindOf(x)))
1556
- problem(file, `${x} is cited in a table but not in evidence or conflicting`);
1557
- for (const s of e.sections)
1558
- for (const t of tables(s.text)) {
1559
- for (const col of ["Effect", "Shows"]) {
1560
- const c = t.header.indexOf(col);
1561
- if (c < 0)
1562
- continue;
1563
- for (const row of t.rows)
1564
- for (const x of idsIn(row[c]))
1565
- if (["RULE", "SCR"].includes(kindOf(x)) && !asList(meta.related).includes(x))
1566
- problem(file, `${col} cell names ${x}; add it to related`);
1567
- }
1568
- }
1569
- for (const x of cited)
1570
- checkResolves(file, [x], "a table");
1571
- }
1572
- // ---------------------------------------------------------------------------------------------
1573
- // Rules: procedures against the glossary and the formats
1574
- const BUILTINS = new Set([
1575
- "min",
1576
- "max",
1577
- "abs",
1578
- "count",
1579
- "append",
1580
- "insert",
1581
- "remove_at",
1582
- "copy",
1583
- "stable_sort",
1584
- "sprintf",
1585
- "floor",
1586
- "ceil",
1587
- "round_even",
1588
- "draw",
1589
- "resource",
1590
- "read_file",
1591
- "write_file",
1592
- "free",
1593
- "fmod",
1594
- "UINT8",
1595
- "INT8",
1596
- "UINT16",
1597
- "INT16",
1598
- "UINT32",
1599
- "INT32",
1600
- "UINT64",
1601
- "INT64",
1602
- "FLOAT32",
1603
- "FLOAT64",
1604
- "FLOAT80",
1605
- "REAL48",
1606
- ]);
1607
- const KEYWORDS = new Set([
1608
- "for",
1609
- "each",
1610
- "in",
1611
- "if",
1612
- "else",
1613
- "while",
1614
- "break",
1615
- "continue",
1616
- "return",
1617
- "let",
1618
- "and",
1619
- "or",
1620
- "not",
1621
- "true",
1622
- "false",
1623
- "call",
1624
- "define",
1625
- "emit",
1626
- "drain",
1627
- "show",
1628
- "new",
1629
- "table",
1630
- "clock",
1631
- "from",
1632
- "Hz",
1633
- ]);
1634
- const defined = new Map(); // function/table/clock name -> rule IDs
1635
- // A superseded rule keeps its procedure for history, but its declarations own no active name.
1636
- const historical = new Map(); // name declared by superseded rules -> rule IDs
1637
- for (const [id, e] of entries) {
1638
- if (e.kind !== "RULE")
1639
- continue;
1640
- const proc = e.sections.find((s) => s.title === "Procedure")?.text ?? "";
1641
- e.code = [...proc.matchAll(/```text\n([\s\S]*?)```/g)].map((m) => m[1]).join("\n");
1642
- const owners = e.meta.status === "superseded" ? historical : defined;
1643
- for (const m of e.code.matchAll(/^\s*(?:define\s+([a-z_][a-z0-9_]*)\s*\(|table\s+([a-z_][a-z0-9_]*)\s*:|clock\s+([a-z_][a-z0-9_]*)\s*:)/gm)) {
1644
- const name = m[1] ?? m[2] ?? m[3];
1645
- if (!owners.has(name))
1646
- owners.set(name, []);
1647
- owners.get(name).push(id);
1648
- }
1649
- }
1650
- // A live procedure cannot rely on a name only a superseded rule declares.
1651
- const onlyHistorical = (name) => !defined.has(name) && historical.has(name);
1652
- for (const [name, ids] of defined) {
1653
- const splitGroup = asList(entries.get(ids[0]).meta.split_with).concat(ids[0]);
1654
- if (ids.length > 1 && !ids.every((x) => splitGroup.includes(x)))
1655
- problem(null, `${name} is defined by more than one rule: ${ids.join(", ")}`);
1656
- if (!glossary.has(name))
1657
- problem(glossaryDir, `${name}, defined by ${ids[0]}, has no glossary entry`);
1658
- }
1659
- for (const [id, e] of entries) {
1660
- if (e.kind !== "RULE" || e.meta.status === "superseded")
1661
- continue;
1662
- const { file, meta } = e;
1663
- // Types in a define's signature (`define roll(n: UINT16) -> char[]:`) are not names the
1664
- // procedure reads, so they are dropped before the name checks.
1665
- const TYPE = String.raw `(?:[A-Za-z][A-Za-z0-9]*|FMT-[A-Z0-9]+-\d+)(?:\[[^\]]*\])?`;
1666
- const code = e
1667
- .code.replace(/#.*$/gm, "")
1668
- .replace(/"[^"]*"/g, '""')
1669
- .replace(new RegExp(String.raw `(\bdefine\s+[a-z_][a-z0-9_]*\s*\()([^)]*)\)(\s*->\s*${TYPE})?`, "g"), (_, head, params) => `${head}${params
1670
- .split(",")
1671
- .map((p) => p.split(":")[0].trim())
1672
- .join(", ")})`);
1673
- const related = asList(meta.related);
1674
- const openQuestions = e.sections.find((s) => s.title === "Open questions")?.text ?? "";
1675
- // Names are letters, digits and underscores, so best_score does not count as listing score.
1676
- const listsOpen = (name) => new RegExp(`(?<![A-Za-z0-9_])${name}(?![A-Za-z0-9_])`).test(openQuestions);
1677
- if (meta.status !== "unknown" && e.code.trim() === "")
1678
- problem(file, "Procedure has no ```text block");
1679
- const usedTerms = new Set();
1680
- const useTerm = (name) => {
1681
- if (glossary.has(name))
1682
- usedTerms.add(name);
1683
- return glossary.has(name);
1684
- };
1685
- // Lists written out hold at most LIST_LIMIT values; a table of more takes them from a value file.
1686
- for (const m of code.matchAll(/=\s*\[([^\]]*)\]/g)) {
1687
- const count = m[1].split(",").filter((x) => x.trim() !== "").length;
1688
- if (count > LIST_LIMIT)
1689
- problem(file, `writes out a list of ${count} values; a list of more than ${LIST_LIMIT} is a table with a value file`);
1690
- }
1691
- for (const m of e.code.matchAll(/^\s*table\s+([a-z_][a-z0-9_]*)\s*:\s*[A-Za-z0-9]+\[([^\]]*)\]\s*from\s*"([^"]*)"/gm)) {
1692
- const [, name, count, csvName] = m;
1693
- const expected = `${id}.${name}.csv`;
1694
- if (csvName !== expected) {
1695
- problem(file, `table ${name} takes its values from ${expected}, not ${csvName}`);
1696
- continue;
1697
- }
1698
- const path = join(dirname(file), csvName);
1699
- if (!existsSync(path)) {
1700
- problem(file, `value file ${csvName} does not exist`);
1701
- continue;
1702
- }
1703
- const csv = readCsv(path);
1704
- if (!csv)
1705
- continue;
1706
- if (csv.header.join("|") !== "value")
1707
- problem(path, "a table's value file has the single column value");
1708
- if (/^\d+$/.test(count.trim()) && csv.rows.length !== Number(count))
1709
- problem(path, `has ${csv.rows.length} values, but table ${name} has ${count}`);
1710
- for (const [v] of csv.rows)
1711
- if (!/^-?(?:0x[0-9A-Fa-f]+|\d+(?:\.\d+)?)$/.test(v.trim())) {
1712
- problem(path, `${v} is not a number`);
1713
- break;
1714
- }
1715
- }
1716
- for (const m of code.matchAll(/\bcall\s+(RULE-[A-Z0-9]+-\d+)/g))
1717
- if (!related.includes(m[1]))
1718
- problem(file, `calls ${m[1]}; add it to related`);
1719
- for (const m of code.matchAll(/\bshow\s+(SCR-[A-Z0-9]+-\d+)/g))
1720
- if (!related.includes(m[1]))
1721
- problem(file, `shows ${m[1]}; add it to related`);
1722
- for (const m of code.matchAll(/\b(FMT-[A-Z0-9]+-\d+)/g))
1723
- if (!related.includes(m[1]))
1724
- problem(file, `uses ${m[1]}; add it to related`);
1725
- for (const m of e.code.matchAll(/# may run: (RULE-[A-Z0-9]+-\d+)/g))
1726
- if (!related.includes(m[1]))
1727
- problem(file, `may be interrupted by ${m[1]}; add it to related`);
1728
- if (meta.status === "established" && mayBeInterrupted(e) && onlyEmulatedRuns(e))
1729
- problem(file, "another rule may interrupt this procedure (# may run:), so emulated calls alone cannot establish it");
1730
- if (asList(meta.complete_reading).length > 0 && /# may run: RULE-/.test(e.code))
1731
- problem(file, "another rule may interrupt this procedure (# may run:), so a complete reading cannot establish it; leave complete_reading empty");
1732
- for (const x of idsIn(code))
1733
- if (!entries.has(x))
1734
- problem(file, `procedure names ${x}, which does not exist`);
1735
- for (const m of code.matchAll(/\bemit\s+([A-Za-z_][A-Za-z0-9_]*)/g))
1736
- if (!useTerm(m[1]))
1737
- problem(file, `emits ${m[1]}, which has no glossary entry`);
1738
- for (const m of code.matchAll(/\bdrain\s+([A-Za-z_][A-Za-z0-9_]*)/g))
1739
- if (!useTerm(m[1]))
1740
- problem(file, `drains ${m[1]}, which has no glossary entry`);
1741
- for (const m of code.matchAll(/\b((?:fn|g|scr)_[A-Za-z0-9_]+)\b/g)) {
1742
- if (!useTerm(m[1]))
1743
- problem(file, `uses the neutral name ${m[1]}, which has no glossary entry`);
1744
- if (!listsOpen(m[1]))
1745
- problem(file, `uses the neutral name ${m[1]}; list it in Open questions`);
1746
- }
1747
- const noNeutral = code.replace(/\b(?:fn|g|scr)_[A-Za-z0-9_]+\b/g, "");
1748
- if (/\b0x[0-9A-Fa-f]{6,}\b/.test(noNeutral) && /\b0x00[4-9A-F][0-9A-F]{5}\b/.test(noNeutral))
1749
- problem(file, "the procedure contains what looks like an address outside a neutral name");
1750
- // Functions called without `call`
1751
- const locals = new Set();
1752
- for (const m of code.matchAll(/\blet\s+([a-z_][a-z0-9_]*)/g))
1753
- locals.add(m[1]);
1754
- for (const m of code.matchAll(/\bfor\s+(?:each\s+)?([a-z_][a-z0-9_]*)\s+in\b/g))
1755
- locals.add(m[1]);
1756
- for (const m of code.matchAll(/\bdefine\s+[a-z_][a-z0-9_]*\s*\(([^)]*)\)/g))
1757
- for (const p of m[1].split(","))
1758
- locals.add(p.split(":")[0].trim());
1759
- const params = e.sections.find((s) => s.title === "Parameters")?.text ?? "";
1760
- for (const m of params.matchAll(/`([a-z_][a-z0-9_]*)`/g))
1761
- locals.add(m[1]);
1762
- for (const m of code.matchAll(/(?<![.\w])([a-z_][a-z0-9_]*)\s*\(/g)) {
1763
- const name = m[1];
1764
- if (BUILTINS.has(name) || KEYWORDS.has(name) || locals.has(name))
1765
- continue;
1766
- if (onlyHistorical(name)) {
1767
- problem(file, `calls ${name}(), which only superseded ${historical.get(name).join(", ")} defines`);
1768
- continue;
1769
- }
1770
- if (!defined.has(name) && !useTerm(name)) {
1771
- problem(file, `calls ${name}(), which no rule defines and the glossary does not list`);
1772
- continue;
1773
- }
1774
- for (const owner of defined.get(name) ?? [])
1775
- if (owner !== id && !related.includes(owner))
1776
- problem(file, `uses ${name} from ${owner}; add ${owner} to related`);
1777
- }
1778
- for (const m of code.matchAll(/(?<![.\w])([A-Z][A-Z0-9_]*[A-Z0-9])(?![\w-])/g)) {
1779
- const name = m[1];
1780
- if (BUILTINS.has(name) || KINDS[name] || /^(?:FMT|RULE|SCR)$/.test(name))
1781
- continue;
1782
- if (!enumNames.has(name)) {
1783
- problem(file, `upper-case name ${name} is not an enumeration name of any format`);
1784
- continue;
1785
- }
1786
- for (const fmt of enumNames.get(name))
1787
- if (!related.includes(fmt))
1788
- problem(file, `uses ${name} from ${fmt}; add it to related`);
1789
- }
1790
- // Names read or assigned without let that are neither locals nor glossary terms
1791
- for (const m of code.matchAll(/(?<![.\w])([a-z_][a-z0-9_]*)(?=\s*(?:\.|\[|=[^=]|$))/gm)) {
1792
- const name = m[1];
1793
- if (locals.has(name) || KEYWORDS.has(name) || BUILTINS.has(name) || defined.has(name))
1794
- continue;
1795
- if (onlyHistorical(name)) {
1796
- problem(file, `${name} is declared only by superseded ${historical.get(name).join(", ")}`);
1797
- continue;
1798
- }
1799
- if (!useTerm(name))
1800
- problem(file, `${name} is neither a local nor a glossary term`);
1801
- }
1802
- // A glossary claim a procedure relies on counts toward the rule's status: its evidence is the
1803
- // rule's evidence, and while it is (unknown) the rule lists it as an open question and is not
1804
- // established.
1805
- for (const term of usedTerms) {
1806
- const text = glossary.get(term);
1807
- for (const b of text.matchAll(/\[([^\]]*)\]/g))
1808
- for (const x of idsIn(b[1])) {
1809
- if (["FND", "EXP"].includes(kindOf(x)) && !asList(meta.evidence).includes(x))
1810
- problem(file, `relies on ${term}, whose glossary entry cites ${x}; add it to evidence`);
1811
- }
1812
- if (text.includes("(unknown)")) {
1813
- if (!listsOpen(term))
1814
- problem(file, `relies on ${term}, a glossary claim that is (unknown); list it in Open questions`);
1815
- if (meta.status === "established")
1816
- problem(file, `relies on ${term}, a glossary claim that is (unknown), so it cannot be established`);
1817
- }
1818
- }
1819
- }
1820
- // Glossary claims
1821
- for (const [term, text] of glossary) {
1822
- for (const x of idsIn(text))
1823
- if (!entries.has(x))
1824
- problem(glossaryFile(term), `${term} cites ${x}, which does not exist`);
1825
- else if (isSuperseded(x))
1826
- problem(glossaryFile(term), `${term} cites ${x}, which is superseded`);
1827
- }
1828
- // Body references
1829
- for (const [, e] of entries)
1830
- for (const x of idsIn(e.body))
1831
- if (!entries.has(x))
1832
- problem(e.file, `the body names ${x}, which does not exist`);
1833
- // A path into a build's data directories names a file of some build with its exact case. A
1834
- // directory, or a pattern whose last part holds a placeholder such as nn or xxx, is left alone.
1835
- // The data directories are the top-level directories of the build files unless --data-dirs
1836
- // names them. A build path that holds a space is matched whole, longest first, together with any
1837
- // path that follows it, so Dir/With Space/file.ext is read as one path.
1838
- {
1839
- const exact = new Set();
1840
- const folded = new Map();
1841
- const topDirs = new Set();
1842
- for (const files of buildFiles.values())
1843
- for (const f of files) {
1844
- const p = f.path;
1845
- if (typeof p !== "string")
1846
- continue;
1847
- exact.add(p);
1848
- folded.set(p.toLowerCase(), p);
1849
- const parts = p.split("/");
1850
- if (parts.length > 1)
1851
- topDirs.add(parts[0]);
1852
- for (let i = 1; i < parts.length; i++)
1853
- exact.add(parts.slice(0, i).join("/"));
1854
- }
1855
- const dataDirs = dirList(options["data-dirs"], [...topDirs].sort());
1856
- const escapeRe = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
1857
- const tail = String.raw `[A-Za-z0-9_./-]*[A-Za-z0-9]`;
1858
- const spaced = [...exact]
1859
- .filter((p) => p.includes(" ") && dataDirs.includes(p.split("/")[0]))
1860
- .sort((a, b) => b.length - a.length);
1861
- const whole = spaced.length ? `(?:${spaced.map(escapeRe).join("|")})(?:${tail})?|` : "";
1862
- const dataPath = new RegExp(String.raw `(?<!\w)(?:${whole}(?:${dataDirs.map(escapeRe).join("|")})\/${tail})`, "g");
1863
- const dirsFolded = new Set([...exact].map((p) => p.toLowerCase()));
1864
- const checkPaths = (file, text) => {
1865
- for (const m of text.matchAll(dataPath)) {
1866
- const p = m[0];
1867
- if (exact.has(p))
1868
- continue;
1869
- const last = p.split("/").pop();
1870
- if (folded.has(p.toLowerCase()))
1871
- problem(file, `path ${p} is written ${folded.get(p.toLowerCase())} in the build entry`);
1872
- else if (dirsFolded.has(p.toLowerCase()))
1873
- problem(file, `directory ${p} differs in case from the build entry`);
1874
- else if (/\d/.test(last) && !/nn|NN|xx|XX/.test(last))
1875
- problem(file, `path ${p} is not a file of any build`);
1876
- }
1877
- };
1878
- if (dataDirs.length > 0) {
1879
- for (const [, e] of entries)
1880
- if (e.kind !== "BLD")
1881
- checkPaths(e.file, readFileSync(e.file, "utf8"));
1882
- for (const [term, text] of glossary)
1883
- if (glossaryFiles.has(term))
1884
- checkPaths(glossaryFiles.get(term), text);
1885
- }
1886
- }
1887
- // Enumeration names are unique apart from split formats
1888
- for (const [name, fmts] of enumNames) {
1889
- const uniq = [...new Set(fmts)];
1890
- if (uniq.length > 1 &&
1891
- !uniq.every((f) => uniq.every((g) => f === g || asList(entries.get(f).meta.split_with).includes(g))))
1892
- problem(null, `enumeration name ${name} is defined by ${uniq.join(", ")}`);
1893
- }
1894
- // Saves and recordings must be listed in spec/LICENSE
1895
- {
1896
- const licence = existsSync(join(specDir, "LICENSE")) ? readFileSync(join(specDir, "LICENSE"), "utf8") : "";
1897
- if (!licence)
1898
- problem(null, "spec/LICENSE is missing");
1899
- for (const sub of ["saves", "recordings"]) {
1900
- const d = join(specDir, "experiments", sub);
1901
- if (!existsSync(d))
1902
- continue;
1903
- for (const f of readdirSync(d)) {
1904
- if (f === ".gitkeep" || f.endsWith(".patch.json"))
1905
- continue;
1906
- if (!licence.includes(`experiments/${sub}/${f}`))
1907
- problem(join(d, f), "is not listed in spec/LICENSE as covered by neither licence");
1908
- }
1909
- }
1910
- }
1911
- // Every save and recording is named by some experiment.
1912
- {
1913
- const named = new Set();
1914
- for (const e of entries.values())
1915
- if (e.kind === "EXP")
1916
- for (const v of [e.meta.starting_state, e.meta.recording])
1917
- if (typeof v === "string")
1918
- named.add(v);
1919
- for (const sub of ["saves", "recordings"]) {
1920
- const d = join(specDir, "experiments", sub);
1921
- if (existsSync(d))
1922
- for (const f of readdirSync(d))
1923
- if (f !== ".gitkeep" && !named.has(`${sub}/${f}`))
1924
- problem(join(d, f), "is named by no experiment");
1925
- }
1926
- }
1927
- // Every value file belongs to the entry its name gives, in that entry's directory, and is named by it.
1928
- for (const { dir } of Object.values(KINDS)) {
1929
- const d = join(specDir, dir);
1930
- if (!existsSync(d))
1931
- continue;
1932
- for (const name of readdirSync(d)) {
1933
- if (!name.endsWith(".csv"))
1934
- continue;
1935
- const m = /^([A-Z]+-[A-Z0-9]+-\d{3,})\.[A-Za-z0-9_]+\.csv$/.exec(name);
1936
- const owner = m && entries.get(m[1]);
1937
- if (!m || !owner || owner.file !== join(d, `${m[1]}.md`)) {
1938
- problem(join(d, name), "belongs to no entry; a value file is named <ID>.<table>.csv and sits beside its entry");
1939
- continue;
1940
- }
1941
- if (!readText(owner.file).includes(name))
1942
- problem(join(d, name), `is not named by ${m[1]}`);
1943
- }
1944
- }
1945
- // No chain of superseded_by links leads back to where it started.
1946
- for (const [id, e] of entries) {
1947
- const seen = new Set();
1948
- const stack = [...asList(e.meta.superseded_by)];
1949
- while (stack.length) {
1950
- const x = stack.pop();
1951
- if (x === id) {
1952
- problem(e.file, "its superseded_by links lead back to it");
1953
- break;
1954
- }
1955
- if (seen.has(x) || !entries.has(x))
1956
- continue;
1957
- seen.add(x);
1958
- stack.push(...asList(entries.get(x).meta.superseded_by));
1959
- }
1960
- }
1961
- // An entry that relates to a split rule or format relates to every entry of the split, and every
1962
- // build it lists is listed by one of them.
1963
- for (const [id, e] of entries) {
1964
- if (KINDS[e.kind].statuses !== "claim" || e.meta.status === "superseded")
1965
- continue;
1966
- const related = asList(e.meta.related);
1967
- for (const x of related) {
1968
- const other = entries.get(x);
1969
- // A superseded part cannot be related to, so it is left out of the group.
1970
- const group = other ? [x, ...asList(other.meta.split_with).filter((g) => !isSuperseded(g))] : [];
1971
- if (group.length < 2 || group.includes(id))
1972
- continue;
1973
- for (const g of group)
1974
- if (!related.includes(g))
1975
- problem(e.file, `relates to ${x}, which is split with ${g}; add ${g} to related`);
1976
- for (const b of asList(e.meta.builds))
1977
- if (!group.some((g) => asList(entries.get(g)?.meta.builds).includes(b)))
1978
- problem(e.file, `lists ${b}, which no entry of the split ${group.join(", ")} lists`);
1979
- }
1980
- }
1981
- // ---------------------------------------------------------------------------------------------
1982
- // Kaitai compilation
1983
- if (!skipKsy) {
1984
- const ksys = [];
1985
- const fd = join(specDir, "formats");
1986
- if (existsSync(fd))
1987
- for (const f of readdirSync(fd))
1988
- if (f.endsWith(".ksy"))
1989
- ksys.push(join(fd, f));
1990
- for (const k of ksys) {
1991
- const id = basename(k, ".ksy")
1992
- .toUpperCase()
1993
- .replace(/^FMT_([A-Z0-9]+)_(\d+)$/, "FMT-$1-$2");
1994
- if (!entries.has(id))
1995
- problem(k, `belongs to no format entry (${id})`);
1996
- }
1997
- const compiler = findKaitai();
1998
- if (compiler && ksys.length) {
1999
- const out = mkdtempSync(join(tmpdir(), "ksy-check-"));
2000
- const fixed = [...compiler.args, "--target", "python", "--outdir", out, "--import-path", fd];
2001
- try {
2002
- for (const batch of kaitaiBatches([compiler.cmd, ...fixed], ksys)) {
2003
- try {
2004
- runTool(compiler.cmd, [...fixed, ...batch]);
2005
- }
2006
- catch (err) {
2007
- const failure = err;
2008
- problem(null, `Kaitai definitions do not compile:\n${String(failure.stdout ?? "")}${String(failure.stderr ?? "")}`);
2009
- }
2010
- }
2011
- }
2012
- finally {
2013
- rmSync(out, { recursive: true, force: true });
2014
- }
2015
- }
2016
- else if (ksys.length)
2017
- console.warn("warning: no Kaitai Struct compiler found (set KSC or install kaitai-struct-compiler); definitions were not compiled.");
2018
- }
2019
- // cmd.exe takes a command line of at most 8,191 characters, and the compiler's .bat launcher adds
2020
- // its class path to the arguments it is given. On Windows the definitions are compiled in batches
2021
- // whose quoted command line stays under 4,000 characters. Every batch gets the same --import-path,
2022
- // so imports between definitions still resolve.
2023
- function kaitaiBatches(fixed, files) {
2024
- if (process.platform !== "win32")
2025
- return [files];
2026
- const limit = 4000;
2027
- const quoted = (a) => a.length + 3;
2028
- const start = fixed.reduce((n, a) => n + quoted(a), 0);
2029
- const batches = [];
2030
- let batch = [];
2031
- let length = start;
2032
- for (const f of files) {
2033
- if (batch.length > 0 && length + quoted(f) > limit) {
2034
- batches.push(batch);
2035
- batch = [];
2036
- length = start;
2037
- }
2038
- batch.push(f);
2039
- length += quoted(f);
2040
- }
2041
- if (batch.length > 0)
2042
- batches.push(batch);
2043
- return batches;
2044
- }
2045
- function findKaitai() {
2046
- if (process.env.KSC)
2047
- return { cmd: process.env.KSC, args: [] };
2048
- for (const cmd of ["kaitai-struct-compiler", "ksc"]) {
2049
- try {
2050
- runTool(cmd, ["--version"]);
2051
- return { cmd, args: [] };
2052
- }
2053
- catch { }
2054
- }
2055
- return null;
2056
- }
2057
- // On Windows the compiler is a .bat file, which only cmd.exe can run. Node's shell: true joins the
2058
- // arguments without quoting them, so a path with a space would split. This quotes every argument
2059
- // and hands cmd.exe the line as is. A % in an argument would still expand; paths here have none.
2060
- function runTool(cmd, args) {
2061
- if (process.platform !== "win32")
2062
- return execFileSync(cmd, args, { stdio: "pipe" });
2063
- const line = [cmd, ...args].map((a) => `"${a}"`).join(" ");
2064
- // execFileSync hands its options to spawn, which reads windowsVerbatimArguments; the Node types
2065
- // leave it off ExecFileSyncOptions.
2066
- const verbatim = {
2067
- stdio: "pipe",
2068
- windowsVerbatimArguments: true,
2069
- };
2070
- return execFileSync(process.env.ComSpec ?? "cmd.exe", ["/d", "/s", "/c", `"${line}"`], verbatim);
2071
- }
2072
- // ---------------------------------------------------------------------------------------------
2073
- // Splitting generated files
2074
- // A generated file that would pass the line limit becomes a directory of the same name, split by
2075
- // area (BLD-SRC for builds and sources, which have no area), then by kind, then by a block of 100
2076
- // numbers, or by the first character of a build's or source's alias.
2077
- const groupOf = (id) => (["BLD", "SRC"].includes(kindOf(id)) ? "BLD-SRC" : areaOf(id));
2078
- const blockOf = (id) => {
2079
- const kind = kindOf(id);
2080
- if (kind === "BLD" || kind === "SRC")
2081
- return id.charAt(kind.length + 1);
2082
- return String(Math.floor(Number(id.split("-")[2]) / 100) * 100).padStart(3, "0");
2083
- };
2084
- const SPLIT_LEVELS = [groupOf, kindOf, blockOf];
2085
- function orderKeys(level, keys) {
2086
- if (level === 0)
2087
- return [...areas.filter((a) => keys.includes(a)), ...keys.filter((k) => !areas.includes(k)).sort()];
2088
- if (level === 1)
2089
- return Object.keys(KINDS).filter((k) => keys.includes(k));
2090
- return keys.sort((a, b) => (/^\d+$/.test(a) && /^\d+$/.test(b) ? Number(a) - Number(b) : a < b ? -1 : a > b ? 1 : 0));
2091
- }
2092
- // Lays out one generated file: `${name}.md` when render's text fits, otherwise a directory split only
2093
- // as far as the limit requires. render(ids, path) gives the text of the file at path, which has no
2094
- // .md and is relative to the generated tree. Returns a Map of path -> { ids, text }.
2095
- function layout(name, ids, render, level = 0, out = new Map()) {
2096
- const text = render(ids, name);
2097
- if (lineCount(text) <= LINE_LIMIT || level === SPLIT_LEVELS.length) {
2098
- if (lineCount(text) > LINE_LIMIT)
2099
- problem(null, `${name}.md would pass ${LINE_LIMIT} lines even split by block`);
2100
- out.set(name, { ids, text });
2101
- return out;
2102
- }
2103
- const groups = new Map();
2104
- for (const id of ids) {
2105
- const key = SPLIT_LEVELS[level](id);
2106
- if (!groups.has(key))
2107
- groups.set(key, []);
2108
- groups.get(key).push(id);
2109
- }
2110
- for (const key of orderKeys(level, [...groups.keys()]))
2111
- layout(`${name}/${key}`, groups.get(key), render, level + 1, out);
2112
- return out;
2113
- }
2114
- const GENERATED = "<!-- Generated by the documentation standard check. Do not edit. -->";
2115
- const toSlash = (p) => p.replaceAll("\\", "/");
2116
- // Markdown files under dir, by path relative to dir without .md.
2117
- function markdownTree(dir) {
2118
- const found = new Map();
2119
- walk(dir, (f) => {
2120
- if (f.endsWith(".md"))
2121
- found.set(toSlash(relative(dir, f)).replace(/\.md$/, ""), f);
2122
- });
2123
- return found;
2124
- }
2125
- // ---------------------------------------------------------------------------------------------
2126
- // Deviation log and parity matrix
2127
- const deviations = new Map();
2128
- const devDir = join(repoDir, "deviations");
2129
- const isDeviationFile = (f) => resolve(f).startsWith(devDir + sep);
2130
- {
2131
- if (existsSync(join(repoDir, "DEVIATIONS.md")))
2132
- problem(join(repoDir, "DEVIATIONS.md"), "the deviation log is the directory deviations/; move each ## deviation to deviations/<ID>.md with the ID as its # heading");
2133
- if (!existsSync(devDir))
2134
- problem(null, "deviations/ is missing");
2135
- else
2136
- for (const file of termFiles(devDir)) {
2137
- if (statSync(file).isDirectory() || !file.endsWith(".md")) {
2138
- problem(file, "is not a deviation file; deviations/ holds one <ID>.md per deviation");
2139
- continue;
2140
- }
2141
- const m = /^# (.+)\n?([\s\S]*)$/.exec(readText(file));
2142
- if (!m) {
2143
- problem(file, "opens with the deviation's ID as a # heading");
2144
- continue;
2145
- }
2146
- const title = m[1].trim();
2147
- if (!/^DEV-[A-Z][A-Z0-9]*-\d{3,}$/.test(title)) {
2148
- problem(file, `heading ${title} is not a deviation ID`);
2149
- continue;
2150
- }
2151
- if (basename(file) !== `${title}.md`)
2152
- problem(file, `file name must be ${title}.md`);
2153
- if (deviations.has(title))
2154
- problem(file, `${title} is used twice`);
2155
- if (!areas.includes(areaOf(title)))
2156
- problem(file, `${title}: area is not in the area list`);
2157
- const items = [...m[2].matchAll(/^- ([A-Za-z ]+): (.*)$/gm)].map((x) => [x[1], x[2]]);
2158
- const item = Object.fromEntries(items);
2159
- const order = [
2160
- "Departs from",
2161
- "Reason",
2162
- "Setting",
2163
- "Default",
2164
- ...("Justification" in item ? ["Justification"] : []),
2165
- "Dropped",
2166
- ];
2167
- if (items
2168
- .slice(0, order.length)
2169
- .map((x) => x[0])
2170
- .join("|") !== order.join("|"))
2171
- problem(file, `${title}: items must be ${order.join(", ")} in that order`);
2172
- const departs = idsIn(item["Departs from"]);
2173
- const dropped = Boolean(item.Dropped) && item.Dropped !== "no";
2174
- if (dropped && !/^\d{4}-\d{2}-\d{2}\b/.test(item.Dropped))
2175
- problem(file, `${title}: Dropped gives the date, YYYY-MM-DD, and the reason`);
2176
- checkResolves(file, departs, `${title} Departs from`);
2177
- if (!dropped) {
2178
- if (!departs.some((x) => ["RULE", "FMT", "SCR"].includes(kindOf(x))))
2179
- problem(file, `${title}: Departs from names at least one rule, format or screen`);
2180
- for (const x of departs)
2181
- if (isSuperseded(x))
2182
- problem(file, `${title} departs from ${x}, which is superseded`);
2183
- checkDeviationDefault(file, title, item, departs);
2184
- }
2185
- deviations.set(title, { departs, dropped, file });
2186
- }
2187
- }
2188
- // Default is off, on or mandatory. Only the fix of an unintended, not-relied-on bug is on by right;
2189
- // mandatory, and on for anything else, carry a Justification that the rebuild is strictly better or a
2190
- // small judgement call that makes the game better to play.
2191
- function checkDeviationDefault(path, title, item, departs) {
2192
- const defaults = ["off", "on", "mandatory"];
2193
- const dflt = item.Default;
2194
- if (!defaults.includes(dflt))
2195
- return problem(path, `${title}: Default is one of ${defaults.join(", ")}`);
2196
- if ((item.Setting === "None") !== (dflt === "mandatory"))
2197
- return problem(path, `${title}: Default is mandatory exactly when Setting is None`);
2198
- const bugs = departs
2199
- .filter((x) => kindOf(x) === "BUG")
2200
- .map((x) => entries.get(x))
2201
- .filter((b) => Boolean(b));
2202
- const bugFix = bugs.length > 0 && bugs.every((b) => b.meta.intent === "unintended" && b.meta.player_reliance === "not-relied-on");
2203
- if (bugFix && dflt === "off")
2204
- problem(path, `${title}: Default is on or mandatory for the fix of an unintended, not-relied-on bug`);
2205
- const needsJustification = dflt === "mandatory" || (dflt === "on" && !bugFix);
2206
- const hasJustification = "Justification" in item;
2207
- if (needsJustification && !hasJustification)
2208
- problem(path, `${title}: is ${dflt} but has no Justification saying why the rebuild's behaviour is strictly better, or what the judgement call improves`);
2209
- if (!needsJustification && hasJustification)
2210
- problem(path, `${title}: has a Justification, which only a mandatory deviation or one that is on without fixing an unintended, not-relied-on bug has`);
2211
- }
2212
- // The parity rows live in parity/, one file per area, split by kind and then by block where the
2213
- // limit requires it. PARITY.md holds the totals and is written by this script.
2214
- const PARITY_HEADER = ["Spec ID", "Title", "Spec status", "Code", "Tests", "Deviations", "Status", "Notes"];
2215
- const parityDir = join(repoDir, "parity");
2216
- const parityRows = new Map(); // spec ID -> { cells, file }
2217
- const parityCounts = { status: {}, code: {} };
2218
- const validatedTests = new Map(); // marked test file of a validated row -> [{ specId, file }]
2219
- // A test file that reads the original's files through GAME_DIR says so with this comment. It runs
2220
- // only on a maintainer's machine, so its validated rows need it in VALIDATION.md; every other test
2221
- // runs in CI.
2222
- const NEEDS_GAME = /needs:\s*GAME_DIR/;
2223
- const needsGame = (p) => existsSync(p) && NEEDS_GAME.test(readFileSync(p, "utf8"));
2224
- // A PARITY.md that still holds the rows is left alone until they have moved, so the check does not
2225
- // overwrite them with the totals.
2226
- const legacyParity = existsSync(join(repoDir, "PARITY.md")) &&
2227
- tables(readText(join(repoDir, "PARITY.md"))).some((t) => t.header.join("|") === PARITY_HEADER.join("|"));
2228
- {
2229
- if (!existsSync(parityDir))
2230
- problem(null, "parity/ is missing");
2231
- if (legacyParity)
2232
- problem(join(repoDir, "PARITY.md"), "the rows move to parity/, one <AREA>.md per area, and the check writes PARITY.md");
2233
- const placeholders = collectPlaceholders();
2234
- const files = existsSync(parityDir) ? markdownTree(parityDir) : new Map();
2235
- walk(parityDir, (f) => {
2236
- if (!f.endsWith(".md") && basename(f) !== ".gitkeep")
2237
- problem(f, "is not a parity file; parity/ holds one <AREA>.md per area");
2238
- });
2239
- const byArea = new Map(); // area -> Map of path -> ids in the file
2240
- for (const [path, file] of files) {
2241
- const text = readText(file);
2242
- if (!text.startsWith(`# ${path}\n`))
2243
- problem(file, `opens with its path as a # heading: # ${path}`);
2244
- const [area, kind, block, ...rest] = path.split("/");
2245
- if (!areas.includes(area)) {
2246
- problem(file, `${area} is not an area in the area list`);
2247
- continue;
2248
- }
2249
- if (rest.length > 0 || (kind !== undefined && !["RULE", "FMT", "SCR"].includes(kind))) {
2250
- problem(file, "is not an area, kind or block file of parity/");
2251
- continue;
2252
- }
2253
- const ts = tables(text);
2254
- if (ts.length !== 1 || ts[0].header.join("|") !== PARITY_HEADER.join("|")) {
2255
- problem(file, `holds one table with the columns ${PARITY_HEADER.join(" | ")}`);
2256
- continue;
2257
- }
2258
- if (!byArea.has(area))
2259
- byArea.set(area, new Map());
2260
- const inFile = [];
2261
- byArea.get(area).set(path, inFile);
2262
- let previous = "";
2263
- for (const row of ts[0].rows) {
2264
- if (row.length !== PARITY_HEADER.length) {
2265
- problem(file, `the row ${row.join(" | ")} has ${row.length} cells, not ${PARITY_HEADER.length}`);
2266
- continue;
2267
- }
2268
- const cells = row.map((c) => c.replaceAll("`", "").trim());
2269
- const [specId, title, specStatus, code, tests, devs, status, notes] = cells;
2270
- if (parityRows.has(specId))
2271
- problem(file, `${specId} has more than one row`);
2272
- parityRows.set(specId, { cells: row, file });
2273
- inFile.push(specId);
2274
- if (areaOf(specId) !== area || (kind && kindOf(specId) !== kind) || (block && blockOf(specId) !== block))
2275
- problem(file, `${specId} does not belong in parity/${path}.md`);
2276
- if (compareIds(specId, previous) < 0)
2277
- problem(file, `${specId} is out of ID order`);
2278
- previous = specId;
2279
- const e = entries.get(specId);
2280
- if (!e) {
2281
- problem(file, `${specId} does not exist in the spec`);
2282
- continue;
2283
- }
2284
- if (!["RULE", "FMT", "SCR"].includes(e.kind) || e.meta.status === "superseded")
2285
- problem(file, `${specId} cannot have a row`);
2286
- if (title !== e.meta.title)
2287
- problem(file, `${specId}: Title must be "${e.meta.title}"`);
2288
- if (specStatus !== e.meta.status)
2289
- problem(file, `${specId}: Spec status must be ${e.meta.status}`);
2290
- if (!["missing", "partial", "complete"].includes(code))
2291
- problem(file, `${specId}: Code must be missing, partial or complete`);
2292
- if (code === "complete" && e.meta.status === "unknown")
2293
- problem(file, `${specId}: an unknown entry cannot be complete`);
2294
- if (code === "complete" && placeholders.has(specId))
2295
- problem(file, `${specId}: a PLACEHOLDER comment cites it, so it cannot be complete`);
2296
- const testFiles = tests === "None"
2297
- ? []
2298
- : tests
2299
- .split(",")
2300
- .map((x) => x.trim())
2301
- .filter(Boolean);
2302
- for (const tf of testFiles) {
2303
- const p = join(repoDir, tf);
2304
- if (!existsSync(p))
2305
- problem(file, `${specId}: test file ${tf} does not exist`);
2306
- else {
2307
- const text = readFileSync(p, "utf8");
2308
- if (!text.includes(specId))
2309
- problem(file, `${specId}: test file ${tf} does not mention ${specId}`);
2310
- if (text.includes("GAME_DIR") && !NEEDS_GAME.test(text))
2311
- problem(file, `${specId}: test file ${tf} mentions GAME_DIR without a "needs: GAME_DIR" comment, so CI would skip it unseen`);
2312
- }
2313
- }
2314
- const listedDevs = devs === "None"
2315
- ? []
2316
- : devs
2317
- .split(",")
2318
- .map((x) => x.trim())
2319
- .filter(Boolean);
2320
- const expectedDevs = [...deviations]
2321
- .filter(([, d]) => !d.dropped && d.departs.includes(specId))
2322
- .map(([k]) => k)
2323
- .sort(compareIds);
2324
- if (listedDevs.slice().sort(compareIds).join(",") !== expectedDevs.join(","))
2325
- problem(file, `${specId}: Deviations must be ${expectedDevs.join(", ") || "None"}`);
2326
- let expectedStatus;
2327
- if (code !== "complete" || e.meta.status === "disputed")
2328
- expectedStatus = e.meta.status;
2329
- else if (testFiles.length === 0)
2330
- expectedStatus = "implemented";
2331
- else if (["supported", "established"].includes(e.meta.status))
2332
- expectedStatus = "validated";
2333
- else {
2334
- problem(file, `${specId}: complete with tests while the spec status is ${e.meta.status}; the evidence belongs in the spec entry first`);
2335
- expectedStatus = status;
2336
- }
2337
- if (status !== expectedStatus)
2338
- problem(file, `${specId}: Status must be ${expectedStatus}`);
2339
- if (expectedStatus === "validated")
2340
- for (const tf of testFiles.filter((x) => needsGame(join(repoDir, x)))) {
2341
- if (!validatedTests.has(tf))
2342
- validatedTests.set(tf, []);
2343
- validatedTests.get(tf).push({ specId, file });
2344
- }
2345
- if (expectedStatus === "validated" && mayBeInterrupted(e) && onlyEmulatedRuns(e))
2346
- problem(file, `${specId}: another rule may interrupt it (# may run:), so tests against emulated calls alone cannot validate it`);
2347
- for (const cell of [code, tests, devs, notes])
2348
- if (cell === "")
2349
- problem(file, `${specId}: an empty cell says None`);
2350
- parityCounts.status[status] = (parityCounts.status[status] ?? 0) + 1;
2351
- parityCounts.code[code] = (parityCounts.code[code] ?? 0) + 1;
2352
- }
2353
- }
2354
- for (const [id, e] of entries)
2355
- if (["RULE", "FMT", "SCR"].includes(e.kind) && e.meta.status !== "superseded" && !parityRows.has(id))
2356
- problem(parityDir, `${id} has no row`);
2357
- // Each area is split exactly where the limit requires it, going by the rows it has.
2358
- for (const [area, actual] of byArea) {
2359
- const ids = [...actual.values()]
2360
- .flat()
2361
- .filter((x) => entries.has(x))
2362
- .sort(compareIds);
2363
- const render = (subset, path) => [
2364
- `# ${path}`,
2365
- "",
2366
- `| ${PARITY_HEADER.join(" | ")} |`,
2367
- `|${"---|".repeat(PARITY_HEADER.length)}`,
2368
- ...subset.map((x) => `| ${parityRows.get(x).cells.join(" | ")} |`),
2369
- "",
2370
- ].join("\n");
2371
- const expected = [...layout(area, ids, render, 1).keys()];
2372
- const found = [...actual.keys()].sort();
2373
- if (expected.slice().sort().join(",") !== found.join(","))
2374
- problem(parityDir, `the rows of ${area} belong in ${expected.map((p) => `parity/${p}.md`).join(", ")}, split only where the ${LINE_LIMIT}-line limit requires it; found ${found.map((p) => `parity/${p}.md`).join(", ")}`);
2375
- }
2376
- }
2377
- for (const [dev, d] of deviations)
2378
- if (!d.dropped && !d.departs.some((x) => parityRows.has(x)))
2379
- problem(d.file, `${dev} departs from no entry that has a parity row`);
2380
- // VALIDATION.md records the marked test files of the validated rows as they were when a maintainer ran
2381
- // them against the original's files, which CI never holds. A file is hashed with CRLF read as LF,
2382
- // so a Windows checkout and a Linux one give the same hash.
2383
- const validationPath = join(repoDir, "VALIDATION.md");
2384
- const VALIDATION_HEADER = ["Test file", "SHA-256"];
2385
- const testHash = (p) => createHash("sha256")
2386
- .update(Buffer.from(readFileSync(p).toString("latin1").replaceAll("\r\n", "\n"), "latin1"))
2387
- .digest("hex");
2388
- if (options["record-validation"] !== undefined) {
2389
- const builds = dirList(options["record-validation"], []);
2390
- if (!builds.length) {
2391
- console.error("--record-validation needs at least one build ID");
2392
- process.exit(2);
2393
- }
2394
- for (const b of builds)
2395
- if (entries.get(b)?.kind !== "BLD") {
2396
- console.error(`--record-validation: ${b} is not a build entry`);
2397
- process.exit(2);
2398
- }
2399
- let commit;
2400
- try {
2401
- commit = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repoDir, encoding: "utf8" }).trim();
2402
- }
2403
- catch {
2404
- console.error("--record-validation: git rev-parse HEAD failed");
2405
- process.exit(2);
2406
- }
2407
- const files = [...validatedTests.keys()].sort();
2408
- if (!files.length) {
2409
- console.error('--record-validation: no validated row lists a test file with a "needs: GAME_DIR" comment, so there is nothing to record');
2410
- process.exit(2);
2411
- }
2412
- writeFileSync(validationPath, [
2413
- "# Validation record",
2414
- "",
2415
- "The test files of the validated parity rows that read the original's files, as they were when every test in them passed against those files.",
2416
- "",
2417
- `- Commit: ${commit}`,
2418
- `- Date: ${new Date().toISOString().slice(0, 10)}`,
2419
- `- Builds: ${builds.join(", ")}`,
2420
- "",
2421
- `| ${VALIDATION_HEADER.join(" | ")} |`,
2422
- `|${"---|".repeat(VALIDATION_HEADER.length)}`,
2423
- ...files.map((f) => `| \`${f}\` | \`${testHash(join(repoDir, f))}\` |`),
2424
- "",
2425
- ].join("\n"));
2426
- console.log("wrote VALIDATION.md");
2427
- }
2428
- {
2429
- const recorded = new Map(); // test file -> hash
2430
- if (existsSync(validationPath)) {
2431
- const text = readText(validationPath);
2432
- const item = (key, pattern) => {
2433
- const m = text.match(new RegExp(`^- ${key}: (.*)$`, "m"));
2434
- if (!m || !pattern.test(m[1].trim())) {
2435
- problem(validationPath, `needs a "- ${key}:" item in the form the standard gives`);
2436
- return null;
2437
- }
2438
- return m[1].trim();
2439
- };
2440
- item("Commit", /^[0-9a-f]{40}$/);
2441
- item("Date", /^\d{4}-\d{2}-\d{2}$/);
2442
- const builds = item("Builds", /^\S.*$/);
2443
- if (builds !== null)
2444
- for (const b of builds.split(",").map((x) => x.trim()))
2445
- if (entries.get(b)?.kind !== "BLD")
2446
- problem(validationPath, `Builds names ${b}, which is not a build entry`);
2447
- const ts = tables(text);
2448
- if (ts.length !== 1 || ts[0].header.join("|") !== VALIDATION_HEADER.join("|"))
2449
- problem(validationPath, `holds one table with the columns ${VALIDATION_HEADER.join(" | ")}`);
2450
- else {
2451
- let previous = "";
2452
- for (const row of ts[0].rows) {
2453
- const [path, hash] = row.map((c) => c.replaceAll("`", "").trim());
2454
- if (row.length !== VALIDATION_HEADER.length || !/^[0-9a-f]{64}$/.test(hash ?? "")) {
2455
- problem(validationPath, `the row ${row.join(" | ")} needs a test file and its SHA-256 in lowercase hex`);
2456
- continue;
2457
- }
2458
- if (recorded.has(path))
2459
- problem(validationPath, `${path} is listed twice`);
2460
- if (path < previous)
2461
- problem(validationPath, `${path} is out of order; the files are sorted by path`);
2462
- previous = path;
2463
- recorded.set(path, hash);
2464
- if (!validatedTests.has(path))
2465
- problem(validationPath, `${path} is not a test file with a "needs: GAME_DIR" comment in a validated row's Tests; run the check with --record-validation again`);
2466
- }
2467
- }
2468
- }
2469
- for (const [tf, rows] of validatedTests) {
2470
- const p = join(repoDir, tf);
2471
- if (!existsSync(p))
2472
- continue;
2473
- const hash = recorded.get(tf);
2474
- for (const { specId, file } of rows) {
2475
- if (hash === undefined)
2476
- problem(file, `${specId}: ${tf} is not in VALIDATION.md, so the row cannot be validated until its tests pass against the original's files and are recorded`);
2477
- else if (hash !== testHash(p))
2478
- problem(file, `${specId}: ${tf} has changed since VALIDATION.md recorded it; run its tests against the original's files and record them again`);
2479
- }
2480
- }
2481
- }
2482
- // The --code and --references directories, walked and read once for the placeholder and the
2483
- // implementation-reference checks. The cache is declared with the other module state at the top,
2484
- // because the parity check calls this before a variable declared here would be initialized.
2485
- function codeFiles() {
2486
- if (codeFilesCache)
2487
- return codeFilesCache;
2488
- const files = [];
2489
- const roots = [
2490
- ...codeRoots.map((root) => ({ root, code: true })),
2491
- ...referenceRoots.map((root) => ({ root, code: false })),
2492
- ];
2493
- for (const { root, code } of roots)
2494
- walk(join(repoDir, root), (f) => {
2495
- if (f !== selfPath && /\.(cs|ts|mjs|js|ps1|fs|md|json)$/.test(f))
2496
- files.push({ code, file: f, text: readFileSync(f, "utf8") });
2497
- });
2498
- return (codeFilesCache = files);
2499
- }
2500
- function collectPlaceholders() {
2501
- const found = new Set();
2502
- for (const { code, file, text } of codeFiles()) {
2503
- if (!code || !/\.(cs|ts|mjs|js|ps1|fs)$/.test(file))
2504
- continue;
2505
- for (const m of text.matchAll(/PLACEHOLDER:\s*((?:FMT|RULE|SCR)-[A-Z0-9]+-\d+)/g))
2506
- found.add(m[1]);
2507
- }
2508
- return found;
2509
- }
2510
- function walk(dir, fn) {
2511
- if (!existsSync(dir))
2512
- return;
2513
- for (const name of readdirSync(dir)) {
2514
- if (["bin", "obj", "node_modules", ".git", "artifacts"].includes(name))
2515
- continue;
2516
- const p = join(dir, name);
2517
- if (statSync(p).isDirectory())
2518
- walk(p, fn);
2519
- else
2520
- fn(p);
2521
- }
2522
- }
2523
- // Implementation references: every spec and deviation ID in code, tests, the parity files and the
2524
- // deviation files resolves. A deviation keeps citing what it departed from after that is superseded.
2525
- {
2526
- const scan = codeFiles().filter(({ file }) => !file.endsWith(".fs"));
2527
- for (const dir of [parityDir, devDir])
2528
- walk(dir, (f) => {
2529
- if (f.endsWith(".md"))
2530
- scan.push({ file: f, text: readFileSync(f, "utf8") });
2531
- });
2532
- for (const { file: f, text } of scan) {
2533
- for (const x of idsIn(text)) {
2534
- if (["BLD", "SRC"].includes(kindOf(x)) && !entries.has(x))
2535
- continue; // aliases can collide with ordinary words
2536
- if (!entries.has(x))
2537
- problem(f, `cites ${x}, which does not exist in the spec`);
2538
- else if (isSuperseded(x) && !isDeviationFile(f))
2539
- problem(f, `cites ${x}, which is superseded; cite what replaced it`);
2540
- }
2541
- if (!isDeviationFile(f))
2542
- for (const x of new Set(text.match(DEV_RE) ?? []))
2543
- if (!deviations.has(x))
2544
- problem(f, `cites ${x}, which is not in deviations/`);
2545
- }
2546
- }
2547
- // Addresses in code comments: an address of the original that a comment in the code gives,
2548
- // written 0x… or as a neutral name (fn_…, g_…), is recorded in an entry the comment cites, or in an
2549
- // entry that one of those cites as evidence. Evidence lives in the spec, so a comment that relies on
2550
- // an address cites the finding that shows it; citing an ID only proves that the ID exists. A
2551
- // superseded entry records nothing.
2552
- //
2553
- // Comments are found by reading .cs, .ts, .js and .mjs files as code, so `//` inside a string is
2554
- // not a comment and `/* … */` is. A comment block is a run of consecutive lines that hold only
2555
- // comment; a comment that trails code also takes the block above it and the comment lines below
2556
- // it that start in the same column, and a comment-only line among those finds the same block. An
2557
- // entry records an address written in its locations or its text, singly or inside a half-open
2558
- // range, in either case. A range of more than --max-range bytes describes a section or a whole
2559
- // table, not a place, and records only its two ends, nothing inside it: it would otherwise vouch
2560
- // for every address in the program on behalf of each entry that cites it.
2561
- //
2562
- // A neutral name is always an address. A plain 0x value is one only inside an image that --images
2563
- // gives, so colours, masks and offsets in the same notation are left alone; without --images only
2564
- // neutral names are checked. Only flat 32-bit addresses are read: a segmented address (MZ, NE) is
2565
- // not checked.
2566
- {
2567
- const ADDRESS_RE = /(?<![0-9A-Za-z_])(0x|fn_|g_)([0-9A-Fa-f]{8})(?![0-9A-Za-z_])/g;
2568
- const RANGE_RE = /(?<![0-9A-Za-z_])(?:0x|fn_|g_)([0-9A-Fa-f]{8})(?:\.\.0x([0-9A-Fa-f]{8}))?(?![0-9A-Za-z_])/g;
2569
- const inImage = (value) => images.some(([low, high]) => value >= low && value < high);
2570
- const recorded = new Map(); // entry ID -> half-open [low, high) ranges it records
2571
- const rangesOf = (id) => {
2572
- if (recorded.has(id))
2573
- return recorded.get(id);
2574
- const e = entries.get(id);
2575
- const ranges = [];
2576
- if (e && !isSuperseded(id)) {
2577
- const text = [
2578
- e.body,
2579
- ...asList(e.meta.locations).map((loc) => loc && typeof loc === "object" && "address" in loc ? String(loc.address) : ""),
2580
- ].join("\n");
2581
- for (const [, low, high] of text.matchAll(RANGE_RE)) {
2582
- const range = high === undefined ? [parseInt(low, 16), parseInt(low, 16) + 1] : [parseInt(low, 16), parseInt(high, 16)];
2583
- if (range[1] <= range[0])
2584
- continue;
2585
- // A larger range records only its two ends, which the entry writes out.
2586
- if (range[1] - range[0] <= maxRange)
2587
- ranges.push(range);
2588
- else
2589
- ranges.push([range[0], range[0] + 1], [range[1] - 1, range[1]]);
2590
- }
2591
- }
2592
- recorded.set(id, ranges);
2593
- return ranges;
2594
- };
2595
- const reach = (ids) => [
2596
- ...new Set([...ids, ...ids.flatMap((id) => asList(entries.get(id)?.meta.evidence).map(String))]),
2597
- ];
2598
- for (const { file, text } of codeFiles()) {
2599
- if (!/\.(cs|ts|js|mjs)$/.test(file))
2600
- continue;
2601
- const lines = codeComments(text, !file.endsWith(".cs"));
2602
- const commentOnly = (k) => k >= 0 && k < lines.length && !lines[k].code && lines[k].comments.length > 0;
2603
- // For each comment-only line that continues the comment trailing code on a line above (the
2604
- // rest of a /* … */ begun there, or a comment line starting in its column), that line.
2605
- const trails = [];
2606
- for (let k = 0; k < lines.length; k++) {
2607
- const t = k === 0 ? -1 : lines[k - 1].code ? (lines[k - 1].comments.length ? k - 1 : -1) : trails[k - 1];
2608
- const head = lines[k].comments[0];
2609
- trails[k] =
2610
- commentOnly(k) && t >= 0 && (head.continued || head.column === lines[t].comments.at(-1).column) ? t : -1;
2611
- }
2612
- const blocks = new Map(); // "first,last" -> the entries the block cites, and those within reach
2613
- for (let i = 0; i < lines.length; i++) {
2614
- const own = lines[i].comments.map((c) => c.text).join("\n");
2615
- const addresses = new Map();
2616
- for (const m of own.matchAll(ADDRESS_RE)) {
2617
- // The end of a half-open range is one byte past the last address it covers.
2618
- const rangeEnd = m.index >= 2 && own.slice(m.index - 2, m.index) === "..";
2619
- const value = parseInt(m[2], 16) - (rangeEnd ? 1 : 0);
2620
- if (m[1] === "0x" && !inImage(value))
2621
- continue;
2622
- addresses.set(`${m[0]}@${value}`, [m[0], value]);
2623
- }
2624
- if (!addresses.size)
2625
- continue;
2626
- // A line with code, or one continuing the comment that trails it, belongs to that comment.
2627
- const t = lines[i].code ? i : trails[i];
2628
- let first = t >= 0 ? t : i, last = first;
2629
- while (first > 0 && (commentOnly(first - 1) || lines[first].comments[0]?.continued))
2630
- first--;
2631
- while (t >= 0 ? trails[last + 1] === t : commentOnly(last + 1))
2632
- last++;
2633
- const key = `${first},${last}`;
2634
- if (!blocks.has(key)) {
2635
- const block = lines
2636
- .slice(first, last + 1)
2637
- .flatMap((l) => l.comments.map((c) => c.text))
2638
- .join("\n");
2639
- const cited = idsIn(block).filter((x) => entries.has(x));
2640
- blocks.set(key, { cited, scope: reach(cited) });
2641
- }
2642
- const { cited, scope } = blocks.get(key);
2643
- for (const [address, value] of addresses.values()) {
2644
- if (scope.some((x) => rangesOf(x).some(([low, high]) => value >= low && value < high)))
2645
- continue;
2646
- problem(file, `line ${i + 1} gives ${address}, but ${cited.length ? `neither ${cited.join(", ")} nor the evidence ${cited.length === 1 ? "it cites" : "they cite"} records it` : "the comment cites no entry that records it"}; cite the finding that records it, or record it in a new one`);
2647
- }
2648
- }
2649
- }
2650
- }
2651
- // The comments of a C-family source file (C#, TypeScript, JavaScript), line by line: for each
2652
- // line, whether it holds code and the comment text on it with the column where each piece starts.
2653
- // String and character literals are skipped (regular, verbatim, interpolated and raw in C#;
2654
- // template literals in JavaScript), so a `//` inside one starts no comment. An interpolation hole is
2655
- // read as part of its string, which is enough to find comments. A JavaScript regular expression
2656
- // literal is skipped too, so a quote or a `/*` inside one starts nothing; a `/` is read as one
2657
- // where a value can begin. The lines of a `/* … */` after its first are marked continued.
2658
- function codeComments(source, javascript) {
2659
- const text = source.replace(/\r\n?/g, "\n");
2660
- const lines = [{ code: false, comments: [] }];
2661
- const line = () => lines[lines.length - 1];
2662
- let i = 0, column = 0;
2663
- const advance = (to) => {
2664
- for (; i < to; i++) {
2665
- if (text[i] === "\n") {
2666
- lines.push({ code: false, comments: [] });
2667
- column = 0;
2668
- }
2669
- else
2670
- column++;
2671
- }
2672
- };
2673
- const addComment = (from, to, col, continued = false) => line().comments.push({ column: col, text: text.slice(from, to), continued });
2674
- let lastCode = -1; // the index of the last character read as code
2675
- // A JavaScript `/` starts a regular expression unless it follows a value, where it divides.
2676
- const regexMayStart = () => {
2677
- const before = text.slice(Math.max(0, lastCode - 11), lastCode + 1);
2678
- return (lastCode < 0 ||
2679
- !/[\w$)\]}]$/.test(before) ||
2680
- /(?<![\w$])(?:return|typeof|case|do|else|in|of|new|delete|void|throw|instanceof|yield|await)$/.test(before));
2681
- };
2682
- // Past the closing `/` of the regular expression literal at i, if it closes on its line.
2683
- const regexEnd = () => {
2684
- let inClass = false;
2685
- for (let j = i + 1; j < text.length && text[j] !== "\n"; j++) {
2686
- if (text[j] === "\\") {
2687
- if (text[j + 1] === "\n")
2688
- return -1;
2689
- j++;
2690
- }
2691
- else if (text[j] === "[")
2692
- inClass = true;
2693
- else if (text[j] === "]")
2694
- inClass = false;
2695
- else if (text[j] === "/" && !inClass)
2696
- return j + 1;
2697
- }
2698
- return -1;
2699
- };
2700
- // i is past the opening quote(s); stops past the closing one(s). A regular string ends at the
2701
- // line when it is not closed.
2702
- const skipString = (end, escapes, multiline) => {
2703
- while (i < text.length) {
2704
- if (escapes && text[i] === "\\") {
2705
- advance(i + 2);
2706
- continue;
2707
- }
2708
- if (text.startsWith(end, i)) {
2709
- if (!escapes && end === '"' && text[i + 1] === '"') {
2710
- advance(i + 2);
2711
- continue;
2712
- } // "" in a verbatim string
2713
- advance(i + end.length);
2714
- return;
2715
- }
2716
- if (text[i] === "\n" && !multiline) {
2717
- advance(i + 1);
2718
- return;
2719
- }
2720
- advance(i + 1);
2721
- }
2722
- };
2723
- while (i < text.length) {
2724
- const c = text[i];
2725
- if (c === "\n" || c === " " || c === "\t") {
2726
- advance(i + 1);
2727
- continue;
2728
- }
2729
- if (text.startsWith("//", i)) {
2730
- const nl = text.indexOf("\n", i);
2731
- const end = nl < 0 ? text.length : nl;
2732
- addComment(i, end, column);
2733
- advance(end);
2734
- continue;
2735
- }
2736
- if (text.startsWith("/*", i)) {
2737
- const close = text.indexOf("*/", i + 2);
2738
- const end = close < 0 ? text.length : close + 2;
2739
- let from = i, col = column, continued = false;
2740
- while (i < end) {
2741
- const nl = text.indexOf("\n", i);
2742
- if (nl < 0 || nl >= end) {
2743
- addComment(from, end, col, continued);
2744
- advance(end);
2745
- break;
2746
- }
2747
- addComment(from, nl, col, continued);
2748
- advance(nl + 1);
2749
- continued = true;
2750
- while (i < end && (text[i] === " " || text[i] === "\t"))
2751
- advance(i + 1);
2752
- from = i;
2753
- col = column;
2754
- }
2755
- continue;
2756
- }
2757
- line().code = true;
2758
- if (javascript) {
2759
- const regex = c === "/" && regexMayStart() ? regexEnd() : -1;
2760
- if (regex >= 0)
2761
- advance(regex);
2762
- else if (c === '"' || c === "'") {
2763
- advance(i + 1);
2764
- skipString(c, true, false);
2765
- }
2766
- else if (c === "`") {
2767
- advance(i + 1);
2768
- skipString("`", true, true);
2769
- }
2770
- else
2771
- advance(i + 1);
2772
- lastCode = i - 1;
2773
- continue;
2774
- }
2775
- const prefix = /^(?:\$+@?|@\$*)?(?=")/.exec(text.slice(i, i + 4))?.[0] ?? null;
2776
- if (prefix !== null) {
2777
- advance(i + prefix.length);
2778
- const quotes = /^"{3,}/.exec(text.slice(i, i + 64))?.[0];
2779
- if (quotes) {
2780
- advance(i + quotes.length);
2781
- skipString(quotes, false, true);
2782
- }
2783
- else {
2784
- const verbatim = prefix.includes("@");
2785
- advance(i + 1);
2786
- skipString('"', !verbatim, verbatim);
2787
- }
2788
- continue;
2789
- }
2790
- if (c === "'") {
2791
- advance(i + 1);
2792
- skipString("'", true, false);
2793
- continue;
2794
- }
2795
- advance(i + 1);
2796
- }
2797
- return lines;
2798
- }
2799
- // IDs, areas and deviations that exist on the base branch must not disappear
2800
- {
2801
- // Without --base, compare with the point this branch left the base branch (the pull request's
2802
- // target in CI), not that branch's tip: an entry added on the base branch after this branch
2803
- // forked is not one this branch deleted.
2804
- const git = (...args) => execFileSync("git", ["-C", repoDir, ...args], { stdio: ["ignore", "pipe", "ignore"] }).toString();
2805
- let base = baseArg;
2806
- if (!base) {
2807
- const target = process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : "origin/main";
2808
- try {
2809
- base = git("merge-base", "HEAD", target).trim();
2810
- }
2811
- catch {
2812
- base = null;
2813
- }
2814
- }
2815
- let listing = null;
2816
- if (base)
2817
- try {
2818
- listing = git("ls-tree", "-r", "--name-only", base, "--", "spec", "deviations");
2819
- }
2820
- catch {
2821
- if (baseArg)
2822
- problem(null, `cannot list spec/ at ${base}`);
2823
- }
2824
- if (listing) {
2825
- for (const p of listing.split("\n")) {
2826
- const m = /^spec\/(?:builds|sources|formats|rules|findings|experiments|bugs|screens)\/([A-Z]+-[A-Z0-9.-]+)\.md$/.exec(p);
2827
- if (m && !entries.has(m[1]))
2828
- problem(null, `${m[1]} exists at ${base} and has been deleted or renamed`);
2829
- const d = /^deviations\/(DEV-[A-Z0-9]+-\d+)\.md$/.exec(p);
2830
- if (d && !deviations.has(d[1]))
2831
- problem(null, `${d[1]} exists at ${base} and has been deleted or renamed`);
2832
- }
2833
- // ./ makes the path relative to --root, which need not be the top of the repository. A file
2834
- // that does not exist at the base has nothing to compare, and each is read on its own so a
2835
- // missing README does not skip the deviation comparison.
2836
- const show = (path) => {
2837
- try {
2838
- return git("show", `${base}:./${path}`).replace(/\r\n/g, "\n");
2839
- }
2840
- catch {
2841
- return null;
2842
- }
2843
- };
2844
- const oldReadme = show("spec/README.md");
2845
- // Reads the area table the way the current README is read, so areas with or without
2846
- // backticks are both found.
2847
- const oldAreaSection = oldReadme && splitSections(oldReadme).find((s) => s.title === "Areas");
2848
- const oldAreaTable = oldAreaSection && tables(oldAreaSection.text)[0];
2849
- for (const row of (oldAreaTable || undefined)?.rows ?? []) {
2850
- const a = row[0].replaceAll("`", "");
2851
- if (!areas.includes(a))
2852
- problem(null, `area ${a} exists at ${base} and has been removed or renamed`);
2853
- }
2854
- // A base from before the deviation log became a directory keeps its deviations in DEVIATIONS.md.
2855
- const oldDev = show("DEVIATIONS.md");
2856
- if (oldDev)
2857
- for (const m of oldDev.matchAll(/^## (DEV-[A-Z0-9]+-\d+)$/gm))
2858
- if (!deviations.has(m[1]))
2859
- problem(null, `${m[1]} exists at ${base} and has been removed`);
2860
- }
2861
- }
2862
- // ---------------------------------------------------------------------------------------------
2863
- // Indexes and PARITY.md
2864
- const esc = (s) => String(s ?? "").replaceAll("|", "\\|");
2865
- const sortedIds = [...entries.keys()].sort(compareIds);
2866
- const indexDir = join(specDir, "index");
2867
- const linkFrom = (path) => (id) => `[${id}](${toSlash(relative(dirname(join(indexDir, `${path}.md`)), entries.get(id).file))})`;
2868
- const opening = (path, what) => [`# ${path}`, "", GENERATED, "", what, ""];
2869
- // A file of the full index lists every group, empty ones too, so its counts read as a progress
2870
- // report. A file of a split index lists only the groups it has entries in.
2871
- const isWhole = (path) => !path.includes("/");
2872
- function renderByKind(ids, path) {
2873
- const link = linkFrom(path);
2874
- const out = opening(path, "Entries by kind.");
2875
- for (const [kind, { dir }] of Object.entries(KINDS)) {
2876
- const kindIds = ids.filter((x) => kindOf(x) === kind);
2877
- if (!kindIds.length && !isWhole(path))
2878
- continue;
2879
- out.push(`## ${dir}`, "", `${kindIds.length} entries.`, "");
2880
- if (kindIds.length)
2881
- out.push("| ID | Title | Status |", "|---|---|---|", ...kindIds.map((x) => `| ${link(x)} | ${esc(entries.get(x).meta.title)} | ${entries.get(x).meta.status ?? "None"} |`), "");
2882
- }
2883
- return out.join("\n");
2884
- }
2885
- function renderByArea(ids, path) {
2886
- const link = linkFrom(path);
2887
- const out = opening(path, "Entries by area.");
2888
- for (const a of areas) {
2889
- const areaIds = ids.filter((x) => areaOf(x) === a);
2890
- if (!areaIds.length && !isWhole(path))
2891
- continue;
2892
- out.push(`## ${a}`, "");
2893
- if (!areaIds.length)
2894
- out.push("None.", "");
2895
- else
2896
- out.push("| ID | Title | Status |", "|---|---|---|", ...areaIds.map((x) => `| ${link(x)} | ${esc(entries.get(x).meta.title)} | ${entries.get(x).meta.status} |`), "");
2897
- }
2898
- return out.join("\n");
2899
- }
2900
- function renderByStatus(ids, path) {
2901
- const link = linkFrom(path);
2902
- const out = opening(path, "Entries by status.");
2903
- const section = (title, intro, list, withStatus) => {
2904
- if (!list.length && !isWhole(path))
2905
- return;
2906
- out.push(`## ${title}`, "");
2907
- if (intro)
2908
- out.push(intro, "");
2909
- if (!list.length) {
2910
- out.push(intro ? "None." : "0 entries.", "");
2911
- return;
2912
- }
2913
- if (!intro)
2914
- out.push(`${list.length} entries.`, "");
2915
- out.push(withStatus ? "| ID | Title | Status |" : "| ID | Title |", withStatus ? "|---|---|---|" : "|---|---|", ...list.map((x) => `| ${link(x)} | ${esc(entries.get(x).meta.title)} |${withStatus ? ` ${entries.get(x).meta.status} |` : ""}`), "");
2916
- };
2917
- for (const st of [...CLAIM_STATUSES, ...EVIDENCE_STATUSES.filter((x) => x !== "superseded")])
2918
- section(st, null, ids.filter((x) => entries.get(x).meta.status === st), false);
2919
- section("Established on unreproduced evidence", "Entries whose status is established and whose findings and experiments are all only recorded.", ids.filter((x) => KINDS[kindOf(x)].statuses === "claim" &&
2920
- entries.get(x).meta.status === "established" &&
2921
- asList(entries.get(x).meta.evidence)
2922
- .filter((y) => ["FND", "EXP"].includes(kindOf(y)))
2923
- .every((y) => entries.get(y)?.meta.status !== "reproduced")), false);
2924
- // Listed only when there are any, so that indexes written before complete readings stay fresh.
2925
- const byReading = ids.filter((x) => KINDS[kindOf(x)].statuses === "claim" &&
2926
- entries.get(x).meta.status === "established" &&
2927
- evidenceFacts(entries.get(x)).dynamic === 0);
2928
- if (byReading.length)
2929
- section("Established by a complete reading alone", "Entries whose status is established and that no dynamic finding or experiment confirms.", byReading, false);
2930
- section("Open questions", "Entries whose Open questions section says more than None known.", ids.filter((x) => {
2931
- const s = entries.get(x).sections.find((y) => y.title === "Open questions");
2932
- return s && !/^\s*None( known)?\.\s*$/.test(s.text);
2933
- }), true);
2934
- return out.join("\n");
2935
- }
2936
- const refs = new Map(sortedIds.map((x) => [x, new Map()]));
2937
- const addRef = (to, from, how) => {
2938
- if (refs.has(to) && to !== from) {
2939
- const m = refs.get(to);
2940
- if (!m.has(from))
2941
- m.set(from, new Set());
2942
- m.get(from).add(how);
2943
- }
2944
- };
2945
- for (const [id, e] of entries) {
2946
- for (const f of ["evidence", "conflicting", "related", "split_with", "superseded_by", "builds"])
2947
- for (const x of asList(e.meta[f]))
2948
- addRef(x, id, f);
2949
- for (const loc of asList(e.meta.locations))
2950
- if (loc?.build)
2951
- addRef(loc.build, id, "locations");
2952
- for (const x of idsIn(e.body))
2953
- addRef(x, id, "body");
2954
- }
2955
- for (const [term, text] of glossary)
2956
- for (const x of idsIn(text))
2957
- addRef(x, `glossary:${term}`, "glossary");
2958
- // One row per entry: everything that cites it goes in one cell.
2959
- function renderReferences(ids, path) {
2960
- const link = linkFrom(path);
2961
- const termLink = (term) => glossaryFiles.has(term)
2962
- ? `[${term}](${toSlash(relative(dirname(join(indexDir, `${path}.md`)), glossaryFiles.get(term)))})`
2963
- : esc(term);
2964
- const out = opening(path, "For each entry, the entries and glossary terms that cite or relate to it, and the field they do it in.");
2965
- out.push("| ID | Cited by |", "|---|---|");
2966
- for (const x of ids) {
2967
- const cited = [...refs.get(x)]
2968
- .sort((a, b) => compareIds(a[0], b[0]))
2969
- .map(([from, hows]) => `${from.startsWith("glossary:") ? termLink(from.slice(9)) : link(from)} (${[...hows].sort().join(", ")})`);
2970
- out.push(`| ${link(x)} | ${cited.join(", ") || "None"} |`);
2971
- }
2972
- out.push("");
2973
- return out.join("\n");
2974
- }
2975
- const generated = new Map(); // absolute path -> text
2976
- {
2977
- const areaIds = sortedIds.filter((x) => !["BLD", "SRC"].includes(kindOf(x)));
2978
- const indexes = {
2979
- "by-kind": [renderByKind, sortedIds],
2980
- "by-area": [renderByArea, areaIds],
2981
- "by-status": [renderByStatus, sortedIds],
2982
- references: [renderReferences, sortedIds],
2983
- };
2984
- for (const [name, [render, ids]] of Object.entries(indexes))
2985
- for (const [path, { text }] of layout(name, ids, render))
2986
- generated.set(join(indexDir, `${path}.md`), text);
2987
- const out = [
2988
- "# Parity matrix",
2989
- "",
2990
- GENERATED,
2991
- "",
2992
- "How much of the spec in `spec/` the rebuild does. The rows are in `parity/`, one file per area.",
2993
- "",
2994
- "| Status | Rows |",
2995
- "|---|---|",
2996
- ...["unknown", "sourced", "supported", "established", "disputed", "implemented", "validated"].map((k) => `| ${k} | ${parityCounts.status[k] ?? 0} |`),
2997
- "",
2998
- "| Code | Rows |",
2999
- "|---|---|",
3000
- ...["missing", "partial", "complete"].map((k) => `| ${k} | ${parityCounts.code[k] ?? 0} |`),
3001
- "",
3002
- "## Areas",
3003
- "",
3004
- ];
3005
- const withRows = areas.filter((a) => existsSync(join(parityDir, `${a}.md`)) || existsSync(join(parityDir, a)));
3006
- if (!withRows.length)
3007
- out.push("None yet.");
3008
- else
3009
- out.push("| Area | Rows |", "|---|---|", ...withRows.map((a) => {
3010
- const target = existsSync(join(parityDir, `${a}.md`)) ? `parity/${a}.md` : `parity/${a}/`;
3011
- return `| [${a}](${target}) | ${[...parityRows.keys()].filter((x) => areaOf(x) === a).length} |`;
3012
- }));
3013
- out.push("");
3014
- if (!legacyParity)
3015
- generated.set(join(repoDir, "PARITY.md"), out.join("\n"));
3016
- }
3017
- {
3018
- const stale = [];
3019
- for (const [p, content] of generated) {
3020
- const current = existsSync(p) ? readText(p) : null;
3021
- if (current !== content)
3022
- stale.push(p);
3023
- }
3024
- const extra = [...(existsSync(indexDir) ? markdownTree(indexDir).values() : [])].filter((f) => !generated.has(f));
3025
- if (checkOnly) {
3026
- for (const p of stale)
3027
- problem(p, existsSync(p)
3028
- ? "is stale; run the check without --check to rewrite it"
3029
- : "is missing; run the check without --check to write it");
3030
- for (const f of extra)
3031
- problem(f, "is not a file the check writes; run the check without --check to remove it");
3032
- }
3033
- else {
3034
- for (const p of stale) {
3035
- mkdirSync(dirname(p), { recursive: true });
3036
- writeFileSync(p, generated.get(p));
3037
- console.log(`wrote ${toSlash(relative(repoDir, p))}`);
3038
- }
3039
- for (const f of extra) {
3040
- rmSync(f);
3041
- console.log(`removed ${toSlash(relative(repoDir, f))}`);
3042
- }
3043
- // Directories left empty by a file that moved.
3044
- const prune = (dir) => {
3045
- for (const name of readdirSync(dir)) {
3046
- const p = join(dir, name);
3047
- if (statSync(p).isDirectory())
3048
- prune(p);
3049
- }
3050
- if (dir !== indexDir && readdirSync(dir).length === 0)
3051
- rmSync(dir, { recursive: true });
3052
- };
3053
- if (existsSync(indexDir))
3054
- prune(indexDir);
3055
- }
3056
- }
3057
- // Every Markdown file the standard defines is at most LINE_LIMIT lines long.
3058
- {
3059
- const files = [join(repoDir, "PARITY.md"), validationPath];
3060
- for (const dir of [specDir, parityDir, devDir])
3061
- walk(dir, (f) => {
3062
- if (f.endsWith(".md"))
3063
- files.push(f);
3064
- });
3065
- for (const f of files) {
3066
- if (!existsSync(f))
3067
- continue;
3068
- const lines = lineCount(readText(f));
3069
- if (lines > LINE_LIMIT)
3070
- problem(f, `has ${lines} lines; the documentation standard allows ${LINE_LIMIT}. Split it as the standard's File size section describes`);
3071
- }
3072
- }
3073
- // ---------------------------------------------------------------------------------------------
3074
- // The same problem can be found twice, such as a term that cites one finding in two places.
3075
- const unique = [...new Set(problems)];
3076
- if (unique.length) {
3077
- for (const p of unique)
3078
- console.error(p);
3079
- console.error(`\n${unique.length} problem(s) in ${entries.size} spec entries.`);
3080
- process.exit(1);
3081
- }
3082
- console.log(`spec check passed: ${entries.size} entries, ${parityRows.size} parity rows, ${deviations.size} deviations.`);
79
+ const config = parseOptions(argv);
80
+ const { problem, report } = createProblems(config.repoDir);
81
+ const spec = loadSpec({ config, problem });
82
+ // The checker's modules all sit in this file's directory, which holds nothing else, so the code
83
+ // checks leave the whole directory out.
84
+ const ctx = { config, problem, spec, codeFiles: createCodeFiles(config, dirname(selfPath)) };
85
+ const formatNames = checkEntries(ctx);
86
+ checkRules(ctx, formatNames);
87
+ checkAcrossEntries(ctx, formatNames);
88
+ compileKaitai(ctx);
89
+ const deviations = checkDeviations(ctx);
90
+ const parity = checkParity(ctx, deviations);
91
+ checkValidation(ctx, parity);
92
+ checkReferences(ctx, deviations);
93
+ checkCommentAddresses(ctx);
94
+ checkBase(ctx, deviations);
95
+ const generated = generateIndexes(ctx);
96
+ generateParity(ctx, parity, generated);
97
+ writeGenerated(ctx, generated);
98
+ checkLineLimits(ctx);
99
+ report(spec.entries.size);
100
+ console.log(`spec check passed: ${spec.entries.size} entries, ${parity.rows.size} parity rows, ${deviations.size} deviations.`);