@scientific-method/standard-checker 1.1.0 → 2.1.0

package/dist/checks/deviations.d.ts

@@ -3,8 +3,14 @@ import type { Context } from "../context.ts";
 export interface Deviation {
     /** The spec IDs its Departs from item names. */
     departs: string[];
+    /** The spec IDs its Replaces item names: the entries it replaces entirely. */
+    replaces: string[];
     /** Whether its Dropped item says it was dropped. */
     dropped: boolean;
+    /** Whether its Default is mandatory. */
+    mandatory: boolean;
+    /** The test files its Tests item lists, which check the rebuild does what the deviation says. */
+    tests: string[];
     /** Its file. */
     file: string;
 }

package/dist/checks/deviations.js

@@ -3,7 +3,7 @@
 import { existsSync, statSync } from "node:fs";
 import { basename, join } from "node:path";
 import { checkResolves, isSuperseded } from "../evidence.js";
-import { termFiles } from "../files.js";
+import { checkTestFiles, termFiles } from "../files.js";
 import { areaOf, idsIn, kindOf } from "../ids.js";
 import { readText } from "../markdown.js";
 /** Reads and checks deviations/. Returns deviation ID -> the deviation. */
@@ -44,10 +44,12 @@ export function checkDeviations(ctx) {
                 const item = Object.fromEntries(items);
                 const order = [
                     "Departs from",
+                    ...("Replaces" in item ? ["Replaces"] : []),
                     "Reason",
                     "Setting",
                     "Default",
                     ...("Justification" in item ? ["Justification"] : []),
+                    ...("Tests" in item ? ["Tests"] : []),
                     "Dropped",
                 ];
                 if (items
@@ -59,7 +61,12 @@ export function checkDeviations(ctx) {
                 const dropped = Boolean(item.Dropped) && item.Dropped !== "no";
                 if (dropped && !/^\d{4}-\d{2}-\d{2}\b/.test(item.Dropped))
                     problem(file, `${title}: Dropped gives the date, YYYY-MM-DD, and the reason`);
+                const replaces = idsIn(item.Replaces);
                 checkResolves(ctx, file, departs, `${title} Departs from`);
+                checkResolves(ctx, file, replaces, `${title} Replaces`);
+                // The Tests item lists the test files that check the rebuild does what the deviation says. A
+                // dropped deviation's tests may have gone with it.
+                let tests = [];
                 if (!dropped) {
                     if (!departs.some((x) => ["RULE", "FMT", "SCR"].includes(kindOf(x))))
                         problem(file, `${title}: Departs from names at least one rule, format or screen`);
@@ -67,8 +74,28 @@ export function checkDeviations(ctx) {
                         if (isSuperseded(entries, x))
                             problem(file, `${title} departs from ${x}, which is superseded`);
                     checkDeviationDefault(ctx, file, title, item, departs);
+                    // Replaces names the entries of Departs from that a mandatory deviation replaces entirely, which
+                    // is what lets their rows become deviated.
+                    if ("Replaces" in item) {
+                        if (item.Default !== "mandatory")
+                            problem(file, `${title}: only a mandatory deviation has a Replaces item`);
+                        if (replaces.length === 0)
+                            problem(file, `${title}: Replaces names at least one entry`);
+                        for (const x of replaces) {
+                            if (!departs.includes(x))
+                                problem(file, `${title}: Replaces names ${x}, which Departs from does not`);
+                            if (!["RULE", "FMT", "SCR"].includes(kindOf(x)))
+                                problem(file, `${title}: Replaces names ${x}, which is not a rule, format or screen`);
+                        }
+                    }
+                    // Nothing records a local run of a deviation's tests, so they run in CI and never need GAME_DIR.
+                    if ("Tests" in item) {
+                        tests = checkTestFiles(ctx, file, title, item.Tests, false);
+                        if (tests.length === 0)
+                            problem(file, `${title}: Tests lists at least one test file; leave the item out when there is none`);
+                    }
                 }
-                deviations.set(title, { departs, dropped, file });
+                deviations.set(title, { departs, replaces, dropped, mandatory: item.Default === "mandatory", tests, file });
             }
     }
     return deviations;

package/dist/checks/evidence-entries.js

@@ -6,6 +6,7 @@ import { isSuperseded } from "../evidence.js";
 import { asList } from "../ids.js";
 import { checkAddress, checkOffset } from "../locations.js";
 import { locationRule } from "../standard.js";
+import { parseScalar } from "../yaml.js";
 /** Checks a finding's method, environment and locations. */
 export function checkFinding(ctx, e) {
     const { problem } = ctx;
@@ -121,6 +122,24 @@ export function checkExperiment(ctx, id, e, isSup) {
     const builds = asList(meta.builds);
     if (builds.length !== 1)
         problem(file, "an experiment lists exactly one build", "ENTRY-TYPES-7");
+    // The standard's experiment entry lists the starting states: a save patch, a save, new-game,
+    // emulated-call or null, where a save or a patch is in saves/. A missing field is reported with
+    // the other required fields. Which save hashes the fixture needs depends on the form, so an
+    // unknown one gets no save hash problem. A path in saves/ names a file under it, so it has a
+    // name after saves/ and no empty, . or .. segment.
+    const state = meta.starting_state;
+    const savePath = typeof state === "string" &&
+        state.startsWith("saves/") &&
+        state
+            .split("/")
+            .slice(1)
+            .every((s) => s !== "" && s !== "." && s !== "..");
+    const fromSave = state === null || savePath;
+    if (state !== undefined && !fromSave && state !== "new-game" && state !== "emulated-call") {
+        // A string the front matter would read as another value, such as a quoted "null", is shown quoted.
+        const shown = typeof state === "string" && state !== "" && parseScalar(state) === state ? state : JSON.stringify(state);
+        problem(file, `starting_state ${shown} is none of the forms the standard defines: a save or save patch in saves/, new-game, emulated-call or null`);
+    }
     const fixture = meta.fixture && join(specDir, "experiments", meta.fixture);
     if (!fixture || !existsSync(fixture))
         problem(file, `fixture ${meta.fixture} does not exist`);
@@ -132,13 +151,13 @@ export function checkExperiment(ctx, id, e, isSup) {
             // Only a new game and an emulated call start without a save. starting_state null still has
             // one: a save that cannot be committed, kept with the captures and found by its hash.
             const save = fx.starting_state;
-            if (!["new-game", "emulated-call"].includes(meta.starting_state) && !save?.xxh3) {
+            if (fromSave && !save?.xxh3) {
                 const where = "gives the hash of the save its runs started from in starting_state.xxh3";
-                problem(fixture, meta.starting_state === null
+                problem(fixture, state === null
                     ? `${where}; starting_state null names a save kept with the captures, and an experiment that starts without a save has starting_state new-game or emulated-call`
                     : where);
             }
-            if (typeof meta.starting_state === "string" && meta.starting_state.endsWith(".patch.json") && !save?.base_xxh3)
+            if (savePath && state.endsWith(".patch.json") && !save?.base_xxh3)
                 problem(fixture, "a patch fixture gives the base save's hash as well");
             for (const key of ["xxh3", "base_xxh3"])
                 if (save?.[key] && !/^[0-9a-f]{32}$/.test(String(save[key])))
@@ -156,10 +175,8 @@ export function checkExperiment(ctx, id, e, isSup) {
             problem(fixture, `is not valid JSON: ${err.message}`);
         }
     }
-    if (typeof meta.starting_state === "string" &&
-        meta.starting_state.startsWith("saves/") &&
-        !existsSync(join(specDir, "experiments", meta.starting_state)))
-        problem(file, `starting_state ${meta.starting_state} does not exist`);
+    if (savePath && !existsSync(join(specDir, "experiments", state)))
+        problem(file, `starting_state ${state} does not exist`);
     // A recording is committed in recordings/, kept with the captures, or one of the build's files.
     if (typeof meta.recording === "string" && meta.recording !== "") {
         const rec = meta.recording;

package/dist/checks/parity.js

@@ -4,7 +4,7 @@ import { existsSync, readFileSync } from "node:fs";
 import { basename, join } from "node:path";
 import { collectPlaceholders } from "../code-files.js";
 import { mayBeInterrupted, onlyEmulatedRuns } from "../evidence.js";
-import { markdownTree, walk } from "../files.js";
+import { checkTestFiles, markdownTree, NEEDS_GAME, walk } from "../files.js";
 import { areaOf, compareIds, kindOf } from "../ids.js";
 import { readText, tables } from "../markdown.js";
 import { LINE_LIMIT } from "../standard.js";
@@ -24,10 +24,9 @@ export function checkParity(ctx, deviations) {
     const parityRows = new Map(); // spec ID -> { cells, file }
     const parityCounts = { status: {}, code: {} };
     const validatedTests = new Map(); // marked test file of a validated row -> [{ specId, file }]
-    // A test file that reads the original's files through GAME_DIR says so with this comment. It runs
+    // A test file that reads the original's files through GAME_DIR says so with NEEDS_GAME. It runs
     // only on a maintainer's machine, so its validated rows need it in VALIDATION.md; every other test
     // runs in CI.
-    const NEEDS_GAME = /needs:\s*GAME_DIR/;
     const needsGame = (p) => existsSync(p) && NEEDS_GAME.test(readFileSync(p, "utf8"));
     // A PARITY.md that still holds the rows is left alone until they have moved, so the check does not
     // overwrite them with the totals.
@@ -101,24 +100,7 @@ export function checkParity(ctx, deviations) {
                     problem(file, `${specId}: an unknown entry cannot be complete`);
                 if (code === "complete" && placeholders.has(specId))
                     problem(file, `${specId}: a PLACEHOLDER comment cites it, so it cannot be complete`);
-                const testFiles = tests === "None"
-                    ? []
-                    : tests
-                        .split(",")
-                        .map((x) => x.trim())
-                        .filter(Boolean);
-                for (const tf of testFiles) {
-                    const p = join(repoDir, tf);
-                    if (!existsSync(p))
-                        problem(file, `${specId}: test file ${tf} does not exist`);
-                    else {
-                        const text = readFileSync(p, "utf8");
-                        if (!text.includes(specId))
-                            problem(file, `${specId}: test file ${tf} does not mention ${specId}`);
-                        if (text.includes("GAME_DIR") && !NEEDS_GAME.test(text))
-                            problem(file, `${specId}: test file ${tf} mentions GAME_DIR without a "needs: GAME_DIR" comment, so CI would skip it unseen`);
-                    }
-                }
+                const testFiles = checkTestFiles(ctx, file, specId, tests);
                 const listedDevs = devs === "None"
                     ? []
                     : devs
@@ -131,11 +113,25 @@ export function checkParity(ctx, deviations) {
                     .sort(compareIds);
                 if (listedDevs.slice().sort(compareIds).join(",") !== expectedDevs.join(","))
                     problem(file, `${specId}: Deviations must be ${expectedDevs.join(", ") || "None"}`);
+                // A row whose entry a mandatory deviation replaces entirely cannot be compared with the original,
+                // so it has no tests of its own: they belong in the deviation's Tests item. It is deviated once
+                // every mandatory deviation it lists has tests.
+                const mandatory = listedDevs
+                    .map((x) => deviations.get(x))
+                    .filter((d) => Boolean(d?.mandatory && !d.dropped));
+                const replacing = listedDevs.find((x) => {
+                    const d = deviations.get(x);
+                    return d?.mandatory && !d.dropped && d.replaces.includes(specId);
+                });
+                const replaced = replacing !== undefined;
+                if (replaced && testFiles.length > 0)
+                    problem(file, `${specId}: ${replacing} replaces it, so Tests must be None; list the tests in the deviation's Tests item`);
+                const deviated = replaced && mandatory.every((d) => d.tests.length > 0);
                 let expectedStatus;
                 if (code !== "complete" || e.meta.status === "disputed")
                     expectedStatus = e.meta.status;
-                else if (testFiles.length === 0)
-                    expectedStatus = "implemented";
+                else if (testFiles.length === 0 || replaced)
+                    expectedStatus = deviated ? "deviated" : "implemented";
                 else if (["supported", "established"].includes(e.meta.status))
                     expectedStatus = "validated";
                 else {

package/dist/files.d.ts

@@ -1,3 +1,4 @@
+import type { Context } from "./context.ts";
 /** A path with backslashes turned into forward slashes. */
 export declare const toSlash: (p: string) => string;
 /**
@@ -9,3 +10,12 @@ export declare function walk(dir: string, fn: (path: string) => void): void;
 export declare function markdownTree(dir: string): Map<string, string>;
 /** The paths of the files and directories in dir, apart from .gitkeep. */
 export declare const termFiles: (dir: string) => string[];
+/** The comment of a test file that reads the original's files through GAME_DIR, which CI skips. */
+export declare const NEEDS_GAME: RegExp;
+/**
+ * Checks the test files that cell lists for id, comma-separated by their path from the repository
+ * root, and returns them. `None` lists none. Each listed path is a file that mentions id, and one that
+ * mentions GAME_DIR carries a "needs: GAME_DIR" comment, since CI skips it. Without needsGame, a file
+ * that mentions GAME_DIR at all is a problem. Each problem goes to file.
+ */
+export declare function checkTestFiles(ctx: Context, file: string, id: string, cell: string, needsGame?: boolean): string[];

package/dist/files.js

@@ -1,5 +1,5 @@
-// Walking the repository's directories.
-import { existsSync, readdirSync, statSync } from "node:fs";
+// Walking the repository's directories, and reading the test files a row or deviation lists.
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
 import { join, relative } from "node:path";
 /** A path with backslashes turned into forward slashes. */
 export const toSlash = (p) => p.replaceAll("\\", "/");
@@ -33,3 +33,39 @@ export function markdownTree(dir) {
 export const termFiles = (dir) => readdirSync(dir)
     .filter((name) => name !== ".gitkeep")
     .map((name) => join(dir, name));
+/** The comment of a test file that reads the original's files through GAME_DIR, which CI skips. */
+export const NEEDS_GAME = /needs:\s*GAME_DIR/;
+/**
+ * Checks the test files that cell lists for id, comma-separated by their path from the repository
+ * root, and returns them. `None` lists none. Each listed path is a file that mentions id, and one that
+ * mentions GAME_DIR carries a "needs: GAME_DIR" comment, since CI skips it. Without needsGame, a file
+ * that mentions GAME_DIR at all is a problem. Each problem goes to file.
+ */
+export function checkTestFiles(ctx, file, id, cell, needsGame = true) {
+    const { problem } = ctx;
+    const listed = cell === "None"
+        ? []
+        : cell
+            .split(",")
+            .map((x) => x.replaceAll("`", "").trim())
+            .filter(Boolean);
+    // Whole IDs only, as ID_RE reads them, so RULE-SCORE-0010 does not mention RULE-SCORE-001.
+    const mention = new RegExp(`\\b${id}\\b`);
+    for (const tf of listed) {
+        const p = join(ctx.config.repoDir, tf);
+        if (!existsSync(p))
+            problem(file, `${id}: test file ${tf} does not exist`);
+        else if (!statSync(p).isFile())
+            problem(file, `${id}: test file ${tf} is not a file`);
+        else {
+            const text = readFileSync(p, "utf8");
+            if (!mention.test(text))
+                problem(file, `${id}: test file ${tf} does not mention ${id}`);
+            if (text.includes("GAME_DIR") && !needsGame)
+                problem(file, `${id}: test file ${tf} mentions GAME_DIR, but it has to run in CI`);
+            else if (text.includes("GAME_DIR") && !NEEDS_GAME.test(text))
+                problem(file, `${id}: test file ${tf} mentions GAME_DIR without a "needs: GAME_DIR" comment, so CI would skip it unseen`);
+        }
+    }
+    return listed;
+}

package/dist/generate/parity-md.js

@@ -21,7 +21,7 @@ export function generateParity(ctx, parity, generated) {
         "",
         "| Status | Rows |",
         "|---|---|",
-        ...["unknown", "sourced", "supported", "established", "disputed", "implemented", "validated"].map((k) => `| ${k} | ${parityCounts.status[k] ?? 0} |`),
+        ...["unknown", "sourced", "supported", "established", "disputed", "implemented", "deviated", "validated"].map((k) => `| ${k} | ${parityCounts.status[k] ?? 0} |`),
         "",
         "| Code | Rows |",
         "|---|---|",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scientific-method/standard-checker",
-  "version": "1.1.0",
+  "version": "2.1.0",
   "description": "Checks a restoration's spec/, parity/ and deviations/ against version 1 of the dinorefurb documentation standard.",
   "type": "module",
   "license": "MIT",