@aria-framework/testkit 0.3.0 → 0.4.0

package/index.js

@@ -9,6 +9,9 @@
  *                                         in scope (the silently-empty-list-in-a-catch class).
  *   assertVersionSingleSource({...})    — package.json vs derived version files vs the lockfile's
  *                                         two copies.
+ *   assertInstalledMatchesLockfile({...}) — what is in node_modules is what `npm ci` would install.
+ *                                         The hand-copied-package class: green locally, different
+ *                                         code on the server, no error either side.
  */
 
 'use strict';
@@ -18,6 +21,7 @@ module.exports = Object.assign(
   require('./harness'),
   require('./importsInScope'),
   require('./versionSync'),
+  require('./installedMatchesLockfile'),
   // copyDatabase / assertIsCopy / sweep — a private copy of the app's database for one test run,
   // which clears whatever an earlier run with this pid left behind and takes itself away
   // afterwards. See tempDb.js for the flake that made it necessary.

package/installedMatchesLockfile.js

@@ -0,0 +1,99 @@
+/**
+ * assertInstalledMatchesLockfile — what is ON DISK is what `npm ci` would install.
+ *
+ * ── THE FAILURE THIS EXISTS FOR ─────────────────────────────────────────────────────────────────
+ * A framework package is edited in its own checkout and hand-copied into an app's node_modules.
+ * Everything local goes green against it. The lockfile still pins the PUBLISHED version, and the
+ * documented deploy is `git pull && npm ci --omit=dev` — which installs the lockfile's version, not
+ * the one every test just passed against.
+ *
+ * It happened, it was not small, and it was invisible: @aria-framework/ai sat at 0.18.1 in
+ * node_modules and 0.18.0 in the lockfile, and the ONLY file that differed between them was
+ * untrusted.js — the prompt-injection fence the consuming app's hardening was built on. 0.18.0's
+ * rule pointed the model at the wrong text. Production would have run a fence that says "below"
+ * about a block placed above, while a suite asserting the opposite passed on every machine that
+ * had the copy.
+ *
+ * ── WHY A TEST AND NOT A NOTE ───────────────────────────────────────────────────────────────────
+ * Because the failure is silent in the direction that matters. Nothing errors: the app runs, the
+ * tests pass, and the difference appears only on the server, in behaviour, with no message. A note
+ * describing exactly this already existed in one app's project memory, written after the previous
+ * time it happened — and it happened again anyway. A note that has been ignored once is a test that
+ * had not been written yet.
+ *
+ * ── WHAT IT DOES NOT CHECK ──────────────────────────────────────────────────────────────────────
+ * Whether the lockfile's version exists on the registry. That needs the network, and a check that
+ * fails on a train is a check people learn to skip. This compares two files on disk, which is
+ * enough to catch the hand-copy — the only way the two can disagree.
+ *
+ * Sibling of assertVersionSingleSource: same class of problem (a second copy nothing validates),
+ * same shape of answer.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @param {{
+ *   root: string,        app root (absolute)
+ *   scope?: string,      package scope to check (default '@aria-framework')
+ *   lockfile?: string    lockfile name (default 'package-lock.json')
+ * }} opts
+ * @returns {{checked: number}} how many packages were compared — ASSERT ON THIS. A misspelled
+ *          scope matches nothing and passes, and a check that inspects nothing is also a claim.
+ * @throws Error naming every package whose installed version differs from the pin
+ */
+function assertInstalledMatchesLockfile(opts = {}) {
+  const root = opts.root;
+  if (!root) throw new Error('assertInstalledMatchesLockfile({ root }): the app root is required');
+  const scope = opts.scope || '@aria-framework';
+  const lockPath = path.join(root, opts.lockfile || 'package-lock.json');
+
+  let lock;
+  try {
+    lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
+  } catch (e) {
+    throw new Error(`could not read ${lockPath}: ${e.message}`);
+  }
+
+  // BOTH SHAPES OF NAME. A scope is `@aria-framework`, whose keys are `node_modules/@scope/name`;
+  // a bare package name is its own key, `node_modules/left-pad`, with nothing after it. Matching
+  // only the first form made this return `{ checked: 0 }` and PASS for the second — inspecting
+  // nothing while reporting success, which is worse than no check because it is also a claim.
+  const nested = `node_modules/${scope}/`;
+  const exact = `node_modules/${scope}`;
+  const wrong = [];
+  let checked = 0;
+
+  for (const [key, entry] of Object.entries(lock.packages || {})) {
+    // STARTSWITH FROM THE FRONT, NOT INCLUDES. A nested copy —
+    // node_modules/x/node_modules/@scope/y — is a different resolution with its own pin, and
+    // comparing it against the top-level install would report a disagreement that is not one.
+    if (!key.startsWith(nested) && key !== exact) continue;
+    const name = key.slice('node_modules/'.length);
+    checked += 1;
+
+    let installed;
+    try {
+      installed = JSON.parse(fs.readFileSync(path.join(root, key, 'package.json'), 'utf8')).version;
+    } catch (e) {
+      wrong.push(`${name}: pinned ${entry.version}, NOT INSTALLED`);
+      continue;
+    }
+    if (installed !== entry.version) {
+      wrong.push(`${name}: lockfile ${entry.version}, installed ${installed}`);
+    }
+  }
+
+  if (wrong.length) {
+    throw new Error(
+      'node_modules does not match the lockfile, so `npm ci` on the server installs something these '
+      + 'tests never ran against. Publish the package and `npm install` it, or reinstall the pinned '
+      + 'version — do NOT hand-copy a package into node_modules:\n      ' + wrong.join('\n      '));
+  }
+  return { checked };
+}
+
+module.exports = { assertInstalledMatchesLockfile };

package/package.json

@@ -1,21 +1,22 @@
-{
-  "name": "@aria-framework/testkit",
-  "description": "Aria App Framework \u2014 test infrastructure. The \u2713/\u2717 micro test-harness (createHarness, with the async-callback guard that stops a suite silently passing), assertImportsInScope (every consumer of a module has every function it calls in scope), and assertVersionSingleSource (package.json vs derived version files vs the lockfile's two copies). Dev-dependency only: nothing here belongs in a production install.",
-  "version": "0.3.0",
-  "license": "UNLICENSED",
-  "private": false,
-  "publishConfig": {
-    "access": "public"
-  },
-  "main": "index.js",
-  "files": [
-    "index.js",
-    "harness.js",
-    "importsInScope.js",
-    "versionSync.js",
-    "tempDb.js"
-  ],
-  "scripts": {
-    "test": "node test/smoke.js && node test/tempDb.js"
-  }
-}
+{
+  "name": "@aria-framework/testkit",
+  "description": "Aria App Framework — test infrastructure. The ✓/✗ micro test-harness (createHarness, with the async-callback guard that stops a suite silently passing), assertImportsInScope (every consumer of a module has every function it calls in scope), and assertVersionSingleSource (package.json vs derived version files vs the lockfile's two copies). Dev-dependency only: nothing here belongs in a production install.",
+  "version": "0.4.0",
+  "license": "UNLICENSED",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "index.js",
+  "files": [
+    "harness.js",
+    "importsInScope.js",
+    "index.js",
+    "installedMatchesLockfile.js",
+    "tempDb.js",
+    "versionSync.js"
+  ],
+  "scripts": {
+    "test": "node test/smoke.js && node test/tempDb.js && node test/installedMatchesLockfile.js"
+  }
+}