@aria-framework/testkit 0.2.0 → 0.4.0

package/index.js

@@ -9,6 +9,9 @@
  *                                         in scope (the silently-empty-list-in-a-catch class).
  *   assertVersionSingleSource({...})    — package.json vs derived version files vs the lockfile's
  *                                         two copies.
+ *   assertInstalledMatchesLockfile({...}) — what is in node_modules is what `npm ci` would install.
+ *                                         The hand-copied-package class: green locally, different
+ *                                         code on the server, no error either side.
  */
 
 'use strict';
@@ -18,6 +21,7 @@ module.exports = Object.assign(
   require('./harness'),
   require('./importsInScope'),
   require('./versionSync'),
+  require('./installedMatchesLockfile'),
   // copyDatabase / assertIsCopy / sweep — a private copy of the app's database for one test run,
   // which clears whatever an earlier run with this pid left behind and takes itself away
   // afterwards. See tempDb.js for the flake that made it necessary.

package/installedMatchesLockfile.js

@@ -0,0 +1,99 @@
+/**
+ * assertInstalledMatchesLockfile — what is ON DISK is what `npm ci` would install.
+ *
+ * ── THE FAILURE THIS EXISTS FOR ─────────────────────────────────────────────────────────────────
+ * A framework package is edited in its own checkout and hand-copied into an app's node_modules.
+ * Everything local goes green against it. The lockfile still pins the PUBLISHED version, and the
+ * documented deploy is `git pull && npm ci --omit=dev` — which installs the lockfile's version, not
+ * the one every test just passed against.
+ *
+ * It happened, it was not small, and it was invisible: @aria-framework/ai sat at 0.18.1 in
+ * node_modules and 0.18.0 in the lockfile, and the ONLY file that differed between them was
+ * untrusted.js — the prompt-injection fence the consuming app's hardening was built on. 0.18.0's
+ * rule pointed the model at the wrong text. Production would have run a fence that says "below"
+ * about a block placed above, while a suite asserting the opposite passed on every machine that
+ * had the copy.
+ *
+ * ── WHY A TEST AND NOT A NOTE ───────────────────────────────────────────────────────────────────
+ * Because the failure is silent in the direction that matters. Nothing errors: the app runs, the
+ * tests pass, and the difference appears only on the server, in behaviour, with no message. A note
+ * describing exactly this already existed in one app's project memory, written after the previous
+ * time it happened — and it happened again anyway. A note that has been ignored once is a test that
+ * had not been written yet.
+ *
+ * ── WHAT IT DOES NOT CHECK ──────────────────────────────────────────────────────────────────────
+ * Whether the lockfile's version exists on the registry. That needs the network, and a check that
+ * fails on a train is a check people learn to skip. This compares two files on disk, which is
+ * enough to catch the hand-copy — the only way the two can disagree.
+ *
+ * Sibling of assertVersionSingleSource: same class of problem (a second copy nothing validates),
+ * same shape of answer.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+
+/**
+ * @param {{
+ *   root: string,        app root (absolute)
+ *   scope?: string,      package scope to check (default '@aria-framework')
+ *   lockfile?: string    lockfile name (default 'package-lock.json')
+ * }} opts
+ * @returns {{checked: number}} how many packages were compared — ASSERT ON THIS. A misspelled
+ *          scope matches nothing and passes, and a check that inspects nothing is also a claim.
+ * @throws Error naming every package whose installed version differs from the pin
+ */
+function assertInstalledMatchesLockfile(opts = {}) {
+  const root = opts.root;
+  if (!root) throw new Error('assertInstalledMatchesLockfile({ root }): the app root is required');
+  const scope = opts.scope || '@aria-framework';
+  const lockPath = path.join(root, opts.lockfile || 'package-lock.json');
+
+  let lock;
+  try {
+    lock = JSON.parse(fs.readFileSync(lockPath, 'utf8'));
+  } catch (e) {
+    throw new Error(`could not read ${lockPath}: ${e.message}`);
+  }
+
+  // BOTH SHAPES OF NAME. A scope is `@aria-framework`, whose keys are `node_modules/@scope/name`;
+  // a bare package name is its own key, `node_modules/left-pad`, with nothing after it. Matching
+  // only the first form made this return `{ checked: 0 }` and PASS for the second — inspecting
+  // nothing while reporting success, which is worse than no check because it is also a claim.
+  const nested = `node_modules/${scope}/`;
+  const exact = `node_modules/${scope}`;
+  const wrong = [];
+  let checked = 0;
+
+  for (const [key, entry] of Object.entries(lock.packages || {})) {
+    // STARTSWITH FROM THE FRONT, NOT INCLUDES. A nested copy —
+    // node_modules/x/node_modules/@scope/y — is a different resolution with its own pin, and
+    // comparing it against the top-level install would report a disagreement that is not one.
+    if (!key.startsWith(nested) && key !== exact) continue;
+    const name = key.slice('node_modules/'.length);
+    checked += 1;
+
+    let installed;
+    try {
+      installed = JSON.parse(fs.readFileSync(path.join(root, key, 'package.json'), 'utf8')).version;
+    } catch (e) {
+      wrong.push(`${name}: pinned ${entry.version}, NOT INSTALLED`);
+      continue;
+    }
+    if (installed !== entry.version) {
+      wrong.push(`${name}: lockfile ${entry.version}, installed ${installed}`);
+    }
+  }
+
+  if (wrong.length) {
+    throw new Error(
+      'node_modules does not match the lockfile, so `npm ci` on the server installs something these '
+      + 'tests never ran against. Publish the package and `npm install` it, or reinstall the pinned '
+      + 'version — do NOT hand-copy a package into node_modules:\n      ' + wrong.join('\n      '));
+  }
+  return { checked };
+}
+
+module.exports = { assertInstalledMatchesLockfile };

package/package.json

@@ -1,21 +1,22 @@
-{
-  "name": "@aria-framework/testkit",
-  "description": "Aria App Framework \u2014 test infrastructure. The \u2713/\u2717 micro test-harness (createHarness, with the async-callback guard that stops a suite silently passing), assertImportsInScope (every consumer of a module has every function it calls in scope), and assertVersionSingleSource (package.json vs derived version files vs the lockfile's two copies). Dev-dependency only: nothing here belongs in a production install.",
-  "version": "0.2.0",
-  "license": "UNLICENSED",
-  "private": false,
-  "publishConfig": {
-    "access": "public"
-  },
-  "main": "index.js",
-  "files": [
-    "index.js",
-    "harness.js",
-    "importsInScope.js",
-    "versionSync.js",
-    "tempDb.js"
-  ],
-  "scripts": {
-    "test": "node test/smoke.js && node test/tempDb.js"
-  }
-}
+{
+  "name": "@aria-framework/testkit",
+  "description": "Aria App Framework — test infrastructure. The ✓/✗ micro test-harness (createHarness, with the async-callback guard that stops a suite silently passing), assertImportsInScope (every consumer of a module has every function it calls in scope), and assertVersionSingleSource (package.json vs derived version files vs the lockfile's two copies). Dev-dependency only: nothing here belongs in a production install.",
+  "version": "0.4.0",
+  "license": "UNLICENSED",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "index.js",
+  "files": [
+    "harness.js",
+    "importsInScope.js",
+    "index.js",
+    "installedMatchesLockfile.js",
+    "tempDb.js",
+    "versionSync.js"
+  ],
+  "scripts": {
+    "test": "node test/smoke.js && node test/tempDb.js && node test/installedMatchesLockfile.js"
+  }
+}

package/tempDb.js

@@ -54,6 +54,44 @@ function removeAll(base) {
   }
 }
 
+/**
+ * Run a caller's close hook, and refuse to be lied to about it.
+ *
+ * AN EXIT HANDLER CANNOT AWAIT. So a `close` that returns a promise has not closed anything by the
+ * time the unlink runs, and on Windows that unlink then fails with EBUSY into a catch nobody reads.
+ * The copy is left behind and the suite still passes — which is precisely the shape of bug this
+ * whole module exists to end, arriving through the mechanism meant to prevent it.
+ *
+ * It is worse than a plain failure because an async close can APPEAR to work: `async close() {
+ * db.close(); }` runs its body up to the first await synchronously, so a driver written that way
+ * cleans up perfectly until somebody adds an await ahead of that line, or the app moves to a driver
+ * whose close is genuinely asynchronous. Then every leftover returns, silently, and the commit that
+ * introduced the regression is nowhere near the code that breaks.
+ *
+ * So: say so, loudly, on the one occasion it can still be heard.
+ */
+function closeSynchronously(close, file) {
+  if (typeof close !== 'function') return;
+  let result;
+  try {
+    result = close();
+  } catch (e) {
+    // A synchronous throw is fine — already closed, or never opened. The delete still runs.
+    return;
+  }
+  if (result && typeof result.then === 'function') {
+    // Not swallowed: this is the only moment anybody could learn about it.
+    // eslint-disable-next-line no-console
+    console.error(
+      `testkit: the close hook for ${path.basename(file)} returned a promise. An exit handler `
+      + 'cannot await, so the database is still open when the file is deleted — on Windows that '
+      + 'fails silently and the copy is left behind. Pass a SYNCHRONOUS close: '
+      + 'the driver' + String.fromCharCode(39) + 's own db.close() usually is one, even when the wrapper is not.');
+    // ...and do not let it become an unhandled rejection on the way out.
+    result.then(undefined, () => {});
+  }
+}
+
 /**
  * Copy `livePath` to a private temp file and return its path.
  *
@@ -63,6 +101,8 @@ function removeAll(base) {
  *   dir       where to put it; defaults to the OS temp directory
  *   prefix    default 'app' — apps pass their own so two projects cannot collide
  *   cleanup   default true; false leaves the copy behind for inspection after a failure
+ *   allowMissing  default false; true means an absent live database copies nothing rather
+ *             than throwing, for a test that builds its schema from migrations
  *   close     optional () => void, run before deleting. On Windows an open handle makes unlink
  *             fail with EBUSY, so without this the exit hook is a no-op for any test that does not
  *             close its database — which is most of them.
@@ -70,7 +110,17 @@ function removeAll(base) {
 function copyDatabase(o = {}) {
   const live = o.livePath;
   if (!live) throw new Error('copyDatabase({ livePath }): the database to copy is required');
-  if (!fs.existsSync(live)) throw new Error(`copyDatabase: there is no database at ${live}`);
+  // STRICT BY DEFAULT. A test that quietly ran against an empty database when it meant to use real
+  // data is its own kind of wrong, and silent is the worst way for that to happen. But a suite that
+  // BUILDS its schema from migrations is legitimately fine without one, and the hand-rolled loop
+  // this replaced tolerated it — `if (exists) copy` simply copied nothing. On a fresh clone, or a CI
+  // box that has never booted the app, strictness alone turns those tests into a module-scope crash.
+  const missing = !fs.existsSync(live);
+  if (missing && !o.allowMissing) {
+    throw new Error(
+      `copyDatabase: there is no database at ${live}. Pass allowMissing:true if this test builds `
+      + 'its own schema and does not need one.');
+  }
 
   const label = String(o.name || 'test').replace(/[^a-z0-9-]+/gi, '-');
   const copy = path.join(o.dir || os.tmpdir(),
@@ -80,9 +130,11 @@ function copyDatabase(o = {}) {
   removeAll(copy);
 
   // 2. ...then the live database and whichever sidecars it actually has.
-  for (const suffix of SUFFIXES) {
-    const src = live + suffix;
-    if (fs.existsSync(src)) fs.copyFileSync(src, copy + suffix);
+  if (!missing) {
+    for (const suffix of SUFFIXES) {
+      const src = live + suffix;
+      if (fs.existsSync(src)) fs.copyFileSync(src, copy + suffix);
+    }
   }
 
   // 3. And take it away afterwards. Best effort by nature: a process that is killed runs no exit
@@ -91,9 +143,7 @@ function copyDatabase(o = {}) {
     process.on('exit', () => {
       // CLOSE BEFORE DELETING, where the caller can. Windows refuses to unlink an open file, and
       // this handler runs with the database still open unless something closes it.
-      if (typeof o.close === 'function') {
-        try { o.close(); } catch (e) { /* already closed, or never opened */ }
-      }
+      closeSynchronously(o.close, copy);
       removeAll(copy);
     });
   }
@@ -118,9 +168,7 @@ function freshDatabase(o = {}) {
   removeAll(file);
   if (o.cleanup !== false) {
     process.on('exit', () => {
-      if (typeof o.close === 'function') {
-        try { o.close(); } catch (e) { /* already closed, or never opened */ }
-      }
+      closeSynchronously(o.close, file);
       removeAll(file);
     });
   }