@aria-framework/testkit 0.1.0 → 0.3.0

package/index.js

@@ -1,21 +1,25 @@
-/**
- * @aria-framework/testkit — test infrastructure for the aria apps. A DEV DEPENDENCY: nothing here
- * belongs in a production install, which is why it is its own package rather than part of kit.
- *
- *   createHarness()                     — the ✓/✗ micro-harness (~111 hand-rolled copies replaced);
- *                                         check() refuses async callbacks, acheck() awaits them,
- *                                         done() prints the summary and owns process.exit.
- *   assertImportsInScope({...})         — every consumer of a module has every function it calls
- *                                         in scope (the silently-empty-list-in-a-catch class).
- *   assertVersionSingleSource({...})    — package.json vs derived version files vs the lockfile's
- *                                         two copies.
- */
-
-'use strict';
-
-module.exports = Object.assign(
-  {},
-  require('./harness'),
-  require('./importsInScope'),
-  require('./versionSync')
-);
+/**
+ * @aria-framework/testkit — test infrastructure for the aria apps. A DEV DEPENDENCY: nothing here
+ * belongs in a production install, which is why it is its own package rather than part of kit.
+ *
+ *   createHarness()                     — the ✓/✗ micro-harness (~111 hand-rolled copies replaced);
+ *                                         check() refuses async callbacks, acheck() awaits them,
+ *                                         done() prints the summary and owns process.exit.
+ *   assertImportsInScope({...})         — every consumer of a module has every function it calls
+ *                                         in scope (the silently-empty-list-in-a-catch class).
+ *   assertVersionSingleSource({...})    — package.json vs derived version files vs the lockfile's
+ *                                         two copies.
+ */
+
+'use strict';
+
+module.exports = Object.assign(
+  {},
+  require('./harness'),
+  require('./importsInScope'),
+  require('./versionSync'),
+  // copyDatabase / assertIsCopy / sweep — a private copy of the app's database for one test run,
+  // which clears whatever an earlier run with this pid left behind and takes itself away
+  // afterwards. See tempDb.js for the flake that made it necessary.
+  require('./tempDb')
+);

package/package.json

@@ -1,20 +1,21 @@
-{
-  "name": "@aria-framework/testkit",
-  "description": "Aria App Framework — test infrastructure. The ✓/✗ micro test-harness (createHarness, with the async-callback guard that stops a suite silently passing), assertImportsInScope (every consumer of a module has every function it calls in scope), and assertVersionSingleSource (package.json vs derived version files vs the lockfile's two copies). Dev-dependency only: nothing here belongs in a production install.",
-  "version": "0.1.0",
-  "license": "UNLICENSED",
-  "private": false,
-  "publishConfig": {
-    "access": "public"
-  },
-  "main": "index.js",
-  "files": [
-    "index.js",
-    "harness.js",
-    "importsInScope.js",
-    "versionSync.js"
-  ],
-  "scripts": {
-    "test": "node test/smoke.js"
-  }
-}
+{
+  "name": "@aria-framework/testkit",
+  "description": "Aria App Framework \u2014 test infrastructure. The \u2713/\u2717 micro test-harness (createHarness, with the async-callback guard that stops a suite silently passing), assertImportsInScope (every consumer of a module has every function it calls in scope), and assertVersionSingleSource (package.json vs derived version files vs the lockfile's two copies). Dev-dependency only: nothing here belongs in a production install.",
+  "version": "0.3.0",
+  "license": "UNLICENSED",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "index.js",
+  "files": [
+    "index.js",
+    "harness.js",
+    "importsInScope.js",
+    "versionSync.js",
+    "tempDb.js"
+  ],
+  "scripts": {
+    "test": "node test/smoke.js && node test/tempDb.js"
+  }
+}

package/tempDb.js

@@ -0,0 +1,215 @@
+/**
+ * A private copy of the app's database, for one test run, that takes itself away afterwards.
+ *
+ * ── WHY A COPY AT ALL ───────────────────────────────────────────────────────────────────────────
+ * A test that opens the live database can destroy the developer's own data, and worse, can pass for
+ * the wrong reason: it sees rows nobody put there. Every suite here therefore works on a copy — and
+ * the copy is named after the process id so two runs cannot fight over one file.
+ *
+ * ── THE BUG THIS EXISTS TO END ──────────────────────────────────────────────────────────────────
+ * That naming scheme has a tail. The file is never removed, so leftovers accumulate — one repository
+ * reached 15,598 of them — and eventually the OS hands out a pid whose file is still lying there
+ * from a run weeks ago. The naive copy loop only writes the sidecars that exist for the LIVE
+ * database, so a stale `-wal` from that earlier run survives beside the fresh `.db`, and SQLite
+ * dutifully replays it: the previous run's rows come back inside what is supposed to be a clean
+ * copy. It surfaced as `UNIQUE constraint failed` on an id also keyed to the pid, and as
+ * `malformed database schema (ai_usage) - invalid rootpage`.
+ *
+ * It fails perhaps one run in a hundred and passes on an immediate retry with a different pid,
+ * which is the worst shape a flake can have: rare enough to be dismissed, frequent enough to erode
+ * trust in every other failure the suite reports.
+ *
+ * So two things, and the FIRST is the one that actually fixes it:
+ *
+ *   1. CLEAR THE TARGET FIRST, sidecars included, before copying anything onto it. This is the fix.
+ *      It holds however the previous run ended — killed, crashed, or tidy — because it makes no
+ *      assumption about cleanup having happened at all.
+ *   2. Remove it on exit, so the pool of leftovers stops growing. This is housekeeping, and it is
+ *      BEST EFFORT BY NECESSITY.
+ *
+ * ── WHY CLEANUP CANNOT BE RELIED ON ─────────────────────────────────────────────────────────────
+ * On Windows, unlinking a file that is still open fails with EBUSY, and an exit handler runs while
+ * the driver still holds the database. So a test that never closes its connection — which is most
+ * of them, because the process is about to end anyway — leaves its copy behind no matter what is
+ * registered. Pass `close` to fix that for one caller; run `sweep()` from a pretest step to bound
+ * the pool for all of them. Neither is load-bearing: step 1 is.
+ *
+ * ── WHAT THIS DOES NOT DO ───────────────────────────────────────────────────────────────────────
+ * It does not open the database. Which driver, which migrations and which assertions belong to the
+ * app; this only decides where the file goes and guarantees it is clean and temporary.
+ */
+
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+
+/** SQLite writes two sidecars beside the database, and both carry state. */
+const SUFFIXES = ['', '-wal', '-shm'];
+
+function removeAll(base) {
+  for (const suffix of SUFFIXES) {
+    try { fs.unlinkSync(base + suffix); } catch (e) { /* absent, or held open elsewhere */ }
+  }
+}
+
+/**
+ * Run a caller's close hook, and refuse to be lied to about it.
+ *
+ * AN EXIT HANDLER CANNOT AWAIT. So a `close` that returns a promise has not closed anything by the
+ * time the unlink runs, and on Windows that unlink then fails with EBUSY into a catch nobody reads.
+ * The copy is left behind and the suite still passes — which is precisely the shape of bug this
+ * whole module exists to end, arriving through the mechanism meant to prevent it.
+ *
+ * It is worse than a plain failure because an async close can APPEAR to work: `async close() {
+ * db.close(); }` runs its body up to the first await synchronously, so a driver written that way
+ * cleans up perfectly until somebody adds an await ahead of that line, or the app moves to a driver
+ * whose close is genuinely asynchronous. Then every leftover returns, silently, and the commit that
+ * introduced the regression is nowhere near the code that breaks.
+ *
+ * So: say so, loudly, on the one occasion it can still be heard.
+ */
+function closeSynchronously(close, file) {
+  if (typeof close !== 'function') return;
+  let result;
+  try {
+    result = close();
+  } catch (e) {
+    // A synchronous throw is fine — already closed, or never opened. The delete still runs.
+    return;
+  }
+  if (result && typeof result.then === 'function') {
+    // Not swallowed: this is the only moment anybody could learn about it.
+    // eslint-disable-next-line no-console
+    console.error(
+      `testkit: the close hook for ${path.basename(file)} returned a promise. An exit handler `
+      + 'cannot await, so the database is still open when the file is deleted — on Windows that '
+      + 'fails silently and the copy is left behind. Pass a SYNCHRONOUS close: '
+      + 'the driver' + String.fromCharCode(39) + 's own db.close() usually is one, even when the wrapper is not.');
+    // ...and do not let it become an unhandled rejection on the way out.
+    result.then(undefined, () => {});
+  }
+}
+
+/**
+ * Copy `livePath` to a private temp file and return its path.
+ *
+ * @param {object} o
+ *   livePath  the database to copy (required)
+ *   name      a short label, so leftovers can be traced to the test that made them
+ *   dir       where to put it; defaults to the OS temp directory
+ *   prefix    default 'app' — apps pass their own so two projects cannot collide
+ *   cleanup   default true; false leaves the copy behind for inspection after a failure
+ *   allowMissing  default false; true means an absent live database copies nothing rather
+ *             than throwing, for a test that builds its schema from migrations
+ *   close     optional () => void, run before deleting. On Windows an open handle makes unlink
+ *             fail with EBUSY, so without this the exit hook is a no-op for any test that does not
+ *             close its database — which is most of them.
+ */
+function copyDatabase(o = {}) {
+  const live = o.livePath;
+  if (!live) throw new Error('copyDatabase({ livePath }): the database to copy is required');
+  // STRICT BY DEFAULT. A test that quietly ran against an empty database when it meant to use real
+  // data is its own kind of wrong, and silent is the worst way for that to happen. But a suite that
+  // BUILDS its schema from migrations is legitimately fine without one, and the hand-rolled loop
+  // this replaced tolerated it — `if (exists) copy` simply copied nothing. On a fresh clone, or a CI
+  // box that has never booted the app, strictness alone turns those tests into a module-scope crash.
+  const missing = !fs.existsSync(live);
+  if (missing && !o.allowMissing) {
+    throw new Error(
+      `copyDatabase: there is no database at ${live}. Pass allowMissing:true if this test builds `
+      + 'its own schema and does not need one.');
+  }
+
+  const label = String(o.name || 'test').replace(/[^a-z0-9-]+/gi, '-');
+  const copy = path.join(o.dir || os.tmpdir(),
+    `${o.prefix || 'app'}-${label}-${process.pid}.db`);
+
+  // 1. THE TARGET, GONE — including sidecars a previous run with this pid may have left.
+  removeAll(copy);
+
+  // 2. ...then the live database and whichever sidecars it actually has.
+  if (!missing) {
+    for (const suffix of SUFFIXES) {
+      const src = live + suffix;
+      if (fs.existsSync(src)) fs.copyFileSync(src, copy + suffix);
+    }
+  }
+
+  // 3. And take it away afterwards. Best effort by nature: a process that is killed runs no exit
+  //    handler at all, which is exactly why step 1 cannot be skipped.
+  if (o.cleanup !== false) {
+    process.on('exit', () => {
+      // CLOSE BEFORE DELETING, where the caller can. Windows refuses to unlink an open file, and
+      // this handler runs with the database still open unless something closes it.
+      closeSynchronously(o.close, copy);
+      removeAll(copy);
+    });
+  }
+
+  return copy;
+}
+
+/**
+ * A path for a database this test will BUILD, rather than copy — cleared and temporary.
+ *
+ * The same pid-keyed name, and therefore the same trap by a shorter route: a test that runs
+ * migrations onto a fresh file will happily do so beside a `-wal` left by an earlier run with this
+ * pid, and SQLite replays it into the new database. The main file being absent is no protection at
+ * all — it is the SIDECAR that carries the old rows.
+ *
+ * Returns the path. Nothing is created here; the caller's driver does that.
+ */
+function freshDatabase(o = {}) {
+  const label = String(o.name || 'test').replace(/[^a-z0-9-]+/gi, '-');
+  const file = path.join(o.dir || os.tmpdir(),
+    `${o.prefix || 'app'}-${label}-${process.pid}.db`);
+  removeAll(file);
+  if (o.cleanup !== false) {
+    process.on('exit', () => {
+      closeSynchronously(o.close, file);
+      removeAll(file);
+    });
+  }
+  return file;
+}
+
+/**
+ * Prove that what the app opened is the copy and NOT the live database.
+ *
+ * Worth its own function because the consequence of getting it wrong is silent and permanent: a
+ * suite that mutates live data looks exactly like a suite that passes.
+ *
+ * @param {string} openFile  what the driver reports it has open (PRAGMA database_list)
+ */
+function assertIsCopy(openFile, copyPath, livePath) {
+  const same = (a, b) => path.resolve(a).toLowerCase() === path.resolve(b).toLowerCase();
+  if (!openFile) throw new Error('assertIsCopy: the driver did not say which file it opened');
+  if (livePath && same(openFile, livePath)) {
+    throw new Error(`THIS IS THE LIVE DATABASE (${openFile}) — the test would mutate real data`);
+  }
+  if (!same(openFile, copyPath)) {
+    throw new Error(`opened ${openFile}, expected the copy at ${copyPath}`);
+  }
+  return true;
+}
+
+/** Sweep leftovers from earlier runs — for a housekeeping script, never for a test. */
+function sweep(o = {}) {
+  const dir = o.dir || os.tmpdir();
+  const prefix = o.prefix || 'app';
+  let removed = 0;
+  let names;
+  try { names = fs.readdirSync(dir); } catch (e) { return { removed: 0 }; }
+  for (const name of names) {
+    if (!name.startsWith(prefix + '-') || !/\.db(-wal|-shm)?$/.test(name)) continue;
+    // NEVER THIS RUN'S OWN COPY. A sweep that deletes the file the caller is using would turn a
+    // housekeeping convenience into the very corruption it exists to prevent.
+    if (name.includes('-' + process.pid + '.db')) continue;
+    try { fs.unlinkSync(path.join(dir, name)); removed += 1; } catch (e) { /* held open */ }
+  }
+  return { removed };
+}
+
+module.exports = { copyDatabase, freshDatabase, assertIsCopy, sweep, _suffixes: SUFFIXES };