@arraypress/extension-kit 1.10.0 → 1.11.1

package/src/testing/fake-storage.js

@@ -1,33 +1,65 @@
 /**
- * An in-memory `chrome.storage` for tests, behaving as Chrome does: values
- * are stored as JSON, `onChanged` fires only for values that changed, and
- * sync refuses an item over its 8,192-byte quota and more than 120 writes
- * (set or remove) a minute. Tests then check outcomes (what's stored), not
+ * An in-memory `chrome.storage` for tests, behaving as Chrome does (each point
+ * checked against Chromium): values are stored as Chrome converts them (what
+ * JSON can't hold is dropped, a `Date` is `{}`), `get` takes a key, keys, an
+ * object of defaults or nothing, `onChanged` fires only for values that
+ * changed (whatever order an object's keys come in), and every area has
+ * Chrome's quotas. Sync refuses an item over 8,192 bytes as Chrome counts them
+ * (`syncItemBytes`: UTF-8, `<` six bytes), more than 102,400 bytes or 512
+ * items in all, and more than 120 calls a minute or 1,800 an hour of `set`,
+ * `remove` or `clear` (each counted by itself, refused calls included); local
+ * and session hold 10 MB. Tests then check outcomes (what's stored), not
  * calls.
  *
  * @module testing/fake-storage
  */
 
-import { MAX_SYNC_ITEM_BYTES } from '../settings/storage.js';
+import { MAX_SYNC_ITEM_BYTES, sameValue, syncItemBytes } from '../settings/storage.js';
 
 /** @import { StorageChange } from '../settings/storage.js' */
 
-/** Chrome Sync's write limit: `chrome.storage.sync.MAX_WRITE_OPERATIONS_PER_MINUTE`. */
+/** Chrome Sync's write limit a minute: `chrome.storage.sync.MAX_WRITE_OPERATIONS_PER_MINUTE`. */
 export const SYNC_WRITES_PER_MINUTE = 120;
 
+/** Chrome Sync's write limit an hour: `chrome.storage.sync.MAX_WRITE_OPERATIONS_PER_HOUR`. */
+export const SYNC_WRITES_PER_HOUR = 1800;
+
+/** All of sync's items together may take this many bytes (`chrome.storage.sync.QUOTA_BYTES`). */
+export const SYNC_QUOTA_BYTES = 102_400;
+
+/** The most items sync holds (`chrome.storage.sync.MAX_ITEMS`). */
+export const SYNC_MAX_ITEMS = 512;
+
+/** What local and session each hold (`chrome.storage.local.QUOTA_BYTES`), without `unlimitedStorage`. */
+export const LOCAL_QUOTA_BYTES = 10_485_760;
+
 /**
  * @typedef {(changes: Record<string, StorageChange>, area: string) => unknown} ChangeListener
  */
 
+/**
+ * An `onChanged` event.
+ *
+ * @template {(...args: any[]) => unknown} L
+ * @typedef {object} FakeEvent
+ * @property {(listener: L) => void} addListener
+ * @property {(listener: L) => void} removeListener
+ * @property {(listener: L) => boolean} hasListener
+ */
+
 /**
  * One fake storage area.
  *
  * @typedef {object} FakeArea
  * @property {Record<string, any>} data What it holds, to inspect (and seed) in tests.
- * @property {number} writes How many writes it has taken.
- * @property {(keys?: string | string[] | null) => Promise<Record<string, any>>} get
+ * @property {number} writes How many writes (`set`, `remove`, `clear`) it has taken.
+ * @property {(keys?: string | string[] | Record<string, unknown> | null) => Promise<Record<string, any>>} get
+ *   A key's or keys' items, an object's keys with its values for the missing ones, or everything.
  * @property {(items: Record<string, unknown>) => Promise<void>} set
  * @property {(keys: string | string[]) => Promise<void>} remove
+ * @property {() => Promise<void>} clear
+ * @property {(keys?: string | string[] | null) => Promise<number>} getBytesInUse The bytes the keys' items (or all) take.
+ * @property {FakeEvent<(changes: Record<string, StorageChange>) => unknown>} onChanged This area's changes.
  */
 
 /**
@@ -37,28 +69,79 @@ export const SYNC_WRITES_PER_MINUTE = 120;
  * @property {FakeArea} local `chrome.storage.local`.
  * @property {FakeArea} sync `chrome.storage.sync`, with Chrome's quotas.
  * @property {FakeArea} session `chrome.storage.session`.
- * @property {{addListener(listener: ChangeListener): void}} onChanged `chrome.storage.onChanged`.
+ * @property {FakeEvent<ChangeListener>} onChanged `chrome.storage.onChanged`.
  * @property {() => Promise<void>} settled Waits for every listener, including the changes those listeners make.
  */
 
+/** Stands for a value Chrome doesn't store (`undefined`, a function, `NaN`). */
+const DROPPED = Symbol('dropped');
+
 /**
- * Copy a value the way storage does, through JSON.
+ * A value as Chrome stores it: what JSON holds, copied; `undefined`, functions, symbols and
+ * numbers that aren't finite dropped from objects (`null` in arrays); any other object (a
+ * `Date`, a `RegExp`, a `Map`) as `{}`.
  *
- * @param {unknown} value - Anything JSON-able, or undefined.
+ * @param {unknown} value - What an extension gave `set`.
+ * @returns {unknown} The stored copy, or {@link DROPPED}.
+ */
+function asStored(value) {
+  if (value === null || typeof value === 'string' || typeof value === 'boolean') return value;
+  if (typeof value === 'number') return Number.isFinite(value) ? value : DROPPED;
+  if (typeof value !== 'object') return DROPPED;
+  if (Array.isArray(value)) {
+    return value.map((item) => {
+      const stored = asStored(item);
+      return stored === DROPPED ? null : stored;
+    });
+  }
+  const prototype = Object.getPrototypeOf(value);
+  if (prototype !== Object.prototype && prototype !== null) return {};
+  return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, asStored(inner)]).filter(([, inner]) => inner !== DROPPED));
+}
+
+/**
+ * A copy of what's stored, as `get` and `onChanged` hand it out.
+ *
+ * @param {unknown} value - A stored value, or undefined.
  * @returns {any} The copy.
  */
-const viaJson = (value) => (value === undefined ? undefined : JSON.parse(JSON.stringify(value)));
+const copied = (value) => (value === undefined ? undefined : structuredClone(value));
 
-/** All of sync's items together may take this many bytes (`chrome.storage.sync.QUOTA_BYTES`). */
-export const SYNC_QUOTA_BYTES = 102_400;
+/**
+ * An event's listeners.
+ *
+ * @template {(...args: any[]) => unknown} L
+ * @returns {FakeEvent<L> & {listeners: L[]}} The event, and who listens.
+ */
+function fakeEvent() {
+  /** @type {L[]} */
+  const listeners = [];
+  return {
+    listeners,
+    addListener: (listener) => { if (!listeners.includes(listener)) listeners.push(listener); },
+    removeListener: (listener) => { if (listeners.includes(listener)) listeners.splice(listeners.indexOf(listener), 1); },
+    hasListener: (listener) => listeners.includes(listener),
+  };
+}
 
 /**
- * The bytes an area's items take, as Chrome counts sync's: each key plus its value's JSON, in UTF-8.
+ * A limit on calls in a window of time, as Chrome's quota service keeps one: the window opens
+ * with the first call and takes so many; the first call after it has passed opens the next.
  *
- * @param {Record<string, unknown>} items - The items.
- * @returns {number} Bytes.
+ * @param {number} most - Calls a window.
+ * @param {number} ms - The window.
+ * @param {() => number} now - The clock.
+ * @returns {() => boolean} Counts a call; false when it's over the limit.
  */
-const totalBytes = (items) => Object.entries(items).reduce((sum, [key, value]) => sum + new TextEncoder().encode(key + JSON.stringify(value)).length, 0);
+function callLimit(most, ms, now) {
+  let opened = -Infinity;
+  let left = 0;
+  return () => {
+    if (now() >= opened + ms) [opened, left] = [now(), most];
+    left -= 1;
+    return left >= 0;
+  };
+}
 
 /**
  * Make a fake `chrome.storage`.
@@ -67,71 +150,89 @@ const totalBytes = (items) => Object.entries(items).reduce((sum, [key, value]) =
  * @param {Record<string, unknown>} [initial.local] - What local holds.
  * @param {Record<string, unknown>} [initial.sync] - What sync holds.
  * @param {Record<string, unknown>} [initial.session] - What session holds.
- * @param {() => number} [initial.now] - The clock sync's write limit reads. Default `Date.now`.
+ * @param {() => number} [initial.now] - The clock sync's write limits read. Default `Date.now`.
  * @returns {FakeChromeStorage} The fake.
+ *
  * @example
  * const storage = fakeChromeStorage({ sync: { days: 3 } });
  * await store.save(storage, { days: 5 });
  * assert.equal(storage.sync.data.days, 5);
  */
 export function fakeChromeStorage({ local = {}, sync = {}, session = {}, now = Date.now } = {}) {
-  /** @type {ChangeListener[]} */
-  const listeners = [];
+  /** @type {ReturnType<typeof fakeEvent<ChangeListener>>} */
+  const onChanged = fakeEvent();
   /** @type {unknown[]} */
   const running = [];
 
   /**
    * @param {string} name
    * @param {Record<string, unknown>} initialData
-   * @param {{quotaPerItem?: number, quotaBytes?: number, writesPerMinute?: number}} [limits]
+   * @param {{quotaPerItem?: number, quotaBytes: number, maxItems?: number, limited?: boolean}} limits
    * @returns {FakeArea}
    */
-  function area(name, initialData, { quotaPerItem, quotaBytes, writesPerMinute } = {}) {
+  function area(name, initialData, { quotaPerItem, quotaBytes, maxItems, limited = false }) {
     /** @type {Record<string, any>} */
-    const data = viaJson(initialData);
-    /** @type {number[]} */
-    const recent = [];
+    const data = /** @type {Record<string, any>} */ (asStored(initialData));
+    /** @type {ReturnType<typeof fakeEvent<(changes: Record<string, StorageChange>) => unknown>>} */
+    const areaChanged = fakeEvent();
+    /** Each function's own limits, as Chrome counts them. @type {Map<string, (() => boolean)[]>} */
+    const limits = new Map();
     /** @param {Record<string, StorageChange>} changes */
     const fire = (changes) => {
       if (!Object.keys(changes).length) return;
-      for (const listener of listeners) running.push(listener(viaJson(changes), name));
+      for (const listener of [...onChanged.listeners]) running.push(listener(copied(changes), name));
+      for (const listener of [...areaChanged.listeners]) running.push(listener(copied(changes)));
     };
-    /** A write, counted against the per-minute limit; refused over it, as Chrome does. */
-    const take = () => {
-      if (writesPerMinute) {
-        while (recent.length && recent[0] <= now() - 60_000) recent.shift();
-        if (recent.length >= writesPerMinute) throw new Error('This request exceeds the MAX_WRITE_OPERATIONS_PER_MINUTE quota.');
-        recent.push(now());
-      }
-      fake.writes++;
+    /**
+     * A call counted against the function's limits (a minute's, an hour's); refused over one, as Chrome does.
+     *
+     * @param {string} method - `set`, `remove` or `clear`.
+     */
+    const count = (method) => {
+      if (!limited) return;
+      if (!limits.has(method)) limits.set(method, [callLimit(SYNC_WRITES_PER_MINUTE, 60_000, now), callLimit(SYNC_WRITES_PER_HOUR, 3_600_000, now)]);
+      const [minute, hour] = /** @type {(() => boolean)[]} */ (limits.get(method)).map((take) => take());
+      if (!minute) throw new Error('This request exceeds the MAX_WRITE_OPERATIONS_PER_MINUTE quota.');
+      if (!hour) throw new Error('This request exceeds the MAX_WRITE_OPERATIONS_PER_HOUR quota.');
     };
+    /** @param {Record<string, unknown>} items @returns {number} */
+    const bytes = (items) => Object.entries(items).reduce((sum, [key, value]) => sum + syncItemBytes(key, value), 0);
     /** @type {FakeArea} */
     const fake = {
       data,
       writes: 0,
+      onChanged: areaChanged,
       async get(keys) {
+        const defaults = keys !== null && typeof keys === 'object' && !Array.isArray(keys) ? /** @type {Record<string, unknown>} */ (asStored(keys)) : {};
+        const wanted = keys == null ? Object.keys(data) : typeof keys === 'string' ? [keys] : Array.isArray(keys) ? keys : Object.keys(keys);
+        return Object.fromEntries(wanted.filter((key) => key in data || key in defaults).map((key) => [key, copied(key in data ? data[key] : defaults[key])]));
+      },
+      async getBytesInUse(keys) {
         const wanted = keys == null ? Object.keys(data) : [keys].flat();
-        return Object.fromEntries(wanted.filter((key) => key in data).map((key) => [key, viaJson(data[key])]));
+        return bytes(Object.fromEntries(wanted.filter((key) => key in data).map((key) => [key, data[key]])));
       },
       async set(items) {
-        for (const [key, value] of Object.entries(items)) {
-          if (quotaPerItem && key.length + JSON.stringify(value).length > quotaPerItem) {
-            throw new Error('Resource::kQuotaBytesPerItem quota exceeded');
-          }
+        count('set');
+        const stored = /** @type {Record<string, unknown>} */ (asStored(items));
+        for (const [key, value] of Object.entries(stored)) {
+          if (quotaPerItem && syncItemBytes(key, value) > quotaPerItem) throw new Error('Resource::kQuotaBytesPerItem quota exceeded');
         }
-        if (quotaBytes && totalBytes({ ...data, ...items }) > quotaBytes) throw new Error('Resource::kQuotaBytes quota exceeded');
-        take();
+        const next = { ...data, ...stored };
+        if (bytes(next) > quotaBytes) throw new Error('Resource::kQuotaBytes quota exceeded');
+        if (maxItems && Object.keys(next).length > maxItems) throw new Error('Resource::kMaxItems quota exceeded');
+        fake.writes++;
         /** @type {Record<string, StorageChange>} */
         const changes = {};
-        for (const [key, value] of Object.entries(items)) {
-          if (JSON.stringify(data[key]) === JSON.stringify(value)) continue;
-          changes[key] = { oldValue: data[key], newValue: viaJson(value) };
-          data[key] = viaJson(value);
+        for (const [key, value] of Object.entries(stored)) {
+          if (key in data && sameValue(data[key], value)) continue;
+          changes[key] = { ...(key in data ? { oldValue: data[key] } : {}), newValue: value };
+          data[key] = value;
         }
         fire(changes);
       },
       async remove(keys) {
-        take();
+        count('remove');
+        fake.writes++;
         /** @type {Record<string, StorageChange>} */
         const changes = {};
         for (const key of [keys].flat()) {
@@ -141,15 +242,26 @@ export function fakeChromeStorage({ local = {}, sync = {}, session = {}, now = D
         }
         fire(changes);
       },
+      async clear() {
+        count('clear');
+        fake.writes++;
+        /** @type {Record<string, StorageChange>} */
+        const changes = {};
+        for (const key of Object.keys(data)) {
+          changes[key] = { oldValue: data[key] };
+          delete data[key];
+        }
+        fire(changes);
+      },
     };
     return fake;
   }
 
   return {
-    local: area('local', local),
-    sync: area('sync', sync, { quotaPerItem: MAX_SYNC_ITEM_BYTES, quotaBytes: SYNC_QUOTA_BYTES, writesPerMinute: SYNC_WRITES_PER_MINUTE }),
-    session: area('session', session),
-    onChanged: { addListener: (listener) => listeners.push(listener) },
+    local: area('local', local, { quotaBytes: LOCAL_QUOTA_BYTES }),
+    sync: area('sync', sync, { quotaPerItem: MAX_SYNC_ITEM_BYTES, quotaBytes: SYNC_QUOTA_BYTES, maxItems: SYNC_MAX_ITEMS, limited: true }),
+    session: area('session', session, { quotaBytes: LOCAL_QUOTA_BYTES }),
+    onChanged,
     async settled() {
       while (running.length) await Promise.all(running.splice(0));
     },

package/src/testing/index.d.ts

@@ -8,9 +8,9 @@
  */
 export { TEST_BUILDS_ENV, testBuildDir } from '../build/test-build.js';
 export { chromeUserAgent, closeOnSignal, realisticBrowser } from './browser.js';
-export { CHECK_MS, checkRunner, checkToolbarTheme, extensionProfile, launchExtension, offscreenEmulator, recordIconCalls, runChecks, until, } from './e2e.js';
+export { CHECK_MS, checkRunner, checkToolbarTheme, extensionProfile, launchExtension, offscreenEmulator, recordIconCalls, runChecks, stopWorker, until, } from './e2e.js';
 export { exitPoint, parseLiveOptions, parseProxy } from './live.js';
-export { liveUsage, runLiveCheck } from './live-run.js';
+export { REPORT_MARKER, liveUsage, reportFolder, runLiveCheck } from './live-run.js';
 export { eachLimited } from '../time/pool.js';
 export { filesIn, launchDownloading } from './downloads.js';
 export { fakeLanguageModel } from './fake-model.js';
@@ -19,7 +19,7 @@ export { heartbeat } from './heartbeat.js';
 export { playPermissionPrompt } from './prompt.js';
 export { callTabModule, nodeAttribute, nodeText, pierceNodes, tabIdOf } from './pages.js';
 export { serve, testCertificate } from './server.js';
-export { SYNC_QUOTA_BYTES, SYNC_WRITES_PER_MINUTE, fakeChromeStorage } from './fake-storage.js';
+export { LOCAL_QUOTA_BYTES, SYNC_MAX_ITEMS, SYNC_QUOTA_BYTES, SYNC_WRITES_PER_HOUR, SYNC_WRITES_PER_MINUTE, fakeChromeStorage, } from './fake-storage.js';
 export { filesUnder, manifestReferences, missingReferences, referencedPath, zipEntries } from './package.js';
 export { savedTabGroups } from './saved-groups.js';
 export { installWarnings, warningsAddedBy, withoutPermissions } from './warnings.js';

package/src/testing/index.js

@@ -10,10 +10,10 @@
 export { TEST_BUILDS_ENV, testBuildDir } from '../build/test-build.js';
 export { chromeUserAgent, closeOnSignal, realisticBrowser } from './browser.js';
 export {
-  CHECK_MS, checkRunner, checkToolbarTheme, extensionProfile, launchExtension, offscreenEmulator, recordIconCalls, runChecks, until,
+  CHECK_MS, checkRunner, checkToolbarTheme, extensionProfile, launchExtension, offscreenEmulator, recordIconCalls, runChecks, stopWorker, until,
 } from './e2e.js';
 export { exitPoint, parseLiveOptions, parseProxy } from './live.js';
-export { liveUsage, runLiveCheck } from './live-run.js';
+export { REPORT_MARKER, liveUsage, reportFolder, runLiveCheck } from './live-run.js';
 export { eachLimited } from '../time/pool.js';
 export { filesIn, launchDownloading } from './downloads.js';
 export { fakeLanguageModel } from './fake-model.js';
@@ -22,7 +22,9 @@ export { heartbeat } from './heartbeat.js';
 export { playPermissionPrompt } from './prompt.js';
 export { callTabModule, nodeAttribute, nodeText, pierceNodes, tabIdOf } from './pages.js';
 export { serve, testCertificate } from './server.js';
-export { SYNC_QUOTA_BYTES, SYNC_WRITES_PER_MINUTE, fakeChromeStorage } from './fake-storage.js';
+export {
+  LOCAL_QUOTA_BYTES, SYNC_MAX_ITEMS, SYNC_QUOTA_BYTES, SYNC_WRITES_PER_HOUR, SYNC_WRITES_PER_MINUTE, fakeChromeStorage,
+} from './fake-storage.js';
 export { filesUnder, manifestReferences, missingReferences, referencedPath, zipEntries } from './package.js';
 export { savedTabGroups } from './saved-groups.js';
 export { installWarnings, warningsAddedBy, withoutPermissions } from './warnings.js';

package/src/testing/live-run.d.ts

@@ -80,10 +80,31 @@ export type LiveRun<S extends LiveSite, R> = {
  * @returns {string} The text.
  */
 export declare const liveUsage: (categories: string[], concurrency: number) => string;
+/** The file that marks a folder as a live report the kit made, so the next run may replace it. */
+export declare const REPORT_MARKER = ".live-report";
+/**
+ * The folder a run's report goes in, made empty. Only a folder the kit made is ever emptied:
+ * the checker's own (`own`), or one carrying {@link REPORT_MARKER} from an earlier run. A new
+ * or empty folder is taken (and marked); a folder with anything else in it is refused, so
+ * `--out .` or `--out ~/Desktop` can't delete what's there.
+ *
+ * @param {string} dir - The folder.
+ * @param {{own?: boolean}} [options] - `own`: the checker's default folder, replaced without asking.
+ * @returns {Promise<string>} `dir`, empty but for the marker.
+ * @throws {Error} When the folder holds files and isn't a report folder.
+ *
+ * @example
+ * await reportFolder('/tmp/report'); // made, or emptied if an earlier run made it
+ */
+export declare function reportFolder(dir: string, { own }?: {
+    own?: boolean;
+}): Promise<string>;
 /**
  * Run a live check: read the command line, visit the chosen sites, give back every visit.
  * Prints the usage and exits for `--help` (or an option it doesn't know), and exits when no
- * site matches.
+ * site matches or `--out` names a folder with other files in it ({@link reportFolder}). A visit
+ * that throws, runs out of time or loses its browser is `failed` with why, and the run goes on
+ * (the slot gets a new browser).
  *
  * @template {LiveSite} S
  * @template R
@@ -96,7 +117,7 @@ export declare const liveUsage: (categories: string[], concurrency: number) => s
  * @param {string[]} run.categories - The categories' ids, for the usage.
  * @param {number} run.pageMs - The most a visit may take.
  * @param {(browser: LaunchedExtension, page: Page, site: S) => Promise<R>} run.visit - Check one site, on a fresh page.
- * @param {(site: S, error: string) => R} run.failed - A visit that couldn't finish (`'timed out'`).
+ * @param {(site: S, error: string) => R} run.failed - A visit that couldn't finish: `'timed out'`, or the error's message.
  * @param {(result: {site: S, visit: R}) => void} [run.onResult] - Each visit as it finishes (a line of output).
  * @param {{argv?: string[], env?: Record<string, string | undefined>, concurrency?: number, findExit?: typeof exitPoint}} [run.command] - The
  *   arguments and environment (default the process's), the most sites at once (default 4), and how the exit point

package/src/testing/live-run.js

@@ -10,9 +10,9 @@
  * @module testing/live-run
  */
 
-import { mkdir, rm } from 'node:fs/promises';
+import { mkdir, readdir, rm, writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import { within } from '../time/time.js';
+import { deadline, within } from '../time/time.js';
 import { eachLimited } from '../time/pool.js';
 import { chromeUserAgent, closeOnSignal, realisticBrowser } from './browser.js';
 import { launchExtension } from './e2e.js';
@@ -60,10 +60,43 @@ export const liveUsage = (categories, concurrency) => `Usage: npm run live -- [o
   --out <dir>          write the report there (default test/live/report)
   --help`;
 
+/** The file that marks a folder as a live report the kit made, so the next run may replace it. */
+export const REPORT_MARKER = '.live-report';
+
+/**
+ * The folder a run's report goes in, made empty. Only a folder the kit made is ever emptied:
+ * the checker's own (`own`), or one carrying {@link REPORT_MARKER} from an earlier run. A new
+ * or empty folder is taken (and marked); a folder with anything else in it is refused, so
+ * `--out .` or `--out ~/Desktop` can't delete what's there.
+ *
+ * @param {string} dir - The folder.
+ * @param {{own?: boolean}} [options] - `own`: the checker's default folder, replaced without asking.
+ * @returns {Promise<string>} `dir`, empty but for the marker.
+ * @throws {Error} When the folder holds files and isn't a report folder.
+ *
+ * @example
+ * await reportFolder('/tmp/report'); // made, or emptied if an earlier run made it
+ */
+export async function reportFolder(dir, { own = false } = {}) {
+  const entries = await readdir(dir).catch(() => /** @type {string[]} */ ([]));
+  if (entries.length && !own && !entries.includes(REPORT_MARKER)) {
+    throw new Error(`${dir} has files in it and isn't a report folder: choose a new or empty folder for --out`);
+  }
+  if (entries.length) await rm(dir, { recursive: true, force: true });
+  await mkdir(dir, { recursive: true });
+  await writeFile(path.join(dir, REPORT_MARKER), 'A live check\'s report: the next run replaces this folder.\n');
+  return dir;
+}
+
+/** Stands for a visit that ran out of time. */
+const LATE = Symbol('late');
+
 /**
  * Run a live check: read the command line, visit the chosen sites, give back every visit.
  * Prints the usage and exits for `--help` (or an option it doesn't know), and exits when no
- * site matches.
+ * site matches or `--out` names a folder with other files in it ({@link reportFolder}). A visit
+ * that throws, runs out of time or loses its browser is `failed` with why, and the run goes on
+ * (the slot gets a new browser).
  *
  * @template {LiveSite} S
  * @template R
@@ -76,7 +109,7 @@ export const liveUsage = (categories, concurrency) => `Usage: npm run live -- [o
  * @param {string[]} run.categories - The categories' ids, for the usage.
  * @param {number} run.pageMs - The most a visit may take.
  * @param {(browser: LaunchedExtension, page: Page, site: S) => Promise<R>} run.visit - Check one site, on a fresh page.
- * @param {(site: S, error: string) => R} run.failed - A visit that couldn't finish (`'timed out'`).
+ * @param {(site: S, error: string) => R} run.failed - A visit that couldn't finish: `'timed out'`, or the error's message.
  * @param {(result: {site: S, visit: R}) => void} [run.onResult] - Each visit as it finishes (a line of output).
  * @param {{argv?: string[], env?: Record<string, string | undefined>, concurrency?: number, findExit?: typeof exitPoint}} [run.command] - The
  *   arguments and environment (default the process's), the most sites at once (default 4), and how the exit point
@@ -109,9 +142,14 @@ export async function runLiveCheck({ chromium, build, prefix, root, sites, categ
     console.error('No pages match.');
     process.exit(2);
   }
-  const dir = options.out ? path.resolve(options.out) : path.join(root, 'test', 'live', 'report');
-  await rm(dir, { recursive: true, force: true });
-  await mkdir(dir, { recursive: true });
+  /** @type {string} */
+  let dir;
+  try {
+    dir = await reportFolder(options.out ? path.resolve(options.out) : path.join(root, 'test', 'live', 'report'), { own: !options.out });
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : error);
+    process.exit(2);
+  }
 
   const browserOptions = realisticBrowser(options.proxy ? { proxy: options.proxy } : {});
   const plain = await chromium.launch({ channel: 'chromium', ...browserOptions });
@@ -133,19 +171,27 @@ export async function runLiveCheck({ chromium, build, prefix, root, sites, categ
     /** @type {{site: S, visit: R}[]} */
     const results = [];
     await eachLimited(chosen, options.concurrency, async (site, slot, index) => {
-      browsers[slot] ??= await launchExtension(chromium, build, { prefix: `${prefix}${slot}-`, ...launchOptions });
-      const launched = /** @type {LaunchedExtension} */ (browsers[slot]);
-      await within(launched.context.clearCookies(), 10_000, undefined);
-      const page = await launched.context.newPage();
-      const result = await within(visit(launched, page, site), pageMs, undefined);
-      const timedOut = result === undefined;
-      await within(page.close(), 15_000, undefined);
-      if (timedOut) {
-        // A page that ran out of time may have left its browser stuck: the slot gets a new one.
-        await within(launched.close(), 15_000, undefined);
+      /** @type {R} */
+      let outcome;
+      try {
+        browsers[slot] ??= await launchExtension(chromium, build, { prefix: `${prefix}${slot}-`, ...launchOptions });
+        const launched = /** @type {LaunchedExtension} */ (browsers[slot]);
+        await within(launched.context.clearCookies(), 10_000, undefined);
+        const page = await launched.context.newPage();
+        try {
+          const result = await deadline(/** @type {Promise<R | typeof LATE>} */ (visit(launched, page, site)), pageMs, (resolve) => resolve(LATE));
+          if (result === LATE) throw new Error('timed out');
+          outcome = result;
+        } finally {
+          await within(page.close(), 15_000, undefined);
+        }
+      } catch (error) {
+        // A visit that failed may have left its browser stuck, or lost it: the slot gets a new one.
+        await within(Promise.resolve(browsers[slot]?.close()), 15_000, undefined);
         browsers[slot] = null;
+        outcome = failed(site, error instanceof Error ? error.message : String(error));
       }
-      results[index] = { site, visit: timedOut ? failed(site, 'timed out') : /** @type {R} */ (result) };
+      results[index] = { site, visit: outcome };
       onResult(results[index]);
     });
     return {

package/src/testing/messages.d.ts

@@ -21,13 +21,17 @@ export type Messages = Record<string, {
  */
 export declare function readMessages(extensionDir: string, locale: string): Messages;
 /**
- * `chrome.i18n.getMessage` over a locale's real messages ('' for a missing key, as Chrome gives).
+ * `chrome.i18n.getMessage` over a locale's real messages, as Chrome reads them (checked against
+ * Chromium): names found whatever their case, named placeholders and `$1`–`$9` filled, `$$` a
+ * dollar sign, '' for a missing key, `undefined` for more than nine substitutions.
  *
  * @param {string} extensionDir - The extension's folder.
  * @param {string} [locale] - Default "en"; another locale is read over en.
- * @returns {(key: string, substitutions?: string | string[]) => string} The lookup.
+ * @returns {(key: string, substitutions?: string | number | (string | number)[]) => string} The lookup.
+ * @throws {Error} When a message names a placeholder it doesn't define (Chrome refuses to load the extension).
+ *
  * @example
  * const t = messageLookup('extension');
  * t('optionsMoreActions', ['example.com']); // 'More actions for example.com'
  */
-export declare function messageLookup(extensionDir: string, locale?: string): (key: string, substitutions?: string | string[]) => string;
+export declare function messageLookup(extensionDir: string, locale?: string): (key: string, substitutions?: string | number | (string | number)[]) => string;

package/src/testing/messages.js

@@ -2,7 +2,8 @@
  * Chrome's message lookup for an extension's unit tests: its real `_locales`
  * files, a locale's messages over `en`'s, named placeholders ("$SITE$" →
  * `{site: {content: '$1'}}`) filled from substitutions as
- * `chrome.i18n.getMessage` fills them. Node only. From the copies in Better
+ * `chrome.i18n.getMessage` fills them, and refused as Chrome refuses them (a
+ * placeholder that isn't defined). Node only. From the copies in Better
  * Prices, Better Downloads, Better Saves and Clean Links (`test/helpers/i18n.js`).
  *
  * @module testing/messages
@@ -29,23 +30,63 @@ export function readMessages(extensionDir, locale) {
   return JSON.parse(readFileSync(path.join(extensionDir, '_locales', locale, 'messages.json'), 'utf8'));
 }
 
+/** A named placeholder in a message, as Chrome reads one: `$SITE$` (letters, digits, `_`, `@`). */
+const PLACEHOLDER = /\$([A-Za-z0-9_@]+)\$/g;
+
+/**
+ * A message with its named placeholders put in, as Chrome does when it loads an extension.
+ *
+ * @param {string} key - The message's name, for the error.
+ * @param {Messages[string]} entry - The message.
+ * @returns {string} The message, `$1`-style substitutions still to fill.
+ * @throws {Error} When it names a placeholder it doesn't define: Chrome refuses to load such an extension.
+ */
+function withPlaceholders(key, entry) {
+  const contents = new Map(Object.entries(entry.placeholders ?? {}).map(([name, { content }]) => [name.toLowerCase(), content]));
+  return entry.message.replace(PLACEHOLDER, (_, name) => {
+    const content = contents.get(name.toLowerCase());
+    if (content === undefined) throw new Error(`Message “${key}” uses $${name}$ but has no such placeholder: Chrome wouldn't load the extension`);
+    return content;
+  });
+}
+
 /**
- * `chrome.i18n.getMessage` over a locale's real messages ('' for a missing key, as Chrome gives).
+ * Fill a message's substitutions as Chrome does: `$1`–`$9` from the list ('' past its end), `$$`
+ * a dollar sign; a `$` before anything else is dropped with what follows it.
+ *
+ * @param {string} message - The message, named placeholders already put in.
+ * @param {readonly string[]} subs - The substitutions.
+ * @returns {string} The text.
+ */
+function substituted(message, subs) {
+  return message.replace(/\$(\$+|[\s\S])?/g, (_, next) => {
+    if (next === undefined) return '';
+    if (next.startsWith('$')) return next;
+    return /[1-9]/.test(next) ? String(subs[Number(next) - 1] ?? '') : '';
+  });
+}
+
+/**
+ * `chrome.i18n.getMessage` over a locale's real messages, as Chrome reads them (checked against
+ * Chromium): names found whatever their case, named placeholders and `$1`–`$9` filled, `$$` a
+ * dollar sign, '' for a missing key, `undefined` for more than nine substitutions.
  *
  * @param {string} extensionDir - The extension's folder.
  * @param {string} [locale] - Default "en"; another locale is read over en.
- * @returns {(key: string, substitutions?: string | string[]) => string} The lookup.
+ * @returns {(key: string, substitutions?: string | number | (string | number)[]) => string} The lookup.
+ * @throws {Error} When a message names a placeholder it doesn't define (Chrome refuses to load the extension).
+ *
  * @example
  * const t = messageLookup('extension');
  * t('optionsMoreActions', ['example.com']); // 'More actions for example.com'
  */
 export function messageLookup(extensionDir, locale = 'en') {
-  const messages = { ...readMessages(extensionDir, 'en'), ...(locale === 'en' ? {} : readMessages(extensionDir, locale)) };
+  const read = { ...readMessages(extensionDir, 'en'), ...(locale === 'en' ? {} : readMessages(extensionDir, locale)) };
+  const messages = new Map(Object.entries(read).map(([key, entry]) => [key.toLowerCase(), withPlaceholders(key, entry)]));
   return (key, substitutions = []) => {
-    const entry = messages[key];
-    if (!entry) return '';
-    const subs = [substitutions].flat();
-    return entry.message.replace(/\$([A-Za-z_]+)\$/g, (_, name) => (entry.placeholders?.[name.toLowerCase()]?.content ?? '')
-      .replace(/\$(\d)/g, (/** @type {string} */ __, /** @type {string} */ n) => subs[Number(n) - 1] ?? ''));
+    const subs = [substitutions].flat().map(String);
+    if (subs.length > 9) return /** @type {string} */ (/** @type {unknown} */ (undefined));
+    const message = messages.get(key.toLowerCase());
+    return message === undefined ? '' : substituted(message, subs);
   };
 }