@uniflowed/test 0.0.0-alpha.8 → 0.1.0

package/internal/run.js

@@ -13,11 +13,18 @@ import * as output from "./output.js";
 import * as snapshot from "./snapshot.js";
 import { AssertionError } from "./expect.js";
 import { type Site, firstUserSite, siteInFile, userFrames } from "./frames.js";
-import { type Body, type Case, type Suite, collected } from "./registry.js";
+import { type BenchOptions, type Body, type Case, type Suite, collected } from "./registry.js";
 
 /** How one case ended. */
 export type Outcome =
-  | {| readonly status: "passed" |}
+  | {|
+      readonly status: "passed",
+      /**
+       * One timing per measured call, in whole microseconds, for a benchmark
+       * run under `uf test --bench`.
+       */
+      readonly samples?: $ReadOnlyArray<number>,
+    |}
   | {|
       readonly status: "failed",
       readonly message: string,
@@ -27,7 +34,11 @@ export type Outcome =
       /** Where the failing assertion was written, when the stack says. */
       readonly site: {| readonly line: number, readonly column: number |} | null,
     |}
-  | {| readonly status: "skipped", readonly reason: "explicit" | "not-only" | "filtered" |}
+  | {|
+      readonly status: "skipped",
+      readonly reason: "explicit" | "not-only" | "filtered" | "bench" | "not-bench",
+      readonly message?: string | null,
+    |}
   | {| readonly status: "todo" |};
 
 /** One finished case, as the runner reports it. */
@@ -52,11 +63,31 @@ export type RunOptions = {|
    * which file that is — a test's name alone does not locate it.
    */
   readonly file?: string,
+  /**
+   * Run the benchmarks and report the tests skipped, rather than the other way
+   * round. `uf test --bench` sets it.
+   */
+  readonly bench?: boolean,
 |};
 
 /** Default budget for one case, matching what most runners use. */
 export const DEFAULT_TIMEOUT_MS: number = 5000;
 
+/** Calls a benchmark makes and throws away before it times any, unless it says. */
+export const DEFAULT_BENCH_WARMUP: number = 5;
+
+/** Timed calls a benchmark makes, unless it says. */
+export const DEFAULT_BENCH_ITERATIONS: number = 50;
+
+/** Most timed calls one benchmark may ask for, and most samples `uf` keeps. */
+export const MAX_BENCH_ITERATIONS: number = 100000;
+
+/** Most untimed calls one benchmark may ask for first. */
+export const MAX_BENCH_WARMUP: number = 10000;
+
+/** The longest delay `setTimeout` honours; past it, the timer fires at once. */
+const MAX_TIMER_MS = 2147483647;
+
 /** The separator between a suite's name and its child's. */
 export const NAME_SEPARATOR: string = " > ";
 
@@ -72,7 +103,9 @@ function fullName(path: $ReadOnlyArray<string>): string {
  */
 function hasOnly(node: Suite | Case, inherited: boolean): boolean {
   const marked = inherited || node.modifier === "only";
-  if (node.kind === "test") {
+  // A benchmark is a case like a test: `bench.only` restricts a file as
+  // `it.only` does.
+  if (node.kind !== "suite") {
     return marked;
   }
   return node.children.some((child) => hasOnly(child, marked));
@@ -188,12 +221,17 @@ async function runCase(
     });
   };
 
-  if (test.modifier === "todo" || test.body == null) {
+  if (test.modifier === "todo") {
     report({ status: "todo" });
     return true;
   }
   if (context.skipped || test.modifier === "skip") {
-    report({ status: "skipped", reason: "explicit" });
+    report({ status: "skipped", reason: "explicit", message: test.skipReason });
+    return true;
+  }
+  const body = test.body;
+  if (body == null) {
+    report({ status: "todo" });
     return true;
   }
   if (!context.onlyPath) {
@@ -205,9 +243,19 @@ async function runCase(
     report({ status: "skipped", reason: "filtered" });
     return true;
   }
+  // A run is the tests or the benchmarks, never both. A benchmark timed inside
+  // an ordinary run would slow every suite that has one, beside workers busy
+  // with other files; a test inside a run of benchmarks would be timed with
+  // them.
+  const benchmark = test.kind === "bench";
+  if (benchmark !== (options.bench === true)) {
+    report({ status: "skipped", reason: benchmark ? "bench" : "not-bench" });
+    return true;
+  }
 
   const timeoutMs = test.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   let outcome: Outcome = { status: "passed" };
+  let samples: $ReadOnlyArray<number> | null = null;
   // Setup, body and teardown run *inside* the case's output context, and that
   // nesting is the whole of the fix for #207. What names a printed line is no
   // longer where the runner had got to when the line arrived — which named the
@@ -222,7 +270,11 @@ async function runCase(
       for (const hook of context.beforeEach) {
         await withTimeout(hook, timeoutMs);
       }
-      await withTimeout(test.body, timeoutMs);
+      if (benchmark) {
+        samples = await measure(body, test.bench, timeoutMs);
+      } else {
+        await withTimeout(body, timeoutMs);
+      }
     } catch (thrown) {
       outcome = failure(thrown, options.file ?? null);
     }
@@ -243,10 +295,74 @@ async function runCase(
   // is the file's, and a snapshot taken outside one fails with something better
   // than a key belonging to whichever test happened to run last.
   snapshot.exitTest();
-  report(outcome);
+  report(outcome.status === "passed" && samples != null ? { status: "passed", samples } : outcome);
   return outcome.status !== "failed";
 }
 
+/**
+ * Time a benchmark's body: `warmup` calls thrown away, then `iterations` timed.
+ *
+ * The body is called directly and awaited, with nothing else between the two
+ * readings of the clock: a timer set around every call would be timed with
+ * it, and on a body that takes microseconds that is most of the number. The
+ * budget is held around the whole loop instead, at one call's budget per
+ * call, so a benchmark that hangs still fails rather than holding the worker.
+ */
+async function measure(
+  body: Body,
+  options: BenchOptions | null,
+  timeoutMs: number,
+): Promise<$ReadOnlyArray<number>> {
+  const warmup = count("warmup", options?.warmup, DEFAULT_BENCH_WARMUP, 0, MAX_BENCH_WARMUP);
+  const iterations = count(
+    "iterations",
+    options?.iterations,
+    DEFAULT_BENCH_ITERATIONS,
+    1,
+    MAX_BENCH_ITERATIONS,
+  );
+  const rounds = warmup + iterations;
+  const samples: Array<number> = [];
+  await withTimeout(
+    async () => {
+      for (let round = 0; round < rounds; round += 1) {
+        const started = performance.now();
+        await body();
+        if (round >= warmup) {
+          samples.push(Math.round((performance.now() - started) * 1000));
+        }
+      }
+    },
+    Math.min(timeoutMs * rounds, MAX_TIMER_MS),
+  );
+  return samples;
+}
+
+/**
+ * A benchmark option that has to be a whole number from `least` to `most`.
+ *
+ * Refused rather than clamped: `iterations: 0` or `warmup: 1e9` is a mistake,
+ * and a benchmark quietly run some other number of times reports numbers
+ * about a run nobody asked for.
+ */
+function count(
+  option: string,
+  value: ?number,
+  fallback: number,
+  least: number,
+  most: number,
+): number {
+  if (value == null) {
+    return fallback;
+  }
+  if (!Number.isInteger(value) || value < least || value > most) {
+    throw new Error(
+      `bench \`${option}\` has to be a whole number from ${least} to ${most}, and was ${String(value)}`,
+    );
+  }
+  return value;
+}
+
 /**
  * Walk one suite, running what it contains.
  *
@@ -297,13 +413,16 @@ async function runSuite(
     if (state.bail) {
       break;
     }
-    if (child.kind === "test") {
+    if (child.kind !== "suite") {
       const willRun =
         !inner.skipped &&
         child.modifier !== "skip" &&
         child.modifier !== "todo" &&
         child.body != null &&
-        (!onlyMode || inner.onlyPath || child.modifier === "only");
+        (!onlyMode || inner.onlyPath || child.modifier === "only") &&
+        // A benchmark in a run of the tests, or a test in a run of the
+        // benchmarks, is reported skipped and sets nothing up.
+        (child.kind === "bench") === (options.bench === true);
       if (willRun) {
         try {
           await setUpOnce();
@@ -315,7 +434,7 @@ async function runSuite(
             line: child.line,
             column: child.column,
             durationMicros: 0,
-            outcome: failure(thrown),
+            outcome: failure(thrown, options.file ?? null),
           });
           passed = false;
           continue;

package/internal/timers.js

@@ -82,7 +82,7 @@ function host(): $FlowFixMe {
  * formats a date sees a plausible one — and `setSystemTime` is how a test that
  * cares says which.
  */
-export function useFakeTimers(): void {
+export function installFakeClock(): void {
   if (installed != null) {
     return;
   }
@@ -120,7 +120,7 @@ export function useFakeTimers(): void {
 }
 
 /** Put the real scheduling globals back. */
-export function useRealTimers(): void {
+export function restoreRealClock(): void {
   if (installed == null) {
     return;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniflowed/test",
-  "version": "0.0.0-alpha.8",
+  "version": "0.1.0",
   "description": "The test API and worker for `uf test`: describe/it, a full matcher set, and the process uf fans test files out to.",
   "type": "module",
   "license": "MIT",
@@ -11,16 +11,52 @@
     "directory": "packages/test"
   },
   "exports": {
-    ".": "./index.js",
+    ".": {
+      "uniflowed-bun-test": "./bun/index.js",
+      "default": "./index.js"
+    },
+    "./in-source": "./in-source.js",
     "./worker": "./worker.js",
-    "./package.json": "./package.json"
+    "./browser-worker": "./browser-worker.js",
+    "./package.json": "./package.json",
+    "./app": {
+      "browser": "./app-browser.js",
+      "default": "./app.js"
+    },
+    "./browser": "./browser.js"
   },
   "files": [
+    "browser-worker.js",
+    "bun",
+    "in-source.js",
     "index.js",
+    "internal",
     "worker.js",
-    "internal"
+    "app.js",
+    "browser.js",
+    "app-browser.js",
+    "!*.test.js"
   ],
   "dependencies": {
-    "@uniflowed/host": "0.0.0-alpha.8"
+    "@uniflowed/host": "0.1.0",
+    "pixelmatch": "^7.1.0",
+    "pngjs": "^7.0.0"
+  },
+  "peerDependencies": {
+    "axe-core": ">=4"
+  },
+  "peerDependenciesMeta": {
+    "axe-core": {
+      "optional": true
+    }
+  },
+  "browser": {
+    "node:async_hooks": "./internal/browser/node.js",
+    "node:fs": "./internal/browser/node.js",
+    "node:module": "./internal/browser/node.js",
+    "node:path": "./internal/browser/node.js",
+    "node:url": "./internal/browser/node.js",
+    "node:util": "./internal/browser/node.js",
+    "./internal/browser/transport.js": "./internal/browser/page-transport.js"
   }
 }

package/worker.js

@@ -49,14 +49,29 @@
 
 import * as output from "./internal/output.js";
 import { AsyncLocalStorage } from "node:async_hooks";
+import process from "node:process";
 import { writeChangedSnapshots } from "./internal/snapshot.js";
 import { createInterface } from "node:readline";
 import { fileURLToPath, pathToFileURL } from "node:url";
 
-import { reset } from "./internal/registry.js";
-import { resetModuleState } from "./internal/modules.js";
+import { installInSourceTests } from "./in-source.js";
+import { restoreSharedState } from "./internal/isolation.js";
 import { run } from "./internal/run.js";
 
+// `process` is imported rather than read off the global because this file is
+// the process entry point, and an entry point that names what it depends on is
+// one a reader does not have to trust.
+//
+// It used to be put *on* the global as well. Deno 1.x exposed the object as
+// `node:process` and nothing as `globalThis.process`, and every module this
+// worker reaches that wants the process reads the global: `internal/output.js`
+// replaces `process.stdout.write`, `internal/namespace.js` reads
+// `process.env` for `stubEnv`, and `internal/axe.js` reads `UF_AXE`. The Deno
+// uf starts is 2.8 or newer — `@uniflowed/host/deno-preload` is built on that
+// release's `registerHooks`, and `uf test` refuses anything older — and Deno 2
+// has the global like Node and Bun. A shim for a host uf no longer starts would
+// be a line claiming to be load-bearing while holding nothing up.
+
 /** What `uf` sends for one file. */
 type Request = {|
   readonly file: string,
@@ -137,18 +152,80 @@ function write(event: { readonly [string]: mixed }): void {
  * the `serving` store this runs inside — it is what every event written from
  * this file, or from anything this file leaves behind, is stamped with.
  */
+/**
+ * Whether this is a run of the benchmarks, `uf test --bench`.
+ *
+ * Read from the environment, as snapshot updates are: it is a property of the
+ * run, set once on the worker by `uf`, and not something each request says.
+ */
+function benching(): boolean {
+  const value = (globalThis: $FlowFixMe).process?.env?.UF_TEST_BENCH;
+  return value != null && value !== "" && value !== "0";
+}
+
 async function runFile(request: Request, generation: number): Promise<void> {
   const started = performance.now();
-  reset();
-  // Every module this file stood in for goes back, before the next file can
-  // import one of them and be handed the previous file's stand-in. A worker
-  // serves many files out of one module registry, so this is the difference
-  // between "one file at a time" and "one file's mocks at a time".
-  resetModuleState();
+  // Everything the previous file changed and this package shares with it goes
+  // back: the registry, the stubbed environment and globals, the clock, the
+  // module stand-ins, and the document. What each of those is and why it is on
+  // the list is `internal/isolation.js`, which is one place rather than five
+  // calls here — a worker serves many files out of one process, `uf test` fans
+  // files across workers by size, and a leak therefore makes the *result* of a
+  // suite a function of the timings file rather than of the code under test.
+  // See ubugeeei-prod/uf#417, #581 and #607, which are that sentence three
+  // times over.
+  //
+  // Before the import rather than after the run, so a file that throws while
+  // loading still hands the next one a clean process.
+  restoreSharedState();
+  // Not a restore, and so not on that list: this is the output budget for the
+  // file about to run, rather than something the previous file left behind.
   output.startFile();
 
+  // In-source blocks in *this* file get uf's test API; the same blocks in
+  // every module this file imports get `undefined` and do not register. See
+  // `./in-source.js` for why the marker is a call rather than a constant.
+  const url = pathToFileURL(request.file).href;
+  const uninstallInSourceTests = installInSourceTests(url);
+  try {
+    await runImportedFile(request, generation, url, started);
+  } finally {
+    // For the whole file rather than only for its import. A block reads the
+    // marker at module scope, but a *case body* may read it too — `const
+    // { uft } = import.meta.uf.test` inside an `it` is an ordinary thing to
+    // write — and a marker that answered during the import and not during the
+    // run would be a value that changed under the file that read it.
+    //
+    // Taking it away earlier would not buy the isolation it looks like it
+    // buys: work a finished file left behind can register through a binding it
+    // captured just as easily as through this global, so the thing that keeps
+    // a straggler out of the next file is the registry reset at the top of
+    // this function and the generation stamped on every event, not the
+    // lifetime of one accessor.
+    uninstallInSourceTests();
+  }
+}
+
+/**
+ * The half of [`runFile`] that has a file to run.
+ *
+ * Split out so the caller's `finally` covers the import *and* the run without
+ * either of the two `return`s below escaping it.
+ */
+async function runImportedFile(
+  request: Request,
+  generation: number,
+  url: string,
+  started: number,
+): Promise<void> {
+  let closeNative: (() => Promise<void>) | null = null;
   try {
-    await import(`${pathToFileURL(request.file).href}?uf-run=${generation}`);
+    if (process.env.UF_TEST_TARGET === "react-native") {
+      const { loadNativeFile } = await import("./internal/native-host.js");
+      closeNative = await loadNativeFile(request.file);
+    } else {
+      await import(`${url}?uf-run=${generation}`);
+    }
   } catch (thrown) {
     const error = thrown instanceof Error ? thrown : new Error(String(thrown));
     write({
@@ -164,7 +241,12 @@ async function runFile(request: Request, generation: number): Promise<void> {
   try {
     const absolute = fileURLToPath(pathToFileURL(request.file).href);
     await run(
-      { filter: request.filter ?? null, timeoutMs: request.timeoutMs, file: absolute },
+      {
+        filter: request.filter ?? null,
+        timeoutMs: request.timeoutMs,
+        file: absolute,
+        bench: benching(),
+      },
       (result) => {
         write({
           event: "test",
@@ -180,12 +262,18 @@ async function runFile(request: Request, generation: number): Promise<void> {
     // with forty snapshots would otherwise rewrite its snapshot file forty
     // times, and a crash halfway through would leave a partial one.
     writeChangedSnapshots();
+    await closeNative?.();
     write({
       event: "file",
       status: "completed",
       durationMicros: Math.round((performance.now() - started) * 1000),
     });
   } catch (thrown) {
+    try {
+      await closeNative?.();
+    } catch {
+      // Preserve the original run or teardown failure.
+    }
     const error = thrown instanceof Error ? thrown : new Error(String(thrown));
     write({
       event: "file",