@uniflowed/test 0.0.0-alpha.9 → 0.2.0

package/internal/registry.js

@@ -19,7 +19,7 @@
 // * `.only` anywhere in the file restricts the file to marked cases and their
 //   ancestors; everything else is reported skipped, never silently dropped.
 
-import { firstUserSite } from "./frames.js";
+import { callerSite, firstUserSite } from "./frames.js";
 
 /** The placeholders `it.each` substitutes a row into. */
 const ROW_TOKEN = /%[sjdi]/g;
@@ -27,16 +27,42 @@ const ROW_TOKEN = /%[sjdi]/g;
 /** What a test or hook body may return. */
 export type Body = () => mixed | Promise<mixed>;
 
+/**
+ * One `beforeAll`, `afterAll`, `beforeEach` or `afterEach`.
+ *
+ * `timeoutMs` is the hook's own budget when its registration named one, and
+ * `null` when the hook runs under the budget of the case it belongs to. A hook
+ * that starts a process or a server needs longer than the cases it sets up for,
+ * and without a budget of its own the only lever was the file-wide timeout.
+ */
+export type Hook = {| readonly body: Body, readonly timeoutMs: number | null |};
+
 /** The suffix written on a registration call. */
 export type Modifier = "none" | "only" | "skip" | "todo";
 
-/** One registered test case. */
+/**
+ * How a benchmark runs under `uf test --bench`.
+ *
+ * `warmup` calls are made and their times thrown away, then `iterations` calls
+ * are each timed. `timeout` is the budget for one call, in milliseconds, and
+ * the benchmark as a whole is held to that budget once per call.
+ */
+export type BenchOptions = {|
+  readonly warmup?: number,
+  readonly iterations?: number,
+  readonly timeout?: number,
+|};
+
+/** One registered test case, or one benchmark. */
 export type Case = {|
-  readonly kind: "test",
+  readonly kind: "test" | "bench",
   readonly name: string,
   readonly body: Body | null,
   readonly modifier: Modifier,
+  readonly skipReason: string | null,
   readonly timeoutMs: number | null,
+  /** How to run it, for a benchmark; `null` for a test. */
+  readonly bench: BenchOptions | null,
   readonly line: number,
   readonly column: number,
 |};
@@ -47,10 +73,10 @@ export type Suite = {|
   readonly name: string,
   readonly modifier: Modifier,
   readonly children: Array<Suite | Case>,
-  readonly beforeAll: Array<Body>,
-  readonly afterAll: Array<Body>,
-  readonly beforeEach: Array<Body>,
-  readonly afterEach: Array<Body>,
+  readonly beforeAll: Array<Hook>,
+  readonly afterAll: Array<Hook>,
+  readonly beforeEach: Array<Hook>,
+  readonly afterEach: Array<Hook>,
   readonly line: number,
   readonly column: number,
 |};
@@ -102,6 +128,10 @@ export function collected(): Suite {
  * treats as "unknown" rather than as line one.
  */
 function callSite(): {| readonly line: number, readonly column: number |} {
+  const site = callerSite(callSite);
+  if (site !== undefined) {
+    return site ?? { line: 0, column: 0 };
+  }
   return firstUserSite(new Error("position").stack) ?? { line: 0, column: 0 };
 }
 
@@ -123,14 +153,18 @@ function addCase(
   body: Body | null,
   modifier: Modifier,
   timeoutMs: number | null,
+  skipReason: string | null = null,
+  bench: BenchOptions | null = null,
 ): void {
   const position = callSite();
   current.children.push({
-    kind: "test",
+    kind: bench == null ? "test" : "bench",
     name,
     body,
     modifier,
+    skipReason,
     timeoutMs,
+    bench,
     line: position.line,
     column: position.column,
   });
@@ -179,6 +213,9 @@ function caseApi(): $FlowFixMe {
   api.skip = (name: string, body?: Body) => {
     addCase(name, body ?? null, "skip", null);
   };
+  api.skipBecause = (name: string, reason: string, body?: Body) => {
+    addCase(name, body ?? null, "skip", null, reason);
+  };
   api.todo = (name: string, body?: Body) => {
     addCase(name, body ?? null, "todo", null);
   };
@@ -211,6 +248,34 @@ export const it: $FlowFixMe = caseApi();
 /** `test` is `it`, for people who write it that way. */
 export const test: $FlowFixMe = it;
 
+/** The `bench` API, and its modifiers. See [`suiteApi`] for the shape. */
+function benchApi(): $FlowFixMe {
+  const api: $FlowFixMe = (name: string, body: Body, options?: BenchOptions) => {
+    addCase(name, body, "none", options?.timeout ?? null, null, options ?? {});
+  };
+  api.only = (name: string, body: Body, options?: BenchOptions) => {
+    addCase(name, body, "only", options?.timeout ?? null, null, options ?? {});
+  };
+  api.skip = (name: string, body?: Body) => {
+    addCase(name, body ?? null, "skip", null, null, {});
+  };
+  api.todo = (name: string) => {
+    addCase(name, null, "todo", null, null, {});
+  };
+  return api;
+}
+
+/**
+ * Register one benchmark.
+ *
+ * `uf test` reports a benchmark as skipped, so a suite does not pay for timing
+ * one, and `uf test --bench` runs the benchmarks in place of the tests and
+ * reports how long each call took. `bench.only`, `bench.skip` and `bench.todo`
+ * do what `it`'s do. See [`BenchOptions`] for `warmup`, `iterations` and
+ * `timeout`.
+ */
+export const bench: $FlowFixMe = benchApi();
+
 /**
  * Substitute a row into a name, the way every runner spells it: `%s` for the
  * value, `%j` for its JSON.
@@ -225,22 +290,42 @@ function formatRow(name: string, row: mixed): string {
   });
 }
 
-/** Run once before the first test in this suite that runs. */
-export function beforeAll(body: Body): void {
-  current.beforeAll.push(body);
+/**
+ * A hook's own budget, from the second argument of its registration.
+ *
+ * `{ timeout }` is what `it` takes, and a bare number is what Jest and Vitest
+ * take — a suite moved from either writes `beforeAll(start, 30_000)`, and a
+ * budget that was silently dropped would fail as a timeout the file never
+ * asked for.
+ */
+function hookTimeout(options: ?(TestOptions | number)): number | null {
+  if (typeof options === "number") {
+    return options;
+  }
+  return options?.timeout ?? null;
+}
+
+/**
+ * Run once before the first test in this suite that runs.
+ *
+ * When it fails, every case it was setting up for fails with its error: none
+ * of them runs against a setup that did not happen.
+ */
+export function beforeAll(body: Body, options?: TestOptions | number): void {
+  current.beforeAll.push({ body, timeoutMs: hookTimeout(options) });
 }
 
 /** Run once after the last test in this suite that ran. */
-export function afterAll(body: Body): void {
-  current.afterAll.push(body);
+export function afterAll(body: Body, options?: TestOptions | number): void {
+  current.afterAll.push({ body, timeoutMs: hookTimeout(options) });
 }
 
 /** Run before every test in this suite and its children. */
-export function beforeEach(body: Body): void {
-  current.beforeEach.push(body);
+export function beforeEach(body: Body, options?: TestOptions | number): void {
+  current.beforeEach.push({ body, timeoutMs: hookTimeout(options) });
 }
 
 /** Run after every test in this suite and its children, including failures. */
-export function afterEach(body: Body): void {
-  current.afterEach.push(body);
+export function afterEach(body: Body, options?: TestOptions | number): void {
+  current.afterEach.push({ body, timeoutMs: hookTimeout(options) });
 }

package/internal/run.js

@@ -13,11 +13,25 @@ import * as output from "./output.js";
 import * as snapshot from "./snapshot.js";
 import { AssertionError } from "./expect.js";
 import { type Site, firstUserSite, siteInFile, userFrames } from "./frames.js";
-import { type Body, type Case, type Suite, collected } from "./registry.js";
+import {
+  type BenchOptions,
+  type Body,
+  type Case,
+  type Hook,
+  type Suite,
+  collected,
+} from "./registry.js";
 
 /** How one case ended. */
 export type Outcome =
-  | {| readonly status: "passed" |}
+  | {|
+      readonly status: "passed",
+      /**
+       * One timing per measured call, in whole microseconds, for a benchmark
+       * run under `uf test --bench`.
+       */
+      readonly samples?: $ReadOnlyArray<number>,
+    |}
   | {|
       readonly status: "failed",
       readonly message: string,
@@ -27,7 +41,11 @@ export type Outcome =
       /** Where the failing assertion was written, when the stack says. */
       readonly site: {| readonly line: number, readonly column: number |} | null,
     |}
-  | {| readonly status: "skipped", readonly reason: "explicit" | "not-only" | "filtered" |}
+  | {|
+      readonly status: "skipped",
+      readonly reason: "explicit" | "not-only" | "filtered" | "bench" | "not-bench",
+      readonly message?: string | null,
+    |}
   | {| readonly status: "todo" |};
 
 /** One finished case, as the runner reports it. */
@@ -52,11 +70,31 @@ export type RunOptions = {|
    * which file that is — a test's name alone does not locate it.
    */
   readonly file?: string,
+  /**
+   * Run the benchmarks and report the tests skipped, rather than the other way
+   * round. `uf test --bench` sets it.
+   */
+  readonly bench?: boolean,
 |};
 
 /** Default budget for one case, matching what most runners use. */
 export const DEFAULT_TIMEOUT_MS: number = 5000;
 
+/** Calls a benchmark makes and throws away before it times any, unless it says. */
+export const DEFAULT_BENCH_WARMUP: number = 5;
+
+/** Timed calls a benchmark makes, unless it says. */
+export const DEFAULT_BENCH_ITERATIONS: number = 50;
+
+/** Most timed calls one benchmark may ask for, and most samples `uf` keeps. */
+export const MAX_BENCH_ITERATIONS: number = 100000;
+
+/** Most untimed calls one benchmark may ask for first. */
+export const MAX_BENCH_WARMUP: number = 10000;
+
+/** The longest delay `setTimeout` honours; past it, the timer fires at once. */
+const MAX_TIMER_MS = 2147483647;
+
 /** The separator between a suite's name and its child's. */
 export const NAME_SEPARATOR: string = " > ";
 
@@ -72,7 +110,9 @@ function fullName(path: $ReadOnlyArray<string>): string {
  */
 function hasOnly(node: Suite | Case, inherited: boolean): boolean {
   const marked = inherited || node.modifier === "only";
-  if (node.kind === "test") {
+  // A benchmark is a case like a test: `bench.only` restricts a file as
+  // `it.only` does.
+  if (node.kind !== "suite") {
     return marked;
   }
   return node.children.some((child) => hasOnly(child, marked));
@@ -153,8 +193,8 @@ function failure(thrown: mixed, file: string | null): Outcome {
 /** Everything one case needs from the suites above it. */
 type Context = {|
   readonly path: $ReadOnlyArray<string>,
-  readonly beforeEach: $ReadOnlyArray<Body>,
-  readonly afterEach: $ReadOnlyArray<Body>,
+  readonly beforeEach: $ReadOnlyArray<Hook>,
+  readonly afterEach: $ReadOnlyArray<Hook>,
   readonly skipped: boolean,
   readonly onlyPath: boolean,
 |};
@@ -188,12 +228,17 @@ async function runCase(
     });
   };
 
-  if (test.modifier === "todo" || test.body == null) {
+  if (test.modifier === "todo") {
     report({ status: "todo" });
     return true;
   }
   if (context.skipped || test.modifier === "skip") {
-    report({ status: "skipped", reason: "explicit" });
+    report({ status: "skipped", reason: "explicit", message: test.skipReason });
+    return true;
+  }
+  const body = test.body;
+  if (body == null) {
+    report({ status: "todo" });
     return true;
   }
   if (!context.onlyPath) {
@@ -205,9 +250,19 @@ async function runCase(
     report({ status: "skipped", reason: "filtered" });
     return true;
   }
+  // A run is the tests or the benchmarks, never both. A benchmark timed inside
+  // an ordinary run would slow every suite that has one, beside workers busy
+  // with other files; a test inside a run of benchmarks would be timed with
+  // them.
+  const benchmark = test.kind === "bench";
+  if (benchmark !== (options.bench === true)) {
+    report({ status: "skipped", reason: benchmark ? "bench" : "not-bench" });
+    return true;
+  }
 
   const timeoutMs = test.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   let outcome: Outcome = { status: "passed" };
+  let samples: $ReadOnlyArray<number> | null = null;
   // Setup, body and teardown run *inside* the case's output context, and that
   // nesting is the whole of the fix for #207. What names a printed line is no
   // longer where the runner had got to when the line arrived — which named the
@@ -220,9 +275,13 @@ async function runCase(
   await output.runInTest(name, async () => {
     try {
       for (const hook of context.beforeEach) {
-        await withTimeout(hook, timeoutMs);
+        await withTimeout(hook.body, hook.timeoutMs ?? timeoutMs);
+      }
+      if (benchmark) {
+        samples = await measure(body, test.bench, timeoutMs);
+      } else {
+        await withTimeout(body, timeoutMs);
       }
-      await withTimeout(test.body, timeoutMs);
     } catch (thrown) {
       outcome = failure(thrown, options.file ?? null);
     }
@@ -230,7 +289,7 @@ async function runCase(
     // when the body had not already failed.
     for (const hook of context.afterEach) {
       try {
-        await withTimeout(hook, timeoutMs);
+        await withTimeout(hook.body, hook.timeoutMs ?? timeoutMs);
       } catch (thrown) {
         if (outcome.status === "passed") {
           outcome = failure(thrown, options.file ?? null);
@@ -243,10 +302,74 @@ async function runCase(
   // is the file's, and a snapshot taken outside one fails with something better
   // than a key belonging to whichever test happened to run last.
   snapshot.exitTest();
-  report(outcome);
+  report(outcome.status === "passed" && samples != null ? { status: "passed", samples } : outcome);
   return outcome.status !== "failed";
 }
 
+/**
+ * Time a benchmark's body: `warmup` calls thrown away, then `iterations` timed.
+ *
+ * The body is called directly and awaited, with nothing else between the two
+ * readings of the clock: a timer set around every call would be timed with
+ * it, and on a body that takes microseconds that is most of the number. The
+ * budget is held around the whole loop instead, at one call's budget per
+ * call, so a benchmark that hangs still fails rather than holding the worker.
+ */
+async function measure(
+  body: Body,
+  options: BenchOptions | null,
+  timeoutMs: number,
+): Promise<$ReadOnlyArray<number>> {
+  const warmup = count("warmup", options?.warmup, DEFAULT_BENCH_WARMUP, 0, MAX_BENCH_WARMUP);
+  const iterations = count(
+    "iterations",
+    options?.iterations,
+    DEFAULT_BENCH_ITERATIONS,
+    1,
+    MAX_BENCH_ITERATIONS,
+  );
+  const rounds = warmup + iterations;
+  const samples: Array<number> = [];
+  await withTimeout(
+    async () => {
+      for (let round = 0; round < rounds; round += 1) {
+        const started = performance.now();
+        await body();
+        if (round >= warmup) {
+          samples.push(Math.round((performance.now() - started) * 1000));
+        }
+      }
+    },
+    Math.min(timeoutMs * rounds, MAX_TIMER_MS),
+  );
+  return samples;
+}
+
+/**
+ * A benchmark option that has to be a whole number from `least` to `most`.
+ *
+ * Refused rather than clamped: `iterations: 0` or `warmup: 1e9` is a mistake,
+ * and a benchmark quietly run some other number of times reports numbers
+ * about a run nobody asked for.
+ */
+function count(
+  option: string,
+  value: ?number,
+  fallback: number,
+  least: number,
+  most: number,
+): number {
+  if (value == null) {
+    return fallback;
+  }
+  if (!Number.isInteger(value) || value < least || value > most) {
+    throw new Error(
+      `bench \`${option}\` has to be a whole number from ${least} to ${most}, and was ${String(value)}`,
+    );
+  }
+  return value;
+}
+
 /**
  * Walk one suite, running what it contains.
  *
@@ -280,16 +403,25 @@ async function runSuite(
   // every `beforeAll` in an ordinary file — one where the tests live inside a
   // `describe` — never running at all, silently. The chain is walked outermost
   // first, so an inner suite's setup sees what the outer one did.
+  //
+  // Once, and remembered either way. The outcome is kept rather than a "done"
+  // flag, because a flag raised before the hooks ran let every case after the
+  // first go ahead when setup had failed: the first case reported the hook's
+  // error and the rest ran against a fixture that was never made, failing — or
+  // worse, passing — for reasons that had nothing to do with them.
   let setUp = false;
-  const setUpOnce = async () => {
-    if (setUp) {
-      return;
-    }
-    await setUpAncestors();
-    setUp = true;
-    for (const hook of node.beforeAll) {
-      await withTimeout(hook, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+  let setup: Promise<void> | null = null;
+  const setUpOnce = (): Promise<void> => {
+    if (setup == null) {
+      setUp = true;
+      setup = (async () => {
+        await setUpAncestors();
+        for (const hook of node.beforeAll) {
+          await withTimeout(hook.body, hook.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+        }
+      })();
     }
+    return setup;
   };
 
   let passed = true;
@@ -297,13 +429,16 @@ async function runSuite(
     if (state.bail) {
       break;
     }
-    if (child.kind === "test") {
+    if (child.kind !== "suite") {
       const willRun =
         !inner.skipped &&
         child.modifier !== "skip" &&
         child.modifier !== "todo" &&
         child.body != null &&
-        (!onlyMode || inner.onlyPath || child.modifier === "only");
+        (!onlyMode || inner.onlyPath || child.modifier === "only") &&
+        // A benchmark in a run of the tests, or a test in a run of the
+        // benchmarks, is reported skipped and sets nothing up.
+        (child.kind === "bench") === (options.bench === true);
       if (willRun) {
         try {
           await setUpOnce();
@@ -315,7 +450,7 @@ async function runSuite(
             line: child.line,
             column: child.column,
             durationMicros: 0,
-            outcome: failure(thrown),
+            outcome: failure(thrown, options.file ?? null),
           });
           passed = false;
           continue;
@@ -336,7 +471,7 @@ async function runSuite(
   if (setUp) {
     for (const hook of node.afterAll) {
       try {
-        await withTimeout(hook, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+        await withTimeout(hook.body, hook.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
       } catch {
         // A teardown failure cannot fail a test that already reported, and
         // there is nothing left to attach it to; the file's own status carries

package/internal/timers.js

@@ -82,7 +82,7 @@ function host(): $FlowFixMe {
  * formats a date sees a plausible one — and `setSystemTime` is how a test that
  * cares says which.
  */
-export function useFakeTimers(): void {
+export function installFakeClock(): void {
   if (installed != null) {
     return;
   }
@@ -120,7 +120,7 @@ export function useFakeTimers(): void {
 }
 
 /** Put the real scheduling globals back. */
-export function useRealTimers(): void {
+export function restoreRealClock(): void {
   if (installed == null) {
     return;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniflowed/test",
-  "version": "0.0.0-alpha.9",
+  "version": "0.2.0",
   "description": "The test API and worker for `uf test`: describe/it, a full matcher set, and the process uf fans test files out to.",
   "type": "module",
   "license": "MIT",
@@ -11,16 +11,52 @@
     "directory": "packages/test"
   },
   "exports": {
-    ".": "./index.js",
+    ".": {
+      "uniflowed-bun-test": "./bun/index.js",
+      "default": "./index.js"
+    },
+    "./in-source": "./in-source.js",
     "./worker": "./worker.js",
-    "./package.json": "./package.json"
+    "./browser-worker": "./browser-worker.js",
+    "./package.json": "./package.json",
+    "./app": {
+      "browser": "./app-browser.js",
+      "default": "./app.js"
+    },
+    "./browser": "./browser.js"
   },
   "files": [
+    "browser-worker.js",
+    "bun",
+    "in-source.js",
     "index.js",
+    "internal",
     "worker.js",
-    "internal"
+    "app.js",
+    "browser.js",
+    "app-browser.js",
+    "!*.test.js"
   ],
   "dependencies": {
-    "@uniflowed/host": "0.0.0-alpha.9"
+    "@uniflowed/host": "0.2.0",
+    "pixelmatch": "^7.1.0",
+    "pngjs": "^7.0.0"
+  },
+  "peerDependencies": {
+    "axe-core": ">=4"
+  },
+  "peerDependenciesMeta": {
+    "axe-core": {
+      "optional": true
+    }
+  },
+  "browser": {
+    "node:async_hooks": "./internal/browser/node.js",
+    "node:fs": "./internal/browser/node.js",
+    "node:module": "./internal/browser/node.js",
+    "node:path": "./internal/browser/node.js",
+    "node:url": "./internal/browser/node.js",
+    "node:util": "./internal/browser/node.js",
+    "./internal/browser/transport.js": "./internal/browser/page-transport.js"
   }
 }