@uniflowed/test 0.0.0-alpha.4 → 0.0.0-alpha.41

package/internal/run.js

@@ -12,12 +12,19 @@
 import * as output from "./output.js";
 import * as snapshot from "./snapshot.js";
 import { AssertionError } from "./expect.js";
-import { firstUserSite, userFrames } from "./frames.js";
-import { type Body, type Case, type Suite, collected } from "./registry.js";
+import { type Site, firstUserSite, siteInFile, userFrames } from "./frames.js";
+import { type BenchOptions, type Body, type Case, type Suite, collected } from "./registry.js";
 
 /** How one case ended. */
 export type Outcome =
-  | {| readonly status: "passed" |}
+  | {|
+      readonly status: "passed",
+      /**
+       * One timing per measured call, in whole microseconds, for a benchmark
+       * run under `uf test --bench`.
+       */
+      readonly samples?: $ReadOnlyArray<number>,
+    |}
   | {|
       readonly status: "failed",
       readonly message: string,
@@ -27,7 +34,11 @@ export type Outcome =
       /** Where the failing assertion was written, when the stack says. */
       readonly site: {| readonly line: number, readonly column: number |} | null,
     |}
-  | {| readonly status: "skipped", readonly reason: "explicit" | "not-only" | "filtered" |}
+  | {|
+      readonly status: "skipped",
+      readonly reason: "explicit" | "not-only" | "filtered" | "bench" | "not-bench",
+      readonly message?: string | null,
+    |}
   | {| readonly status: "todo" |};
 
 /** One finished case, as the runner reports it. */
@@ -52,11 +63,31 @@ export type RunOptions = {|
    * which file that is — a test's name alone does not locate it.
    */
   readonly file?: string,
+  /**
+   * Run the benchmarks and report the tests skipped, rather than the other way
+   * round. `uf test --bench` sets it.
+   */
+  readonly bench?: boolean,
 |};
 
 /** Default budget for one case, matching what most runners use. */
 export const DEFAULT_TIMEOUT_MS: number = 5000;
 
+/** Calls a benchmark makes and throws away before it times any, unless it says. */
+export const DEFAULT_BENCH_WARMUP: number = 5;
+
+/** Timed calls a benchmark makes, unless it says. */
+export const DEFAULT_BENCH_ITERATIONS: number = 50;
+
+/** Most timed calls one benchmark may ask for, and most samples `uf` keeps. */
+export const MAX_BENCH_ITERATIONS: number = 100000;
+
+/** Most untimed calls one benchmark may ask for first. */
+export const MAX_BENCH_WARMUP: number = 10000;
+
+/** The longest delay `setTimeout` honours; past it, the timer fires at once. */
+const MAX_TIMER_MS = 2147483647;
+
 /** The separator between a suite's name and its child's. */
 export const NAME_SEPARATOR: string = " > ";
 
@@ -72,7 +103,9 @@ function fullName(path: $ReadOnlyArray<string>): string {
  */
 function hasOnly(node: Suite | Case, inherited: boolean): boolean {
   const marked = inherited || node.modifier === "only";
-  if (node.kind === "test") {
+  // A benchmark is a case like a test: `bench.only` restricts a file as
+  // `it.only` does.
+  if (node.kind !== "suite") {
     return marked;
   }
   return node.children.some((child) => hasOnly(child, marked));
@@ -102,7 +135,22 @@ async function withTimeout(body: Body, timeoutMs: number): Promise<void> {
   }
 }
 
-function failure(thrown: mixed): Outcome {
+/**
+ * Where to say a failure happened.
+ *
+ * The reported position is printed under the path of the file being run, so
+ * when that file is known the line has to come from it: `firstUserSite` will
+ * hand back the first frame of whatever library raised, and a library's line
+ * number wearing the test file's path sends the reader to the wrong place
+ * (ubugeeei-prod/uf#319). The fallback is for a caller that did not say which
+ * file it is running — `run` is driven directly by this repository's own
+ * tests as well as by the worker — and is what every failure used before.
+ */
+function siteOf(stack: string | null, file: string | null): Site | null {
+  return file == null || file === "" ? firstUserSite(stack, false) : siteInFile(stack, file);
+}
+
+function failure(thrown: mixed, file: string | null): Outcome {
   if (thrown instanceof AssertionError) {
     const stack = userFrames(thrown.stack);
     return {
@@ -111,7 +159,7 @@ function failure(thrown: mixed): Outcome {
       stack,
       expected: thrown.expected,
       received: thrown.received,
-      site: firstUserSite(stack, false),
+      site: siteOf(stack, file),
     };
   }
   if (thrown instanceof Error) {
@@ -122,7 +170,7 @@ function failure(thrown: mixed): Outcome {
       stack,
       expected: null,
       received: null,
-      site: firstUserSite(stack, false),
+      site: siteOf(stack, file),
     };
   }
   return {
@@ -173,12 +221,17 @@ async function runCase(
     });
   };
 
-  if (test.modifier === "todo" || test.body == null) {
+  if (test.modifier === "todo") {
     report({ status: "todo" });
     return true;
   }
   if (context.skipped || test.modifier === "skip") {
-    report({ status: "skipped", reason: "explicit" });
+    report({ status: "skipped", reason: "explicit", message: test.skipReason });
+    return true;
+  }
+  const body = test.body;
+  if (body == null) {
+    report({ status: "todo" });
     return true;
   }
   if (!context.onlyPath) {
@@ -190,42 +243,126 @@ async function runCase(
     report({ status: "skipped", reason: "filtered" });
     return true;
   }
+  // A run is the tests or the benchmarks, never both. A benchmark timed inside
+  // an ordinary run would slow every suite that has one, beside workers busy
+  // with other files; a test inside a run of benchmarks would be timed with
+  // them.
+  const benchmark = test.kind === "bench";
+  if (benchmark !== (options.bench === true)) {
+    report({ status: "skipped", reason: benchmark ? "bench" : "not-bench" });
+    return true;
+  }
 
   const timeoutMs = test.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   let outcome: Outcome = { status: "passed" };
-  // From here to the end of teardown is exactly the window in which this
-  // case's own code runs, so it is exactly the window whose printing is this
-  // case's. The cases above never reach it: nothing runs for a `todo`, a
-  // `skip` or a filtered-out case, so nothing of theirs can print.
-  output.enterTest(name);
-  try {
-    for (const hook of context.beforeEach) {
-      await withTimeout(hook, timeoutMs);
-    }
-    await withTimeout(test.body, timeoutMs);
-  } catch (thrown) {
-    outcome = failure(thrown);
-  }
-  // Teardown runs whatever happened above, and only reports its own failure
-  // when the body had not already failed.
-  for (const hook of context.afterEach) {
+  let samples: $ReadOnlyArray<number> | null = null;
+  // Setup, body and teardown run *inside* the case's output context, and that
+  // nesting is the whole of the fix for #207. What names a printed line is no
+  // longer where the runner had got to when the line arrived — which named the
+  // next case for anything a `setTimeout` left behind — but which case's work
+  // the write descends from. A callback scheduled here keeps this name however
+  // late it fires.
+  //
+  // The cases above never reach this: nothing runs for a `todo`, a `skip` or a
+  // filtered-out case, so nothing of theirs can print.
+  await output.runInTest(name, async () => {
     try {
-      await withTimeout(hook, timeoutMs);
+      for (const hook of context.beforeEach) {
+        await withTimeout(hook, timeoutMs);
+      }
+      if (benchmark) {
+        samples = await measure(body, test.bench, timeoutMs);
+      } else {
+        await withTimeout(body, timeoutMs);
+      }
     } catch (thrown) {
-      if (outcome.status === "passed") {
-        outcome = failure(thrown);
+      outcome = failure(thrown, options.file ?? null);
+    }
+    // Teardown runs whatever happened above, and only reports its own failure
+    // when the body had not already failed.
+    for (const hook of context.afterEach) {
+      try {
+        await withTimeout(hook, timeoutMs);
+      } catch (thrown) {
+        if (outcome.status === "passed") {
+          outcome = failure(thrown, options.file ?? null);
+        }
       }
     }
-  }
-  // No test is running once this one is reported, so a snapshot taken outside
-  // one fails with something better than a key belonging to whichever test
-  // happened to run last, and a line printed outside one is the file's.
-  output.exitTest();
+  });
+  // The `await` above resumes outside the context it entered, so this line and
+  // everything after it belong to no case again — a line printed between cases
+  // is the file's, and a snapshot taken outside one fails with something better
+  // than a key belonging to whichever test happened to run last.
   snapshot.exitTest();
-  report(outcome);
+  report(outcome.status === "passed" && samples != null ? { status: "passed", samples } : outcome);
   return outcome.status !== "failed";
 }
 
+/**
+ * Time a benchmark's body: `warmup` calls thrown away, then `iterations` timed.
+ *
+ * The body is called directly and awaited, with nothing else between the two
+ * readings of the clock: a timer set around every call would be timed with
+ * it, and on a body that takes microseconds that is most of the number. The
+ * budget is held around the whole loop instead, at one call's budget per
+ * call, so a benchmark that hangs still fails rather than holding the worker.
+ */
+async function measure(
+  body: Body,
+  options: BenchOptions | null,
+  timeoutMs: number,
+): Promise<$ReadOnlyArray<number>> {
+  const warmup = count("warmup", options?.warmup, DEFAULT_BENCH_WARMUP, 0, MAX_BENCH_WARMUP);
+  const iterations = count(
+    "iterations",
+    options?.iterations,
+    DEFAULT_BENCH_ITERATIONS,
+    1,
+    MAX_BENCH_ITERATIONS,
+  );
+  const rounds = warmup + iterations;
+  const samples: Array<number> = [];
+  await withTimeout(
+    async () => {
+      for (let round = 0; round < rounds; round += 1) {
+        const started = performance.now();
+        await body();
+        if (round >= warmup) {
+          samples.push(Math.round((performance.now() - started) * 1000));
+        }
+      }
+    },
+    Math.min(timeoutMs * rounds, MAX_TIMER_MS),
+  );
+  return samples;
+}
+
+/**
+ * A benchmark option that has to be a whole number from `least` to `most`.
+ *
+ * Refused rather than clamped: `iterations: 0` or `warmup: 1e9` is a mistake,
+ * and a benchmark quietly run some other number of times reports numbers
+ * about a run nobody asked for.
+ */
+function count(
+  option: string,
+  value: ?number,
+  fallback: number,
+  least: number,
+  most: number,
+): number {
+  if (value == null) {
+    return fallback;
+  }
+  if (!Number.isInteger(value) || value < least || value > most) {
+    throw new Error(
+      `bench \`${option}\` has to be a whole number from ${least} to ${most}, and was ${String(value)}`,
+    );
+  }
+  return value;
+}
+
 /**
  * Walk one suite, running what it contains.
  *
@@ -238,6 +375,7 @@ async function runSuite(
   onlyMode: boolean,
   emit: (result: Result) => void,
   state: {| bail: boolean |},
+  setUpAncestors: () => Promise<void>,
 ): Promise<boolean> {
   const skipped = context.skipped || node.modifier === "skip" || node.modifier === "todo";
   const onlyPath = !onlyMode || context.onlyPath || node.modifier === "only";
@@ -252,11 +390,18 @@ async function runSuite(
 
   // `beforeAll` is deferred until a case in this suite actually runs, so a
   // fully skipped suite never sets anything up. `afterAll` mirrors it.
+  //
+  // "In this suite" means anywhere under it. A suite whose children are all
+  // suites has no case of its own, and setting up only for a direct child left
+  // every `beforeAll` in an ordinary file — one where the tests live inside a
+  // `describe` — never running at all, silently. The chain is walked outermost
+  // first, so an inner suite's setup sees what the outer one did.
   let setUp = false;
   const setUpOnce = async () => {
     if (setUp) {
       return;
     }
+    await setUpAncestors();
     setUp = true;
     for (const hook of node.beforeAll) {
       await withTimeout(hook, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
@@ -268,13 +413,16 @@ async function runSuite(
     if (state.bail) {
       break;
     }
-    if (child.kind === "test") {
+    if (child.kind !== "suite") {
       const willRun =
         !inner.skipped &&
         child.modifier !== "skip" &&
         child.modifier !== "todo" &&
         child.body != null &&
-        (!onlyMode || inner.onlyPath || child.modifier === "only");
+        (!onlyMode || inner.onlyPath || child.modifier === "only") &&
+        // A benchmark in a run of the tests, or a test in a run of the
+        // benchmarks, is reported skipped and sets nothing up.
+        (child.kind === "bench") === (options.bench === true);
       if (willRun) {
         try {
           await setUpOnce();
@@ -286,7 +434,7 @@ async function runSuite(
             line: child.line,
             column: child.column,
             durationMicros: 0,
-            outcome: failure(thrown),
+            outcome: failure(thrown, options.file ?? null),
           });
           passed = false;
           continue;
@@ -299,7 +447,7 @@ async function runSuite(
       const ok = await runCase(child, childContext, options, emit);
       passed = passed && ok;
     } else {
-      const ok = await runSuite(child, inner, options, onlyMode, emit, state);
+      const ok = await runSuite(child, inner, options, onlyMode, emit, state, setUpOnce);
       passed = passed && ok;
     }
   }
@@ -333,5 +481,5 @@ export async function run(options: RunOptions, emit: (result: Result) => void):
     skipped: false,
     onlyPath: !onlyMode,
   };
-  await runSuite(root, context, options, onlyMode, emit, { bail: false });
+  await runSuite(root, context, options, onlyMode, emit, { bail: false }, async () => {});
 }

package/internal/timers.js

@@ -82,7 +82,7 @@ function host(): $FlowFixMe {
  * formats a date sees a plausible one — and `setSystemTime` is how a test that
  * cares says which.
  */
-export function useFakeTimers(): void {
+export function installFakeClock(): void {
   if (installed != null) {
     return;
   }
@@ -120,7 +120,7 @@ export function useFakeTimers(): void {
 }
 
 /** Put the real scheduling globals back. */
-export function useRealTimers(): void {
+export function restoreRealClock(): void {
   if (installed == null) {
     return;
   }

package/internal/unsupported.js

@@ -0,0 +1,30 @@
+// @flow
+//
+// Internal to `@uniflowed/test`: the error a `uft` binding raises when this
+// host cannot give it.
+//
+// Its own module because two of them need it — the namespace, and the module
+// mocking in `./modules.js` that the namespace imports — and a class that both
+// of a pair of modules reaches for is a third module, not an import cycle.
+//
+// The class is exported from the package, so a test can catch it by name
+// rather than by matching on a message.
+
+/**
+ * Raised for a `uft` binding this host cannot provide.
+ *
+ * Names what the binding needs rather than only that it is missing: a reader
+ * who is told that module interception needs synchronous module hooks can
+ * decide whether to run the suite on another host or to restructure the test,
+ * and neither is a decision a bare "not implemented" supports.
+ */
+export class UnsupportedError extends Error {
+  /** The binding that was called. */
+  binding: string;
+
+  constructor(binding: string, reason: string) {
+    super(`uft.${binding} is not available: ${reason}`);
+    this.name = "UnsupportedError";
+    this.binding = binding;
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uniflowed/test",
-  "version": "0.0.0-alpha.4",
+  "version": "0.0.0-alpha.41",
   "description": "The test API and worker for `uf test`: describe/it, a full matcher set, and the process uf fans test files out to.",
   "type": "module",
   "license": "MIT",
@@ -11,16 +11,52 @@
     "directory": "packages/test"
   },
   "exports": {
-    ".": "./index.js",
+    ".": {
+      "uniflowed-bun-test": "./bun/index.js",
+      "default": "./index.js"
+    },
+    "./in-source": "./in-source.js",
     "./worker": "./worker.js",
-    "./package.json": "./package.json"
+    "./browser-worker": "./browser-worker.js",
+    "./package.json": "./package.json",
+    "./app": {
+      "browser": "./app-browser.js",
+      "default": "./app.js"
+    },
+    "./browser": "./browser.js"
   },
   "files": [
+    "browser-worker.js",
+    "bun",
+    "in-source.js",
     "index.js",
+    "internal",
     "worker.js",
-    "internal"
+    "app.js",
+    "browser.js",
+    "app-browser.js",
+    "!*.test.js"
   ],
   "dependencies": {
-    "@uniflowed/host": "0.0.0-alpha.4"
+    "@uniflowed/host": "0.0.0-alpha.41",
+    "pixelmatch": "^7.1.0",
+    "pngjs": "^7.0.0"
+  },
+  "peerDependencies": {
+    "axe-core": ">=4"
+  },
+  "peerDependenciesMeta": {
+    "axe-core": {
+      "optional": true
+    }
+  },
+  "browser": {
+    "node:async_hooks": "./internal/browser/node.js",
+    "node:fs": "./internal/browser/node.js",
+    "node:module": "./internal/browser/node.js",
+    "node:path": "./internal/browser/node.js",
+    "node:url": "./internal/browser/node.js",
+    "node:util": "./internal/browser/node.js",
+    "./internal/browser/transport.js": "./internal/browser/page-transport.js"
   }
 }