@uniflowed/test 0.0.0-alpha.4 → 0.0.0-alpha.40

package/internal/output.js

@@ -21,25 +21,58 @@
 // # Why this is its own module
 //
 // Two callers need it and neither owns it: `worker.js` installs the capture at
-// start-up, and `run.js` says which case is running so a chunk can be named.
-// "Who printed this" is also state with a lifetime of its own — set around a
-// case's body and hooks, cleared between them — exactly like the snapshot key
-// in `snapshot.js`, and for the same reason it lives beside the thing it
-// describes rather than inside either caller.
+// start-up, and `run.js` runs each case inside the ownership kept here, so a
+// chunk can be named. "Who printed this" is also state with a lifetime of its
+// own — one case's hooks and body — exactly like the snapshot key in
+// `snapshot.js`, and for the same reason it lives beside the thing it describes
+// rather than inside either caller.
 //
 // # What "the test that printed it" means
 //
-// The name a chunk carries is the case the *worker* is running when the chunk
-// arrives, which is not the same as the case whose code produced it. A
-// `setTimeout` a test leaves behind prints while the next case is running and
-// is filed under that one; a chunk from no case at all is filed under the
-// file.
+// The case whose *asynchronous context* the write happened in, which is the
+// case whose code produced it.
 //
-// Getting this exactly right needs the printing to be tied to the asynchronous
-// context the case ran in — `AsyncLocalStorage` and everything under it — and
-// that is a bigger change than this module, because it has to reach the
-// scheduler that runs the cases. It is written down here rather than left to
-// be discovered from a confusing report. See ubugeeei-prod/uf#207.
+// The obvious answer was a module-level variable the runner set before a case
+// and cleared after it, and it was wrong in one shape that matters: the name a
+// chunk carried was whatever the worker happened to be running when the chunk
+// arrived. A `setTimeout` a test left behind fires while the *next* case is
+// running, so the line it printed was reported under that next case — a test
+// accused of printing something it never printed, which is worse than not
+// naming it at all, because a reader chasing the message finds it under code
+// that does not contain it. See ubugeeei-prod/uf#207.
+//
+// So the owner is an `AsyncLocalStorage`, and what `run.js` calls is
+// `runInTest` rather than an `enterTest` / `exitTest` pair: the store is only
+// carried by work started *inside* the case, so the case's setup, body and
+// teardown have to run within it. Everything they schedule inherits it,
+// whenever it eventually runs.
+//
+// # When there is no owner
+//
+// `getStore()` answers nothing outside a case, and a chunk with no owner is
+// filed under the file — which is right for an import, a `beforeAll`, or a
+// straggler from a case that is long gone.
+//
+// It is also the answer on a host whose storage does not reach the callback.
+// Deno 1.31 has `AsyncLocalStorage` and propagates it across `await`, but not
+// through `setTimeout`, so a detached callback there is filed under the file
+// rather than under the case that scheduled it. That degradation is the point:
+// of the two ways to be less than exact, naming the file says less, and naming
+// the next case says something false.
+//
+// `node:async_hooks` itself is not guarded for, because a guard could not run.
+// Node, Deno and Bun all provide it under the `node:` specifier, and a host
+// that had no `node:` builtins could not start this worker at all — `node:util`
+// is imported below, `node:readline` and `node:url` by `worker.js`. A `typeof`
+// check around the constructor would only ever execute on a host where this
+// module had already linked.
+//
+// The one thing this does not answer is a straggler that outlives its *file*:
+// the worker runs the next file in the same process, and a chunk still carrying
+// a name from the file before is a name the next file's report has no test for.
+// The host files it under the file it arrived in, which is honest but not
+// exact, and closing it properly needs the file's generation in the protocol.
+// That is ubugeeei-prod/uf#203, and it is not this module's to fix.
 
 // # Bounds
 //
@@ -49,6 +82,7 @@
 // nothing after it is kept. The budget starts over for each file, so a chatty
 // file does not silence the next one in the same worker.
 
+import { AsyncLocalStorage } from "node:async_hooks";
 import { format, inspect } from "node:util";
 
 import { userFrames } from "./frames.js";
@@ -95,9 +129,16 @@ const DECODER = new TextDecoder();
 /** What a stream's `write` calls when it has taken the chunk. */
 type WriteCallback = () => mixed;
 
+/**
+ * The case a write belongs to, kept in the asynchronous context it ran in.
+ *
+ * Full names rather than a record, because that is the whole of what a chunk
+ * needs to say and the protocol carries it as a string either way.
+ */
+const owner: AsyncLocalStorage<string> = new AsyncLocalStorage();
+
 let sink: OutputSink | null = null;
 let raw: ((chunk: string) => void) | null = null;
-let current: string | null = null;
 let captured = 0;
 let stopped = false;
 
@@ -159,7 +200,7 @@ function capture(stream: OutputStream, text: string): void {
     stopped = true;
   }
   captured += kept.length;
-  to({ stream, test: current, text: kept });
+  to({ stream, test: owner.getStore() ?? null, text: kept });
 }
 
 /** A stand-in for `process.stdout.write` / `process.stderr.write`. */
@@ -200,6 +241,19 @@ function writer(
  * Installed once, for the life of the worker: a worker runs many files, and
  * restoring the real methods between them would leave a window in which a
  * straggling `setTimeout` from the previous file writes into the protocol.
+ *
+ * # A page has no stream to take
+ *
+ * `uf test --browser` runs this same capture inside a page
+ * (`./browser/page.js`), and there the whole premise of the returned value is
+ * absent: there is no `process.stdout`, so there is nothing for a test to
+ * write into by accident, and the protocol's channel is a separate HTTP
+ * request rather than a stream anything else can reach. So the two `write`
+ * methods are only replaced when there are two `write` methods, and the
+ * "raw stream" handed back is a no-op nobody has a use for.
+ *
+ * Deliberately not a `typeof process` check at each use: the question is asked
+ * once, here, because the answer cannot change under a running host.
  */
 export function install(to: OutputSink): (chunk: string) => void {
   const global = host();
@@ -207,11 +261,14 @@ export function install(to: OutputSink): (chunk: string) => void {
   if (already != null) {
     return already;
   }
-  const stdout = global.process.stdout;
-  const real = stdout.write;
-  const protocol = (chunk: string) => {
-    real.call(stdout, chunk);
-  };
+  const stdout = global.process?.stdout;
+  const real = stdout?.write;
+  const protocol =
+    real == null
+      ? (_chunk: string) => {}
+      : (chunk: string) => {
+          real.call(stdout, chunk);
+        };
   raw = protocol;
   sink = to;
   for (const method of Object.keys(CONSOLE_STREAMS)) {
@@ -230,31 +287,41 @@ export function install(to: OutputSink): (chunk: string) => void {
     capture("stderr", `${userFrames(error.stack) ?? `Trace: ${error.message}`}\n`);
   };
 
-  global.process.stdout.write = writer("stdout");
-  global.process.stderr.write = writer("stderr");
+  if (global.process?.stdout != null) {
+    global.process.stdout.write = writer("stdout");
+    global.process.stderr.write = writer("stderr");
+  }
   return protocol;
 }
 
 /**
- * Say which case is running, so what it prints can be named.
+ * Run `body` as `name`, so what it prints — and what it leaves behind to print
+ * later — is filed under that case.
+ *
+ * The runner wraps one case's `beforeEach`, body and `afterEach` in a single
+ * call, because those are the one case's work. Whatever `body` returns is
+ * returned unchanged, so an `await` on this is an `await` on the case.
  *
- * The runner calls this around a case's body and hooks and clears it between
- * them: output written while the module is being imported, from a `beforeAll`,
- * or after the last case finished belongs to the file, not to whichever case
- * happened to run last.
+ * Nothing here needs an "and now nothing is running" counterpart. Output from
+ * an import, a `beforeAll` or a case that has already been reported was never
+ * inside this call, so it has no owner and is the file's — which is the
+ * property the previous module-level variable had to be reset to keep, and
+ * kept only for as long as nothing straggled.
  */
-export function enterTest(name: string): void {
-  current = name;
+export function runInTest<T>(name: string, body: () => T): T {
+  return owner.run(name, body);
 }
 
-/** Say that no case is running. */
-export function exitTest(): void {
-  current = null;
-}
-
-/** Start one file's output budget over, with no case running. */
+/**
+ * Start one file's output budget over.
+ *
+ * Only the budget: there is no current case to clear, because a case's
+ * ownership lives in the callbacks it started rather than in this module. A
+ * straggler from the file before still carries the name it was written under,
+ * which the host cannot match to a test of the new file and files under that
+ * file instead. See ubugeeei-prod/uf#203.
+ */
 export function startFile(): void {
   captured = 0;
   stopped = false;
-  current = null;
 }

package/internal/registry.js

@@ -30,13 +30,29 @@ export type Body = () => mixed | Promise<mixed>;
 /** The suffix written on a registration call. */
 export type Modifier = "none" | "only" | "skip" | "todo";
 
-/** One registered test case. */
+/**
+ * How a benchmark runs under `uf test --bench`.
+ *
+ * `warmup` calls are made and their times thrown away, then `iterations` calls
+ * are each timed. `timeout` is the budget for one call, in milliseconds, and
+ * the benchmark as a whole is held to that budget once per call.
+ */
+export type BenchOptions = {|
+  readonly warmup?: number,
+  readonly iterations?: number,
+  readonly timeout?: number,
+|};
+
+/** One registered test case, or one benchmark. */
 export type Case = {|
-  readonly kind: "test",
+  readonly kind: "test" | "bench",
   readonly name: string,
   readonly body: Body | null,
   readonly modifier: Modifier,
+  readonly skipReason: string | null,
   readonly timeoutMs: number | null,
+  /** How to run it, for a benchmark; `null` for a test. */
+  readonly bench: BenchOptions | null,
   readonly line: number,
   readonly column: number,
 |};
@@ -123,14 +139,18 @@ function addCase(
   body: Body | null,
   modifier: Modifier,
   timeoutMs: number | null,
+  skipReason: string | null = null,
+  bench: BenchOptions | null = null,
 ): void {
   const position = callSite();
   current.children.push({
-    kind: "test",
+    kind: bench == null ? "test" : "bench",
     name,
     body,
     modifier,
+    skipReason,
     timeoutMs,
+    bench,
     line: position.line,
     column: position.column,
   });
@@ -179,6 +199,9 @@ function caseApi(): $FlowFixMe {
   api.skip = (name: string, body?: Body) => {
     addCase(name, body ?? null, "skip", null);
   };
+  api.skipBecause = (name: string, reason: string, body?: Body) => {
+    addCase(name, body ?? null, "skip", null, reason);
+  };
   api.todo = (name: string, body?: Body) => {
     addCase(name, body ?? null, "todo", null);
   };
@@ -211,6 +234,34 @@ export const it: $FlowFixMe = caseApi();
 /** `test` is `it`, for people who write it that way. */
 export const test: $FlowFixMe = it;
 
+/** The `bench` API, and its modifiers. See [`suiteApi`] for the shape. */
+function benchApi(): $FlowFixMe {
+  const api: $FlowFixMe = (name: string, body: Body, options?: BenchOptions) => {
+    addCase(name, body, "none", options?.timeout ?? null, null, options ?? {});
+  };
+  api.only = (name: string, body: Body, options?: BenchOptions) => {
+    addCase(name, body, "only", options?.timeout ?? null, null, options ?? {});
+  };
+  api.skip = (name: string, body?: Body) => {
+    addCase(name, body ?? null, "skip", null, null, {});
+  };
+  api.todo = (name: string) => {
+    addCase(name, null, "todo", null, null, {});
+  };
+  return api;
+}
+
+/**
+ * Register one benchmark.
+ *
+ * `uf test` reports a benchmark as skipped, so a suite does not pay for timing
+ * one, and `uf test --bench` runs the benchmarks in place of the tests and
+ * reports how long each call took. `bench.only`, `bench.skip` and `bench.todo`
+ * do what `it`'s do. See [`BenchOptions`] for `warmup`, `iterations` and
+ * `timeout`.
+ */
+export const bench: $FlowFixMe = benchApi();
+
 /**
  * Substitute a row into a name, the way every runner spells it: `%s` for the
  * value, `%j` for its JSON.

package/internal/run.js

@@ -12,12 +12,19 @@
 import * as output from "./output.js";
 import * as snapshot from "./snapshot.js";
 import { AssertionError } from "./expect.js";
-import { firstUserSite, userFrames } from "./frames.js";
-import { type Body, type Case, type Suite, collected } from "./registry.js";
+import { type Site, firstUserSite, siteInFile, userFrames } from "./frames.js";
+import { type BenchOptions, type Body, type Case, type Suite, collected } from "./registry.js";
 
 /** How one case ended. */
 export type Outcome =
-  | {| readonly status: "passed" |}
+  | {|
+      readonly status: "passed",
+      /**
+       * One timing per measured call, in whole microseconds, for a benchmark
+       * run under `uf test --bench`.
+       */
+      readonly samples?: $ReadOnlyArray<number>,
+    |}
   | {|
       readonly status: "failed",
       readonly message: string,
@@ -27,7 +34,11 @@ export type Outcome =
       /** Where the failing assertion was written, when the stack says. */
       readonly site: {| readonly line: number, readonly column: number |} | null,
     |}
-  | {| readonly status: "skipped", readonly reason: "explicit" | "not-only" | "filtered" |}
+  | {|
+      readonly status: "skipped",
+      readonly reason: "explicit" | "not-only" | "filtered" | "bench" | "not-bench",
+      readonly message?: string | null,
+    |}
   | {| readonly status: "todo" |};
 
 /** One finished case, as the runner reports it. */
@@ -52,11 +63,31 @@ export type RunOptions = {|
    * which file that is — a test's name alone does not locate it.
    */
   readonly file?: string,
+  /**
+   * Run the benchmarks and report the tests skipped, rather than the other way
+   * round. `uf test --bench` sets it.
+   */
+  readonly bench?: boolean,
 |};
 
 /** Default budget for one case, matching what most runners use. */
 export const DEFAULT_TIMEOUT_MS: number = 5000;
 
+/** Calls a benchmark makes and throws away before it times any, unless it says. */
+export const DEFAULT_BENCH_WARMUP: number = 5;
+
+/** Timed calls a benchmark makes, unless it says. */
+export const DEFAULT_BENCH_ITERATIONS: number = 50;
+
+/** Most timed calls one benchmark may ask for, and most samples `uf` keeps. */
+export const MAX_BENCH_ITERATIONS: number = 100000;
+
+/** Most untimed calls one benchmark may ask for first. */
+export const MAX_BENCH_WARMUP: number = 10000;
+
+/** The longest delay `setTimeout` honours; past it, the timer fires at once. */
+const MAX_TIMER_MS = 2147483647;
+
 /** The separator between a suite's name and its child's. */
 export const NAME_SEPARATOR: string = " > ";
 
@@ -72,7 +103,9 @@ function fullName(path: $ReadOnlyArray<string>): string {
  */
 function hasOnly(node: Suite | Case, inherited: boolean): boolean {
   const marked = inherited || node.modifier === "only";
-  if (node.kind === "test") {
+  // A benchmark is a case like a test: `bench.only` restricts a file as
+  // `it.only` does.
+  if (node.kind !== "suite") {
     return marked;
   }
   return node.children.some((child) => hasOnly(child, marked));
@@ -102,7 +135,22 @@ async function withTimeout(body: Body, timeoutMs: number): Promise<void> {
   }
 }
 
-function failure(thrown: mixed): Outcome {
+/**
+ * Where to say a failure happened.
+ *
+ * The reported position is printed under the path of the file being run, so
+ * when that file is known the line has to come from it: `firstUserSite` will
+ * hand back the first frame of whatever library raised, and a library's line
+ * number wearing the test file's path sends the reader to the wrong place
+ * (ubugeeei-prod/uf#319). The fallback is for a caller that did not say which
+ * file it is running — `run` is driven directly by this repository's own
+ * tests as well as by the worker — and is what every failure used before.
+ */
+function siteOf(stack: string | null, file: string | null): Site | null {
+  return file == null || file === "" ? firstUserSite(stack, false) : siteInFile(stack, file);
+}
+
+function failure(thrown: mixed, file: string | null): Outcome {
   if (thrown instanceof AssertionError) {
     const stack = userFrames(thrown.stack);
     return {
@@ -111,7 +159,7 @@ function failure(thrown: mixed): Outcome {
       stack,
       expected: thrown.expected,
       received: thrown.received,
-      site: firstUserSite(stack, false),
+      site: siteOf(stack, file),
     };
   }
   if (thrown instanceof Error) {
@@ -122,7 +170,7 @@ function failure(thrown: mixed): Outcome {
       stack,
       expected: null,
       received: null,
-      site: firstUserSite(stack, false),
+      site: siteOf(stack, file),
     };
   }
   return {
@@ -173,12 +221,17 @@ async function runCase(
     });
   };
 
-  if (test.modifier === "todo" || test.body == null) {
+  if (test.modifier === "todo") {
     report({ status: "todo" });
     return true;
   }
   if (context.skipped || test.modifier === "skip") {
-    report({ status: "skipped", reason: "explicit" });
+    report({ status: "skipped", reason: "explicit", message: test.skipReason });
+    return true;
+  }
+  const body = test.body;
+  if (body == null) {
+    report({ status: "todo" });
     return true;
   }
   if (!context.onlyPath) {
@@ -190,42 +243,126 @@ async function runCase(
     report({ status: "skipped", reason: "filtered" });
     return true;
   }
+  // A run is the tests or the benchmarks, never both. A benchmark timed inside
+  // an ordinary run would slow every suite that has one, beside workers busy
+  // with other files; a test inside a run of benchmarks would be timed with
+  // them.
+  const benchmark = test.kind === "bench";
+  if (benchmark !== (options.bench === true)) {
+    report({ status: "skipped", reason: benchmark ? "bench" : "not-bench" });
+    return true;
+  }
 
   const timeoutMs = test.timeoutMs ?? options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   let outcome: Outcome = { status: "passed" };
-  // From here to the end of teardown is exactly the window in which this
-  // case's own code runs, so it is exactly the window whose printing is this
-  // case's. The cases above never reach it: nothing runs for a `todo`, a
-  // `skip` or a filtered-out case, so nothing of theirs can print.
-  output.enterTest(name);
-  try {
-    for (const hook of context.beforeEach) {
-      await withTimeout(hook, timeoutMs);
-    }
-    await withTimeout(test.body, timeoutMs);
-  } catch (thrown) {
-    outcome = failure(thrown);
-  }
-  // Teardown runs whatever happened above, and only reports its own failure
-  // when the body had not already failed.
-  for (const hook of context.afterEach) {
+  let samples: $ReadOnlyArray<number> | null = null;
+  // Setup, body and teardown run *inside* the case's output context, and that
+  // nesting is the whole of the fix for #207. What names a printed line is no
+  // longer where the runner had got to when the line arrived — which named the
+  // next case for anything a `setTimeout` left behind — but which case's work
+  // the write descends from. A callback scheduled here keeps this name however
+  // late it fires.
+  //
+  // The cases above never reach this: nothing runs for a `todo`, a `skip` or a
+  // filtered-out case, so nothing of theirs can print.
+  await output.runInTest(name, async () => {
     try {
-      await withTimeout(hook, timeoutMs);
+      for (const hook of context.beforeEach) {
+        await withTimeout(hook, timeoutMs);
+      }
+      if (benchmark) {
+        samples = await measure(body, test.bench, timeoutMs);
+      } else {
+        await withTimeout(body, timeoutMs);
+      }
     } catch (thrown) {
-      if (outcome.status === "passed") {
-        outcome = failure(thrown);
+      outcome = failure(thrown, options.file ?? null);
+    }
+    // Teardown runs whatever happened above, and only reports its own failure
+    // when the body had not already failed.
+    for (const hook of context.afterEach) {
+      try {
+        await withTimeout(hook, timeoutMs);
+      } catch (thrown) {
+        if (outcome.status === "passed") {
+          outcome = failure(thrown, options.file ?? null);
+        }
       }
     }
-  }
-  // No test is running once this one is reported, so a snapshot taken outside
-  // one fails with something better than a key belonging to whichever test
-  // happened to run last, and a line printed outside one is the file's.
-  output.exitTest();
+  });
+  // The `await` above resumes outside the context it entered, so this line and
+  // everything after it belong to no case again — a line printed between cases
+  // is the file's, and a snapshot taken outside one fails with something better
+  // than a key belonging to whichever test happened to run last.
   snapshot.exitTest();
-  report(outcome);
+  report(outcome.status === "passed" && samples != null ? { status: "passed", samples } : outcome);
   return outcome.status !== "failed";
 }
 
+/**
+ * Time a benchmark's body: `warmup` calls thrown away, then `iterations` timed.
+ *
+ * The body is called directly and awaited, with nothing else between the two
+ * readings of the clock: a timer set around every call would be timed with
+ * it, and on a body that takes microseconds that is most of the number. The
+ * budget is held around the whole loop instead, at one call's budget per
+ * call, so a benchmark that hangs still fails rather than holding the worker.
+ */
+async function measure(
+  body: Body,
+  options: BenchOptions | null,
+  timeoutMs: number,
+): Promise<$ReadOnlyArray<number>> {
+  const warmup = count("warmup", options?.warmup, DEFAULT_BENCH_WARMUP, 0, MAX_BENCH_WARMUP);
+  const iterations = count(
+    "iterations",
+    options?.iterations,
+    DEFAULT_BENCH_ITERATIONS,
+    1,
+    MAX_BENCH_ITERATIONS,
+  );
+  const rounds = warmup + iterations;
+  const samples: Array<number> = [];
+  await withTimeout(
+    async () => {
+      for (let round = 0; round < rounds; round += 1) {
+        const started = performance.now();
+        await body();
+        if (round >= warmup) {
+          samples.push(Math.round((performance.now() - started) * 1000));
+        }
+      }
+    },
+    Math.min(timeoutMs * rounds, MAX_TIMER_MS),
+  );
+  return samples;
+}
+
+/**
+ * A benchmark option that has to be a whole number from `least` to `most`.
+ *
+ * Refused rather than clamped: `iterations: 0` or `warmup: 1e9` is a mistake,
+ * and a benchmark quietly run some other number of times reports numbers
+ * about a run nobody asked for.
+ */
+function count(
+  option: string,
+  value: ?number,
+  fallback: number,
+  least: number,
+  most: number,
+): number {
+  if (value == null) {
+    return fallback;
+  }
+  if (!Number.isInteger(value) || value < least || value > most) {
+    throw new Error(
+      `bench \`${option}\` has to be a whole number from ${least} to ${most}, and was ${String(value)}`,
+    );
+  }
+  return value;
+}
+
 /**
  * Walk one suite, running what it contains.
  *
@@ -238,6 +375,7 @@ async function runSuite(
   onlyMode: boolean,
   emit: (result: Result) => void,
   state: {| bail: boolean |},
+  setUpAncestors: () => Promise<void>,
 ): Promise<boolean> {
   const skipped = context.skipped || node.modifier === "skip" || node.modifier === "todo";
   const onlyPath = !onlyMode || context.onlyPath || node.modifier === "only";
@@ -252,11 +390,18 @@ async function runSuite(
 
   // `beforeAll` is deferred until a case in this suite actually runs, so a
   // fully skipped suite never sets anything up. `afterAll` mirrors it.
+  //
+  // "In this suite" means anywhere under it. A suite whose children are all
+  // suites has no case of its own, and setting up only for a direct child left
+  // every `beforeAll` in an ordinary file — one where the tests live inside a
+  // `describe` — never running at all, silently. The chain is walked outermost
+  // first, so an inner suite's setup sees what the outer one did.
   let setUp = false;
   const setUpOnce = async () => {
     if (setUp) {
       return;
     }
+    await setUpAncestors();
     setUp = true;
     for (const hook of node.beforeAll) {
       await withTimeout(hook, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
@@ -268,13 +413,16 @@ async function runSuite(
     if (state.bail) {
       break;
     }
-    if (child.kind === "test") {
+    if (child.kind !== "suite") {
       const willRun =
         !inner.skipped &&
         child.modifier !== "skip" &&
         child.modifier !== "todo" &&
         child.body != null &&
-        (!onlyMode || inner.onlyPath || child.modifier === "only");
+        (!onlyMode || inner.onlyPath || child.modifier === "only") &&
+        // A benchmark in a run of the tests, or a test in a run of the
+        // benchmarks, is reported skipped and sets nothing up.
+        (child.kind === "bench") === (options.bench === true);
       if (willRun) {
         try {
           await setUpOnce();
@@ -286,7 +434,7 @@ async function runSuite(
             line: child.line,
             column: child.column,
             durationMicros: 0,
-            outcome: failure(thrown),
+            outcome: failure(thrown, options.file ?? null),
           });
           passed = false;
           continue;
@@ -299,7 +447,7 @@ async function runSuite(
       const ok = await runCase(child, childContext, options, emit);
       passed = passed && ok;
     } else {
-      const ok = await runSuite(child, inner, options, onlyMode, emit, state);
+      const ok = await runSuite(child, inner, options, onlyMode, emit, state, setUpOnce);
       passed = passed && ok;
     }
   }
@@ -333,5 +481,5 @@ export async function run(options: RunOptions, emit: (result: Result) => void):
     skipped: false,
     onlyPath: !onlyMode,
   };
-  await runSuite(root, context, options, onlyMode, emit, { bail: false });
+  await runSuite(root, context, options, onlyMode, emit, { bail: false }, async () => {});
 }