@ultimat3/testing 1.0.0

package/src/harness.ts

@@ -0,0 +1,114 @@
+// Boot an app in-process for a test: its own cloned database, a frozen clock, a seeded RNG and a
+// sealed network, torn down afterwards. In-process because a spawned server turns every assertion
+// into a race and every failure into a log-scraping exercise.
+
+import { afterAll, beforeAll, describe, test } from 'bun:test';
+import { installDeterminism, restoreDeterminism } from './determinism';
+import { allowHost, resetNetwork, sealNetwork, unsealNetwork } from './sealed-network';
+import type { TemplateDbConfig, WorkerDatabase } from './template-db';
+import { acquireWorkerDatabase } from './template-db';
+import type { TestType } from './test-types';
+import { testName } from './test-types';
+
+export interface BootedApp {
+  /** The app's request handler — the same one `ROLE=web` serves in production. */
+  fetch(request: Request): Response | Promise<Response>;
+  close?(): Promise<void>;
+}
+
+export interface AppOptions {
+  /** Boot the app against the database URL the harness just cloned. */
+  boot(context: { readonly databaseUrl: string }): Promise<BootedApp>;
+  readonly db?: TemplateDbConfig;
+  readonly seed?: (databaseUrl: string) => Promise<void>;
+  /** Hosts the test may reach for real. Everything else fails with X_TEST_NETWORK_SEALED. */
+  readonly allowHosts?: readonly string[];
+  readonly seedValue?: number;
+  readonly now?: string;
+}
+
+export interface AppHandle {
+  readonly db: WorkerDatabase;
+  /** Call the app by path: `app.request('/api/health')`. */
+  request(path: string, init?: RequestInit): Promise<Response>;
+  json<T>(path: string, init?: RequestInit): Promise<T>;
+}
+
+const BASE = 'http://app.test';
+
+async function boot(
+  options: AppOptions,
+): Promise<{ handle: AppHandle; close: () => Promise<void> }> {
+  installDeterminism({
+    ...(options.seedValue === undefined ? {} : { seed: options.seedValue }),
+    ...(options.now === undefined ? {} : { now: options.now }),
+  });
+  sealNetwork();
+  for (const host of options.allowHosts ?? []) allowHost(host);
+
+  const db = await acquireWorkerDatabase(options.db ?? {});
+  if (options.seed !== undefined) await options.seed(db.url);
+  const app = await options.boot({ databaseUrl: db.url });
+
+  const handle: AppHandle = {
+    db,
+    request: async (path, init) => app.fetch(new Request(new URL(path, BASE), init)),
+    json: async <T>(path: string, init?: RequestInit): Promise<T> => {
+      const response = await app.fetch(new Request(new URL(path, BASE), init));
+      return (await response.json()) as T;
+    },
+  };
+
+  return {
+    handle,
+    close: async () => {
+      await app.close?.();
+      await db.drop();
+      resetNetwork();
+      unsealNetwork();
+      restoreDeterminism();
+    },
+  };
+}
+
+/**
+ * A describe block with the app booted once for every test inside it. The accessor is a function
+ * because the app does not exist until beforeAll has run.
+ */
+export function describeApp(
+  name: string,
+  options: AppOptions,
+  body: (app: () => AppHandle) => void,
+): void {
+  describe(name, () => {
+    let booted: { handle: AppHandle; close: () => Promise<void> } | undefined;
+    beforeAll(async () => {
+      booted = await boot(options);
+    });
+    afterAll(async () => {
+      await booted?.close();
+      booted = undefined;
+    });
+    body(() => {
+      if (booted === undefined) throw new ReferenceError('app is not booted yet');
+      return booted.handle;
+    });
+  });
+}
+
+/** A single test with its own app, for the cases that need an isolated boot. */
+export function testApp(
+  name: string,
+  options: AppOptions,
+  body: (app: AppHandle) => Promise<void>,
+  type: TestType = 'unit',
+): void {
+  test(testName(type, name), async () => {
+    const booted = await boot(options);
+    try {
+      await body(booted.handle);
+    } finally {
+      await booted.close();
+    }
+  });
+}

package/src/index.ts

@@ -0,0 +1,133 @@
+export type {
+  FixtureBody,
+  FixtureFactory,
+  FixtureMap,
+  FixtureRegistration,
+  Fixtures,
+} from './fixtures';
+export {
+  clearFixtures,
+  defineFixtures,
+  fixtureSnapshot,
+  fixtureTest,
+  registeredFixtures,
+  requestedFixtures,
+} from './fixtures';
+// Public API of @ultimat3/testing. Explicit re-exports: the preload has its own entry point so a
+// bunfig can load side effects without importing the whole harness.
+
+// Re-exported so an app test has one import line instead of two, and so `expect` carries
+// this package's custom matchers (`toBeUltimateError`, `toEmitSteps`, …) already installed.
+// Importing `expect` straight from `bun:test` in an app test gets the matchers only if some
+// other module happened to load first — a load-order dependency, which is a flake.
+// `test` is OURS (fixture-injecting); everything else passes through. Re-exported so an app
+// test has one import line, and so `expect` carries this package's matchers already installed.
+export { afterAll, afterEach, beforeAll, beforeEach, describe, expect } from 'bun:test';
+export type { DeterminismOptions } from './determinism';
+export {
+  advanceClock,
+  assertDeterministic,
+  DEFAULT_NOW,
+  DEFAULT_SEED,
+  frozenClock,
+  frozenNow,
+  installDeterminism,
+  isDeterminismInstalled,
+  restoreDeterminism,
+  seededRandom,
+  seededUuid,
+  setFrozenClock,
+} from './determinism';
+export type { TestingErrorCode } from './errors';
+export {
+  FixtureUnavailableError,
+  NetworkOfflineError,
+  NetworkSealedError,
+  NondeterministicError,
+  TESTING_ERROR_CODES,
+  TESTING_ERROR_TITLES,
+  TestDatabaseUnavailableError,
+} from './errors';
+export type { EntityLike, EntityRegistry, Factory, FactoryOptions } from './factories';
+export { defineFactory, factoriesFor } from './factories';
+export type { TestClock, TestDuration } from './fixture-clock';
+export { createTestClock } from './fixture-clock';
+export type {
+  DriverFixtureName,
+  DriverFixtures,
+  LiveFeed,
+  LiveFeedPatch,
+  LiveTarget,
+  SignIn,
+  Subscribe,
+  TestBudget,
+  TestDeploy,
+} from './fixture-drivers';
+export { DRIVER_FIXTURE_NEEDS, driverFixtures, unavailableFixture } from './fixture-drivers';
+export type { JobRunTrace, RunJobs, StepTally } from './fixture-jobs';
+export { createRunJobs } from './fixture-jobs';
+export type { MailRef, TestMail } from './fixture-mail';
+export { createTestMail } from './fixture-mail';
+export type { TestNetwork } from './fixture-network';
+export { createTestNetwork } from './fixture-network';
+export { fixtureTest as test } from './fixtures';
+export {
+  ALL_FIXTURE_NAMES,
+  DRIVER_FIXTURE_NAMES,
+  FRAMEWORK_FIXTURE_NAMES,
+  registerFrameworkFixtures,
+} from './framework-fixtures';
+export type { AppHandle, AppOptions, BootedApp } from './harness';
+export { describeApp, testApp } from './harness';
+export type { MatcherResult } from './matchers';
+export { matchersInstalled, recordSteps } from './matchers';
+export type { MockRoute, NetworkState } from './sealed-network';
+// `setNetworkState` is deliberately not here: it is the offline gate's one writer, and a test that
+// called it directly would bypass the `network` fixture's disposal and leave the whole process
+// offline for every file after it. The fixture is the way to go offline — there is no second one.
+export {
+  allowHost,
+  isNetworkSealed,
+  mockFetch,
+  mockJson,
+  networkState,
+  requestedUrls,
+  resetNetwork,
+  sealNetwork,
+  unsealNetwork,
+} from './sealed-network';
+export type { SqlRunner, TemplateDbConfig, WorkerDatabase } from './template-db';
+export {
+  acquireWorkerDatabase,
+  cloneSql,
+  createTemplateSql,
+  DEFAULT_TEMPLATE,
+  databaseNameFor,
+  dropSql,
+  lockSql,
+  unlockSql,
+  urlFor,
+  workerId,
+} from './template-db';
+export type {
+  E2eBody,
+  E2eFixtures,
+  EvalCase,
+  EvalOptions,
+  LocatorLike,
+  OpenApiLike,
+  PageLike,
+  TestType,
+} from './test-types';
+export {
+  contractTest,
+  e2eTest,
+  evalTest,
+  jobTest,
+  liveTest,
+  SEPARATOR,
+  TEST_TYPES,
+  testName,
+  unitTest,
+  useE2eDriver,
+} from './test-types';

package/src/matchers.ts

@@ -0,0 +1,182 @@
+// Custom expectations. Each one exists because the assertion it replaces was being written by
+// hand, differently, in every test — and a hand-written version of "did this policy deny?" is how
+// a test ends up asserting on the wrong branch.
+
+import { expect } from 'bun:test';
+import { TestJobExpectedError, TestSchemaExpectedError } from './errors';
+import type { OpenApiLike } from './test-types';
+
+export interface MatcherResult {
+  readonly pass: boolean;
+  message(): string;
+}
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null;
+
+const codeOf = (value: unknown): string | undefined => {
+  if (!isRecord(value)) return undefined;
+  const code = value['code'];
+  return typeof code === 'string' ? code : undefined;
+};
+
+/** Standard Schema is the validation contract, so every blessed schema exposes `~standard`. */
+interface StandardSchema {
+  readonly '~standard': {
+    validate(value: unknown): { issues?: unknown } | Promise<{ issues?: unknown }>;
+  };
+}
+
+const isStandardSchema = (value: unknown): value is StandardSchema =>
+  isRecord(value) && isRecord(value['~standard']);
+
+async function hasIssues(schema: unknown, input: unknown): Promise<boolean> {
+  if (!isStandardSchema(schema)) {
+    throw new TestSchemaExpectedError();
+  }
+  const result = await schema['~standard'].validate(input);
+  const issues = result.issues;
+  return Array.isArray(issues) ? issues.length > 0 : issues !== undefined;
+}
+
+interface PolicyLike {
+  evaluate(context: Readonly<Record<string, unknown>>): boolean | Promise<boolean>;
+}
+
+/** A `Policy` from @ultimat3/policy. It decides through `run()`; there is no `evaluate()` on it. */
+interface RunnablePolicy {
+  run(args: Readonly<Record<string, unknown>>): { readonly allowed: boolean };
+}
+
+const isPolicy = (value: unknown): value is PolicyLike =>
+  isRecord(value) && typeof value['evaluate'] === 'function';
+
+const isRunnablePolicy = (value: unknown): value is RunnablePolicy =>
+  isRecord(value) && typeof value['run'] === 'function';
+
+/**
+ * `undefined` means "not a policy at all", which is a different failure from a denial.
+ * `row` is defaulted the way `evaluate()` in @ultimat3/policy defaults it, so a test about a
+ * rule that decides on input alone does not have to write `row: null` for the predicate to see
+ * the field — and a row-level test still passes its own `row` and wins, because it comes second.
+ */
+async function decide(
+  policy: unknown,
+  context: Readonly<Record<string, unknown>>,
+): Promise<boolean | undefined> {
+  if (isRunnablePolicy(policy)) return policy.run({ row: null, ...context }).allowed;
+  if (isPolicy(policy)) return policy.evaluate(context);
+  return undefined;
+}
+
+interface JobLike {
+  run(context: {
+    input: unknown;
+    step: StepRecorder;
+    ctx: Record<string, unknown>;
+  }): Promise<unknown>;
+}
+
+interface StepRecorder {
+  run<T>(name: string, body: () => T | Promise<T>): Promise<T>;
+  sleep(duration: string): Promise<void>;
+}
+
+const isJob = (value: unknown): value is JobLike =>
+  isRecord(value) && typeof value['run'] === 'function';
+
+/** Runs the job with a recording step API, so the assertion is on the sequence, not the effects. */
+export async function recordSteps(job: unknown, input: unknown = {}): Promise<readonly string[]> {
+  if (!isJob(job)) throw new TestJobExpectedError();
+  const names: string[] = [];
+  const step: StepRecorder = {
+    run: async (name, body) => {
+      names.push(name);
+      return body();
+    },
+    sleep: async () => undefined,
+  };
+  await job.run({ input, step, ctx: {} });
+  return names;
+}
+
+const result = (pass: boolean, message: string): MatcherResult => ({
+  pass,
+  message: () => message,
+});
+
+expect.extend({
+  toBeUltimateError(received: unknown, code?: string) {
+    const actual = codeOf(received);
+    if (actual === undefined) {
+      return result(false, `expected an UltimateError, received ${typeof received}`);
+    }
+    if (code === undefined) return result(true, `expected not to be an UltimateError`);
+    return result(actual === code, `expected error code ${code}, received ${actual}`);
+  },
+
+  async toDenyPolicy(received: unknown, context: Readonly<Record<string, unknown>>) {
+    const allowed = await decide(received, context);
+    if (allowed === undefined) {
+      return result(
+        false,
+        'expected a policy — an object with run() (@ultimat3/policy) or evaluate()',
+      );
+    }
+    return result(!allowed, `expected the policy to deny ${JSON.stringify(context)}`);
+  },
+
+  async toEmitSteps(received: unknown, expected: readonly string[]) {
+    const names = await recordSteps(received);
+    return result(
+      JSON.stringify(names) === JSON.stringify(expected),
+      `expected steps ${expected.join(' -> ')}, ran ${names.join(' -> ')}`,
+    );
+  },
+
+  toMatchOpenApi(received: unknown, committed: OpenApiLike) {
+    const current = received as OpenApiLike;
+    const before = committed.operations.map((operation) => operation.operationId).sort();
+    const after = current.operations.map((operation) => operation.operationId).sort();
+    const removed = before.filter((id) => !after.includes(id));
+    return result(
+      removed.length === 0,
+      `contract removed operation(s): ${removed.join(', ')} — bump the package version or restore them`,
+    );
+  },
+
+  toBeWithinBudget(received: unknown, limit: number) {
+    if (typeof received !== 'number') {
+      return result(
+        false,
+        `expected a number to compare against the budget, got ${typeof received}`,
+      );
+    }
+    return result(received <= limit, `expected ${received} to be within the budget of ${limit}`);
+  },
+
+  async toRejectInput(received: unknown, input: unknown) {
+    const rejected = await hasIssues(received, input);
+    return result(rejected, `expected the schema to reject ${JSON.stringify(input)}`);
+  },
+
+  async toAcceptInput(received: unknown, input: unknown) {
+    const rejected = await hasIssues(received, input);
+    return result(!rejected, `expected the schema to accept ${JSON.stringify(input)}`);
+  },
+});
+
+declare module 'bun:test' {
+  interface Matchers<T> {
+    toBeUltimateError(code?: string): T;
+    toDenyPolicy(context: Readonly<Record<string, unknown>>): Promise<T>;
+    toEmitSteps(steps: readonly string[]): Promise<T>;
+    toMatchOpenApi(committed: OpenApiLike): T;
+    toBeWithinBudget(limit: number): T;
+    toRejectInput(input: unknown): Promise<T>;
+    toAcceptInput(input: unknown): Promise<T>;
+  }
+}
+
+/** Imported for its side effect by the preload; exported so a test can be explicit about it. */
+export const matchersInstalled = true;

package/src/preload.ts

@@ -0,0 +1,25 @@
+// The bunfig preload for apps: frozen clock, seeded RNG, sealed network, custom matchers, and
+// the framework's fixture bag. Loaded once per test process, before any test file — an app never
+// has to remember to call it.
+//
+//   [test]
+//   preload = ["@ultimat3/testing/preload"]
+
+import { installDeterminism } from './determinism';
+import { registerFrameworkFixtures } from './framework-fixtures';
+import './matchers';
+import { sealNetwork } from './sealed-network';
+
+const seed = Number.parseInt(Bun.env['ULTIMATE_TEST_SEED'] ?? '', 10);
+const now = Bun.env['ULTIMATE_TEST_NOW'];
+
+installDeterminism({
+  ...(Number.isFinite(seed) ? { seed } : {}),
+  ...(now === undefined ? {} : { now }),
+});
+
+registerFrameworkFixtures();
+
+// Opt-out exists for one case: a test that deliberately exercises a real integration in a job the
+// team runs on purpose. It is an env var, not an API, so it cannot be set from inside a test file.
+if (Bun.env['ULTIMATE_TEST_ALLOW_NET'] !== '1') sealNetwork();

package/src/sealed-network.ts

@@ -0,0 +1,150 @@
+// Sealed network. Any egress a test did not explicitly mock or allow fails the test with the URL
+// and the line that fixes it. A test that quietly reaches the internet is a test that fails in CI
+// for reasons nobody can reproduce — so the default is "nothing gets out".
+
+import { isSelfOrigin } from '@ultimat3/core';
+import { NetworkOfflineError, NetworkRaceError, NetworkSealedError } from './errors';
+
+export type FetchLike = typeof globalThis.fetch;
+
+/**
+ * `offline` is the cable pulled; `dropped` is the same for a request, but tells a transport its
+ * connection was cut rather than closed — so it reconnects and resumes instead of resubscribing.
+ */
+export type NetworkState = 'online' | 'offline' | 'dropped';
+
+export interface MockRoute {
+  /** Exact URL, or a prefix ending in `*`, or a RegExp. */
+  readonly match: string | RegExp;
+  readonly handler: (request: Request) => Response | Promise<Response>;
+}
+
+interface SealState {
+  readonly allowed: Set<string>;
+  readonly mocks: MockRoute[];
+  readonly seen: string[];
+  original: FetchLike | undefined;
+  network: NetworkState;
+}
+
+const state: SealState = {
+  allowed: new Set(),
+  mocks: [],
+  seen: [],
+  original: undefined,
+  network: 'online',
+};
+
+const matches = (route: MockRoute, url: string): boolean => {
+  if (route.match instanceof RegExp) return route.match.test(url);
+  if (route.match.endsWith('*')) return url.startsWith(route.match.slice(0, -1));
+  return route.match === url;
+};
+
+const urlOf = (input: RequestInfo | URL): string => {
+  if (typeof input === 'string') return input;
+  if (input instanceof URL) return input.toString();
+  return input.url;
+};
+
+const methodOf = (input: RequestInfo | URL, init: RequestInit | undefined): string => {
+  if (init?.method !== undefined) return init.method.toUpperCase();
+  if (typeof input === 'object' && 'method' in input) return input.method.toUpperCase();
+  return 'GET';
+};
+
+/** Replace global fetch. Idempotent: sealing twice keeps the one original around. */
+export function sealNetwork(): void {
+  if (state.original !== undefined) return;
+  state.original = globalThis.fetch;
+  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+    const url = urlOf(input);
+    state.seen.push(url);
+    // Ahead of the mocks on purpose: a test that mocked Stripe and then went offline is asserting
+    // the offline path, and a mock that still answered would be the one thing that path never sees.
+    if (state.network !== 'online') {
+      throw new NetworkOfflineError({
+        url,
+        method: methodOf(input, init),
+        mode: state.network,
+      });
+    }
+    const mock = state.mocks.find((route) => matches(route, url));
+    if (mock !== undefined) {
+      return mock.handler(input instanceof Request ? input : new Request(url, init));
+    }
+    // A server this process booted is not egress — the port is one the kernel just handed us, so
+    // there is nothing to allowlist ahead of time. Without this, a socket test's only option is to
+    // unseal the network wholesale, which then hides the real egress it was meant to catch.
+    const host = safeHost(url);
+    if (isSelfOrigin(url) || (host !== undefined && state.allowed.has(host))) {
+      const original = state.original;
+      if (original === undefined) throw new NetworkRaceError();
+      return original(input, init);
+    }
+    throw new NetworkSealedError({
+      url,
+      method: methodOf(input, init),
+      allowed: [...state.allowed],
+    });
+  }) as FetchLike;
+}
+
+export function unsealNetwork(): void {
+  if (state.original === undefined) return;
+  globalThis.fetch = state.original;
+  state.original = undefined;
+}
+
+/** Whether the patch is installed. The `network` fixture seals before going offline, so that
+ *  `offline()` has teeth in a process that deliberately unsealed (`ULTIMATE_TEST_ALLOW_NET=1`). */
+export const isNetworkSealed = (): boolean => state.original !== undefined;
+
+/** The one writer of the offline gate — `createTestNetwork()`. Never call it from a test body. */
+export function setNetworkState(next: NetworkState): void {
+  state.network = next;
+}
+
+export const networkState = (): NetworkState => state.network;
+
+function safeHost(url: string): string | undefined {
+  try {
+    return new URL(url).host;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Let one host through for real. Deliberately per-host, never per-test-file or global. */
+export function allowHost(host: string): void {
+  state.allowed.add(host);
+}
+
+export function mockFetch(
+  match: string | RegExp,
+  handler: (request: Request) => Response | Promise<Response>,
+): void {
+  state.mocks.unshift({ match, handler });
+}
+
+/** Convenience for the common case: a JSON body and a status. */
+export function mockJson(match: string | RegExp, body: unknown, status = 200): void {
+  mockFetch(
+    match,
+    () =>
+      new Response(JSON.stringify(body), {
+        status,
+        headers: { 'content-type': 'application/json' },
+      }),
+  );
+}
+
+/** Every URL a test attempted, in order — the fastest way to see what a failure actually called. */
+export const requestedUrls = (): readonly string[] => [...state.seen];
+
+export function resetNetwork(): void {
+  state.allowed.clear();
+  state.mocks.length = 0;
+  state.seen.length = 0;
+  state.network = 'online';
+}