@ultimat3/testing 1.0.0

package/src/template-db.ts

@@ -0,0 +1,163 @@
+// Real parallel database isolation: N workers, each owning a Postgres database cloned from one
+// migrated template with `CREATE DATABASE ... TEMPLATE`. Never mock the database — clone it. No
+// transaction-rollback wrapper (which breaks anything that commits), no shared schema with a
+// `truncate` between tests (which serialises the suite and still leaks sequences).
+
+import { TestDatabaseUnavailableError } from './errors';
+
+export type DbKind = 'postgres' | 'pglite';
+
+export interface SqlRunner {
+  /** Run a statement that returns nothing. */
+  exec(sql: string): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface TemplateDbConfig {
+  /** Connection URL with rights to CREATE DATABASE. Absent means "fall back to PGlite". */
+  readonly adminUrl?: string | undefined;
+  readonly templateName?: string;
+  /** Applied once, into the template, under an advisory lock. */
+  readonly migrate?: (url: string) => Promise<void>;
+}
+
+export interface WorkerDatabase {
+  readonly kind: DbKind;
+  readonly worker: number;
+  readonly database: string;
+  readonly url: string;
+  drop(): Promise<void>;
+}
+
+export const DEFAULT_TEMPLATE = 'ultimate_test_template';
+
+/**
+ * Bun exposes the test worker index in the environment; a plain `bun test` run is worker 0. The
+ * pid fallback keeps two hand-run processes from colliding on the same database.
+ */
+export function workerId(env: Readonly<Record<string, string | undefined>>, pid = 0): number {
+  // ULTIMATE_TEST_WORKER is checked FIRST because `x test` assigns it deliberately, one index per
+  // shard. A runner-set value must beat anything the runtime invents: if Bun ever populates
+  // BUN_TEST_WORKER_ID itself, two shards could resolve to the same index and then race on the
+  // same cloned database — a data-dependent failure that would look like a flaky test.
+  for (const key of ['ULTIMATE_TEST_WORKER', 'BUN_TEST_WORKER_ID', 'JEST_WORKER_ID']) {
+    const raw = env[key];
+    if (raw === undefined) continue;
+    const parsed = Number.parseInt(raw, 10);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return pid === 0 ? 0 : pid % 1024;
+}
+
+export const databaseNameFor = (template: string, worker: number): string =>
+  `${template}_w${worker}`;
+
+/** Swap the database segment of a Postgres URL, keeping credentials and query parameters. */
+export function urlFor(adminUrl: string, database: string): string {
+  const url = new URL(adminUrl);
+  url.pathname = `/${database}`;
+  return url.toString();
+}
+
+export const createTemplateSql = (template: string): string =>
+  `CREATE DATABASE "${template}" TEMPLATE template0`;
+
+export const cloneSql = (template: string, target: string): string =>
+  `CREATE DATABASE "${target}" TEMPLATE "${template}"`;
+
+export const dropSql = (target: string): string =>
+  `DROP DATABASE IF EXISTS "${target}" WITH (FORCE)`;
+
+/** hashtext-based advisory lock: one worker migrates the template, the rest wait for it. */
+export const lockSql = (template: string): string =>
+  `SELECT pg_advisory_lock(hashtext('${template}'))`;
+
+export const unlockSql = (template: string): string =>
+  `SELECT pg_advisory_unlock(hashtext('${template}'))`;
+
+export interface TemplateDbDeps {
+  /** Opens an admin connection. Injected so the unit tests never need a server. */
+  readonly connect: (url: string) => SqlRunner;
+  readonly env: Readonly<Record<string, string | undefined>>;
+  readonly pid?: number;
+}
+
+const defaultConnect = (url: string): SqlRunner => {
+  // Bun.SQL is the blessed Postgres client; no driver package, no pool config to get wrong.
+  const sql = new Bun.SQL(url);
+  return {
+    exec: async (statement: string) => {
+      await sql.unsafe(statement);
+    },
+    close: async () => {
+      await sql.close();
+    },
+  };
+};
+
+/**
+ * Give this worker its own database. The first caller creates and migrates the template; every
+ * caller clones it. Returns a PGlite-backed handle when no Postgres is configured, so `bun test`
+ * works on a laptop with nothing installed.
+ */
+export async function acquireWorkerDatabase(
+  config: TemplateDbConfig = {},
+  deps: Partial<TemplateDbDeps> = {},
+): Promise<WorkerDatabase> {
+  const env = deps.env ?? (Bun.env as Readonly<Record<string, string | undefined>>);
+  const connect = deps.connect ?? defaultConnect;
+  const template = config.templateName ?? DEFAULT_TEMPLATE;
+  const worker = workerId(env, deps.pid ?? process.pid);
+  const database = databaseNameFor(template, worker);
+  const adminUrl = config.adminUrl ?? env['TEST_DATABASE_URL'] ?? env['DATABASE_URL'];
+
+  if (adminUrl === undefined || adminUrl.length === 0) {
+    return {
+      kind: 'pglite',
+      worker,
+      database,
+      url: `pglite://memory/${database}`,
+      drop: async () => undefined,
+    };
+  }
+
+  const admin = connect(adminUrl);
+  try {
+    await admin.exec(lockSql(template));
+    try {
+      await admin.exec(createTemplateSql(template));
+      if (config.migrate !== undefined) await config.migrate(urlFor(adminUrl, template));
+    } catch (error) {
+      // "already exists" means another worker migrated it first — the lock made that safe.
+      if (!alreadyExists(error)) throw error;
+    } finally {
+      await admin.exec(unlockSql(template));
+    }
+    await admin.exec(dropSql(database));
+    await admin.exec(cloneSql(template, database));
+  } catch (error) {
+    await admin.close();
+    throw new TestDatabaseUnavailableError({
+      cause: `could not clone ${template} for worker ${worker}: ${messageOf(error)}`,
+    });
+  }
+  await admin.close();
+
+  return {
+    kind: 'postgres',
+    worker,
+    database,
+    url: urlFor(adminUrl, database),
+    drop: async () => {
+      const cleanup = connect(adminUrl);
+      await cleanup.exec(dropSql(database));
+      await cleanup.close();
+    },
+  };
+}
+
+const messageOf = (error: unknown): string =>
+  error instanceof Error ? error.message : String(error);
+
+const alreadyExists = (error: unknown): boolean =>
+  messageOf(error).toLowerCase().includes('already exists');

package/src/test-types.ts

@@ -0,0 +1,145 @@
+// The six first-class test types. `x verify` selects a suite by FILENAME — `*.job.test.ts`, and
+// so on — so these helpers only prefix the reported name with its type, which is what makes a
+// failure line say which of the six steps it belongs to. See packages/cli/src/verify-tests.ts.
+
+import { test } from 'bun:test';
+import { TestEvalThresholdError } from './errors';
+
+export const TEST_TYPES = ['unit', 'contract', 'live', 'job', 'e2e', 'eval'] as const;
+
+export type TestType = (typeof TEST_TYPES)[number];
+
+export const SEPARATOR = ' · ';
+
+export const testName = (type: TestType, name: string): string => `${type}${SEPARATOR}${name}`;
+
+export type TestBody = () => void | Promise<void>;
+
+/** Pure logic: no database, no clock, no network. The cheapest test that can fail for real. */
+export const unitTest = (name: string, body: TestBody): void => {
+  test(testName('unit', name), body);
+};
+
+/** The published surface: OpenAPI diff against the committed spec, MCP exposure, error codes. */
+export const contractTest = (name: string, body: TestBody): void => {
+  test(testName('contract', name), body);
+};
+
+/** Live queries: assert exactly what each subscriber receives, snapshot then incremental patch. */
+export const liveTest = (name: string, body: TestBody): void => {
+  test(testName('live', name), body);
+};
+
+/** Jobs: step sequence, retries, idempotency. Never wall-clock sleeps — advance the frozen clock. */
+export const jobTest = (name: string, body: TestBody): void => {
+  test(testName('job', name), body);
+};
+
+/** One element selection, resolved when it is used rather than when it is built. */
+export interface LocatorLike {
+  count(): Promise<number>;
+  click(): Promise<void>;
+  first(): LocatorLike;
+  isVisible(): Promise<boolean>;
+}
+
+/**
+ * The browser surface an e2e test drives. Every member is one the reference app's e2e suite
+ * already calls — this is the observed contract, not a wish list, and the driver that implements
+ * it (a browser, at milestone 11) is the only thing that may add to it.
+ */
+export interface PageLike {
+  goto(url: string): Promise<unknown>;
+  /** The first flush of a streamed response — the shell, before the holes resolve. */
+  gotoStreamed(url: string): Promise<{ readonly html: string }>;
+  reload(): Promise<unknown>;
+  /** Resolves once the service worker controls the page, so the offline assertions are not racy. */
+  waitForServiceWorker(): Promise<void>;
+  title(): Promise<string>;
+  content(): Promise<string>;
+  url(): string;
+  evaluate<T>(fn: () => T): Promise<T>;
+  locator(selector: string): LocatorLike;
+  getByRole(
+    role: string,
+    options?: { readonly name?: string; readonly level?: number },
+  ): LocatorLike;
+  getByText(text: string): LocatorLike;
+}
+
+export interface E2eFixtures {
+  readonly page: PageLike;
+  /** Drop the network so the service worker has to answer. */
+  offline(): Promise<void>;
+  online(): Promise<void>;
+  /** Publish a new build id so the SW update path runs. */
+  update(): Promise<void>;
+}
+
+export type E2eBody = (fixtures: E2eFixtures) => Promise<void>;
+
+let e2eDriver: ((name: string, body: E2eBody) => void) | undefined;
+
+/**
+ * Register the Playwright-backed driver. The e2e package wires this up; without it e2e tests are
+ * skipped loudly rather than failing on a missing browser, and `x verify` reports the step as
+ * skipped rather than green.
+ */
+export function useE2eDriver(driver: (name: string, body: E2eBody) => void): void {
+  e2eDriver = driver;
+}
+
+export const e2eTest = (name: string, body: E2eBody): void => {
+  if (e2eDriver === undefined) {
+    test.skip(
+      testName('e2e', `${name} (no browser driver: x build --target static && x e2e)`),
+      () => undefined,
+    );
+    return;
+  }
+  e2eDriver(testName('e2e', name), body);
+};
+
+export interface EvalCase<TInput, TOutput> {
+  readonly name: string;
+  readonly input: TInput;
+  /** 0..1. Below the threshold fails the test. */
+  score(output: TOutput): number | Promise<number>;
+}
+
+export interface EvalOptions<TInput, TOutput> {
+  readonly threshold: number;
+  readonly cases: readonly EvalCase<TInput, TOutput>[];
+  run(input: TInput): Promise<TOutput>;
+}
+
+/**
+ * LLM output scoring with a threshold. Deterministic by construction: the model call goes through
+ * the sealed network, so an eval either uses a recorded response or declares its allowlist.
+ */
+export function evalTest<TInput, TOutput>(
+  name: string,
+  options: EvalOptions<TInput, TOutput>,
+): void {
+  test(testName('eval', name), async () => {
+    const scores: { readonly name: string; readonly score: number }[] = [];
+    for (const testCase of options.cases) {
+      const output = await options.run(testCase.input);
+      scores.push({ name: testCase.name, score: await testCase.score(output) });
+    }
+    const failures = scores.filter((entry) => entry.score < options.threshold);
+    if (failures.length > 0) {
+      const detail = failures.map((entry) => `${entry.name}=${entry.score.toFixed(2)}`).join(', ');
+      throw new TestEvalThresholdError({ name, threshold: options.threshold, detail });
+    }
+  });
+}
+
+export interface OpenApiOperationLike {
+  readonly operationId: string;
+  readonly required?: readonly string[];
+}
+
+export interface OpenApiLike {
+  readonly operations: readonly OpenApiOperationLike[];
+}