@ultimat3/testing 29.3.2 → 30.0.0

package/src/drill-fleet.ts

@@ -0,0 +1,251 @@
+// A drill's fleet: the app's `web` and `worker` roles as separate processes (`apps/web/server.ts`
+// under `ROLE`), on a scratch database it creates and drops, each dialling Postgres, Redis and
+// NATS through a fault proxy of its own. The proxies are what the drill switches; the roles are
+// what it signals. Nothing here judges — `drill-run.ts` does.
+
+// why: Bun ships no temp-directory primitive or recursive remove; `tmpdir()` is node:os's alone.
+import { mkdtemp, rm } from 'node:fs/promises';
+// why: Bun exposes no tmpdir() — only node:os answers the platform temp root.
+import { tmpdir } from 'node:os';
+// why: Bun exposes no path API — the state dir is joined, not concatenated.
+import { join } from 'node:path';
+import { probeStatus } from './drill-probe';
+import type { DrillRole, DrillService } from './drills';
+import { E2eAppFailedError } from './e2e-errors';
+import type { FaultProxy } from './fault-proxy';
+import { faultProxy, faultTarget, throughFaultProxy } from './fault-proxy';
+import { freePortApartFrom } from './free-port';
+import { dropSql, urlFor } from './template-db';
+import { testSealKeyEnv } from './test-seal-key';
+
+export interface DrillServiceUrls {
+  /** A Postgres URL with rights to CREATE DATABASE: the drill's scratch database is made there. */
+  readonly postgres: string;
+  readonly redis?: string | undefined;
+  /** NATS, and the variable the app's `realtime.urlEnv` reads it from. */
+  readonly nats?: { readonly url: string; readonly env: string } | undefined;
+}
+
+export interface DrillFleetOptions {
+  readonly root: string;
+  readonly services: DrillServiceUrls;
+  readonly roles: readonly DrillRole[];
+  readonly readyTimeoutMs: number;
+  /** Extra environment for every role. */
+  readonly env?: Readonly<Record<string, string>> | undefined;
+}
+
+export interface DrillRoleProcess {
+  readonly role: DrillRole;
+  /** `http://127.0.0.1:<PORT>`: the port the role was handed (`sync` binds its socket on PORT + 1). */
+  readonly origin: string;
+  /** `/readyz` on this role's own listener: the web port for `web`, the scrape port otherwise. */
+  readonly readyz: string;
+  signal(signal: 'SIGKILL' | 'SIGTERM'): void;
+  /** The exit code, once the current process ends. */
+  exited(): Promise<number>;
+  /** A fresh process for the role on the same ports — what a supervisor does after a kill. */
+  respawn(): Promise<void>;
+  log(): string;
+}
+
+export interface DrillFleet {
+  readonly base: string;
+  /** The scratch database, dialled directly: the evidence's line, never the faulted one. */
+  readonly databaseUrl: string;
+  readonly proxies: ReadonlyMap<DrillService, FaultProxy>;
+  readonly roles: ReadonlyMap<DrillRole, DrillRoleProcess>;
+  stop(): Promise<void>;
+}
+
+const SERVER = 'apps/web/server.ts';
+const TAIL_CHARS = 8_000;
+const KILL_GRACE_MS = 5_000;
+
+/** This process's environment minus every binding the fleet sets itself, and minus `NODE_ENV`. */
+const inheritedEnv = (): Record<string, string | undefined> => {
+  const {
+    NODE_ENV: _node,
+    DATABASE_URL: _db,
+    REDIS_URL: _redis,
+    NATS_URL: _nats,
+    ...rest
+  } = Bun.env;
+  return rest;
+};
+
+function tailOf(child: Bun.Subprocess<'ignore', 'pipe', 'pipe'>): () => string {
+  let text = '';
+  for (const stream of [child.stdout, child.stderr]) {
+    void (async () => {
+      const decoder = new TextDecoder();
+      for await (const chunk of stream) {
+        text = (text + decoder.decode(chunk, { stream: true })).slice(-TAIL_CHARS);
+      }
+    })().catch(() => undefined);
+  }
+  return () => text;
+}
+
+const refuse = (step: string, output: string): E2eAppFailedError =>
+  new E2eAppFailedError({
+    step,
+    output,
+    fix: 'ROLE=web bun apps/web/server.ts   # in the app root, with the same DATABASE_URL: read why the role will not start',
+  });
+
+/** Create the scratch database on the admin URL; answers its own URL and its drop. */
+async function scratchDatabase(adminUrl: string) {
+  const name = `x_drill_${crypto.randomUUID().replaceAll('-', '').slice(0, 16)}`;
+  const admin = new Bun.SQL(adminUrl);
+  try {
+    await admin.unsafe(`create database ${name}`);
+  } finally {
+    await admin.close();
+  }
+  return {
+    url: urlFor(adminUrl, name),
+    async drop(): Promise<void> {
+      const again = new Bun.SQL(adminUrl);
+      try {
+        await again.unsafe(dropSql(name));
+      } finally {
+        await again.close();
+      }
+    },
+  };
+}
+
+async function waitReady(url: string, deadlineMs: number, alive: () => boolean): Promise<boolean> {
+  const until = performance.now() + deadlineMs;
+  while (performance.now() < until && alive()) {
+    if ((await probeStatus(url, 1_000)) === 200) return true;
+    await Bun.sleep(250);
+  }
+  return false;
+}
+
+/** The fleet, ready: database created and migrated, every proxy listening, every role answering. */
+export async function startDrillFleet(options: DrillFleetOptions): Promise<DrillFleet> {
+  const database = await scratchDatabase(options.services.postgres);
+  const stateDir = await mkdtemp(join(tmpdir(), 'ultimate-drill-'));
+  const proxies = new Map<DrillService, FaultProxy>();
+  const roles = new Map<DrillRole, DrillRoleProcess>();
+  const children = new Set<Bun.Subprocess>();
+  const stop = async (): Promise<void> => {
+    for (const child of children) child.kill('SIGKILL');
+    await Promise.all([...children].map((child) => child.exited));
+    for (const proxy of proxies.values()) proxy.close();
+    await rm(stateDir, { recursive: true, force: true });
+    await database.drop();
+  };
+  try {
+    proxies.set('postgres', await faultProxy({ target: faultTarget(database.url) }));
+    const redis = options.services.redis;
+    if (redis !== undefined) proxies.set('redis', await faultProxy({ target: faultTarget(redis) }));
+    const nats = options.services.nats;
+    if (nats !== undefined)
+      proxies.set('nats', await faultProxy({ target: faultTarget(nats.url) }));
+    const via = (service: DrillService, url: string): string =>
+      throughFaultProxy(url, proxies.get(service)?.port ?? 0);
+    const base: Record<string, string> = {
+      ...testSealKeyEnv({ root: options.root, env: { ...Bun.env, NODE_ENV: 'test' } }),
+      // `test`, never `development`: a local environment (the throwaway seal key, no deploy-secret
+      // refusal), without the development-only seams an app installs — a demo viewer answering
+      // every request as ONE actor puts the whole load in one rate-limit bucket.
+      ULTIMATE_ENV: 'test',
+      // The load names a client per request in X-Forwarded-For, as an ingress does: without one
+      // trusted hop every request is ONE client, and the rate limiter's 429 drowns the drill.
+      TRUSTED_PROXY_HOPS: '1',
+      ULTIMATE_STATE_DIR: stateDir,
+      DATABASE_URL: via('postgres', database.url),
+      ...(redis === undefined ? {} : { REDIS_URL: via('redis', redis) }),
+      ...(nats === undefined ? {} : { [nats.env]: via('nats', nats.url) }),
+      ...options.env,
+    };
+    const spawnRole = (env: Record<string, string>) => {
+      const child = Bun.spawn(['bun', SERVER], {
+        cwd: options.root,
+        env: { ...inheritedEnv(), ...base, ...env },
+        stdin: 'ignore',
+        stdout: 'pipe',
+        stderr: 'pipe',
+      });
+      children.add(child);
+      return child;
+    };
+    // The release phase on the DIRECT url: migrations are not what any drill faults.
+    const migrate = spawnRole({ ROLE: 'migrate', DATABASE_URL: database.url });
+    const migrateLog = tailOf(migrate);
+    // Bounded like a role's readiness: a wedged migration is a refusal, and the `catch` below
+    // SIGKILLs it — it is still in `children`.
+    const migrated = await Promise.race([
+      migrate.exited,
+      Bun.sleep(options.readyTimeoutMs).then(() => null),
+    ]);
+    if (migrated !== 0) throw refuse('ROLE=migrate', migrateLog());
+    children.delete(migrate);
+
+    const taken: number[] = [...[...proxies.values()].map((proxy) => proxy.port)];
+    const take = (): number => {
+      const port = freePortApartFrom(taken);
+      taken.push(port);
+      return port;
+    };
+    let webBase = '';
+    for (const role of options.roles) {
+      const port = take();
+      // The sync node listens on PORT + 1 (`syncPortFor`): held too, so no other role is handed it.
+      if (role === 'sync') taken.push(port + 1);
+      const metricsPort = take();
+      const origin = `http://127.0.0.1:${String(port)}`;
+      if (role === 'web') webBase = origin;
+      const env = {
+        ROLE: role,
+        PORT: String(port),
+        METRICS_PORT: String(metricsPort),
+        APP_URL: origin,
+      };
+      const readyz =
+        role === 'web' ? `${origin}/readyz` : `http://127.0.0.1:${String(metricsPort)}/readyz`;
+      let child = spawnRole(env);
+      let log = tailOf(child);
+      const boot = async (): Promise<void> => {
+        const ready = await waitReady(
+          readyz,
+          options.readyTimeoutMs,
+          () => child.exitCode === null,
+        );
+        if (!ready) throw refuse(`ROLE=${role} never answered ${readyz}`, log());
+      };
+      await boot();
+      roles.set(role, {
+        role,
+        origin,
+        readyz,
+        signal: (signal) => child.kill(signal),
+        exited: () => child.exited,
+        async respawn() {
+          children.delete(child);
+          child = spawnRole(env);
+          log = tailOf(child);
+          await boot();
+        },
+        log: () => log(),
+      });
+    }
+    return { base: webBase, databaseUrl: database.url, proxies, roles, stop };
+  } catch (error) {
+    await stop();
+    throw error;
+  }
+}
+
+/** A role's exit, or `null` past `withinMs`: SIGKILLed then, so a wedged drain ends the drill. */
+export async function exitWithin(role: DrillRoleProcess, withinMs: number): Promise<number | null> {
+  const exited = await Promise.race([role.exited(), Bun.sleep(withinMs).then(() => null)]);
+  if (exited !== null) return exited;
+  role.signal('SIGKILL');
+  await Promise.race([role.exited(), Bun.sleep(KILL_GRACE_MS)]);
+  return null;
+}

package/src/drill-jobs.ts

@@ -0,0 +1,79 @@
+// What the jobs tables say after a drill: how many jobs the drill window created, how many are not
+// terminal once the fleet had its chance to reclaim them, and how many idempotency keys completed
+// twice. Read straight off `x_jobs` on a direct connection — never through the faulted proxy.
+
+import { finiteCount } from '@ultimat3/core';
+
+/** The one query surface this file needs; `Bun.SQL` satisfies it, a test hands in rows. */
+export interface DrillSql {
+  rows(text: string, params: readonly unknown[]): Promise<readonly Record<string, unknown>[]>;
+}
+
+export interface DrillJobCounts {
+  readonly ran: number;
+  readonly stuck: number;
+  readonly doubled: number;
+}
+
+/** Not terminal and due: claimable and unclaimed, or running past nothing reclaiming it. */
+export const STUCK_SQL = `select count(*)::int as n from x_jobs
+  where created_at >= $1 and state in ('ready', 'delayed', 'running') and run_at <= now()`;
+
+export const RAN_SQL = 'select count(*)::int as n from x_jobs where created_at >= $1';
+
+/**
+ * A key that ran twice AT ONCE: a second done row created while the first was still live (before
+ * its `updated_at`, the instant it settled). The live-row dedupe only holds while the first is
+ * live, so the same key enqueued again after it finished is a new job by design, never a double.
+ */
+export const DOUBLED_SQL = `select count(distinct (a.name, coalesce(a.tenant_id, ''), a.idempotency_key))::int as n
+  from x_jobs a join x_jobs b
+    on b.name = a.name and coalesce(b.tenant_id, '') = coalesce(a.tenant_id, '')
+   and b.idempotency_key = a.idempotency_key and b.id <> a.id
+  where a.created_at >= $1 and b.created_at >= $1 and a.state = 'done' and b.state = 'done'
+    and b.created_at >= a.created_at and b.created_at < a.updated_at`;
+
+const count = async (sql: DrillSql, text: string, since: Date): Promise<number> => {
+  const [row] = await sql.rows(text, [since]);
+  const value = row?.['n'];
+  return typeof value === 'number' ? value : Number(value ?? 0);
+};
+
+export interface DrillJobsOptions {
+  readonly sql: DrillSql;
+  readonly since: Date;
+  /** How long a killed worker's run may take to be reclaimed and finished. */
+  readonly settleMs: number;
+  readonly pollMs?: number | undefined;
+  readonly now?: (() => number) | undefined;
+  readonly sleep?: ((ms: number) => Promise<void>) | undefined;
+}
+
+/** Counted once nothing is stuck, or once `settleMs` has passed — whichever is first. */
+export async function drillJobCounts(options: DrillJobsOptions): Promise<DrillJobCounts> {
+  const settleMs = finiteCount('drillJobCounts', 'settleMs', options.settleMs);
+  const pollMs = finiteCount('drillJobCounts', 'pollMs', options.pollMs ?? 250, 1);
+  const now = options.now ?? (() => performance.now());
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const until = now() + settleMs;
+  let stuck = await count(options.sql, STUCK_SQL, options.since);
+  while (stuck > 0 && now() < until) {
+    await sleep(pollMs);
+    stuck = await count(options.sql, STUCK_SQL, options.since);
+  }
+  return {
+    ran: await count(options.sql, RAN_SQL, options.since),
+    stuck,
+    doubled: await count(options.sql, DOUBLED_SQL, options.since),
+  };
+}
+
+/** `Bun.SQL` behind `DrillSql`: the drill's direct line to the scratch database. */
+export function bunDrillSql(url: string): DrillSql & { close(): Promise<void> } {
+  const client = new Bun.SQL(url);
+  return {
+    rows: async (text, params) =>
+      (await client.unsafe(text, [...params])) as readonly Record<string, unknown>[],
+    close: () => client.close(),
+  };
+}

package/src/drill-load.ts

@@ -0,0 +1,166 @@
+// A drill's load: lanes of requests against the web role, every answer recorded with its timing
+// and its `X_*` code, then folded into the evidence the verdict reads. Each POST carries its own
+// idempotency key, so a retried write is the framework's to dedupe and a doubled one is a finding.
+
+import { finiteCount } from '@ultimat3/core';
+import type { DrillRequestEvidence } from './drill-verdict';
+
+export interface DrillTarget {
+  readonly method: 'GET' | 'POST';
+  /** Path from the web role's origin, query included. */
+  readonly path: string;
+  /** A POST's JSON body. */
+  readonly body?: unknown;
+  /**
+   * A POST's body per request, given the load's request number (from 1); wins over `body`. A write
+   * that enqueues a job needs a distinct input each time, or its key dedupes every request into one.
+   */
+  readonly draw?: ((run: number) => unknown) | undefined;
+}
+
+/** One request, as the load saw it. Times are ms on the load's own monotonic clock. */
+export interface DrillSample {
+  readonly target: string;
+  readonly startedMs: number;
+  readonly endedMs: number;
+  /** `null`: nothing answered — refused, reset, or past the harness's own bound. */
+  readonly status: number | null;
+  /** The problem document's `code`, when the answer carried one. */
+  readonly code: string | undefined;
+}
+
+export interface DrillLoadOptions {
+  readonly base: string;
+  readonly targets: readonly DrillTarget[];
+  readonly lanes?: number | undefined;
+  /** The harness's own bound on one request: past it the request is `unanswered`. */
+  readonly requestBoundMs: number;
+  /** Between two requests of one lane, so a lane measures rather than floods. */
+  readonly pauseMs?: number | undefined;
+  readonly fetch?: typeof globalThis.fetch | undefined;
+  readonly now?: (() => number) | undefined;
+  readonly sleep?: ((ms: number) => Promise<void>) | undefined;
+}
+
+export interface DrillLoad {
+  /** Every sample so far, in completion order. */
+  samples(): readonly DrillSample[];
+  /** Stop every lane and wait for its in-flight request to settle. */
+  stop(): Promise<readonly DrillSample[]>;
+}
+
+const CODE = /^X_[A-Z0-9_]+$/;
+
+/** The `code` of a problem document, or `undefined` for any other body. */
+export function problemCode(contentType: string | null, body: string): string | undefined {
+  if (contentType === null || !contentType.includes('json')) return undefined;
+  try {
+    const parsed: unknown = JSON.parse(body);
+    const code =
+      typeof parsed === 'object' && parsed !== null ? Reflect.get(parsed, 'code') : undefined;
+    return typeof code === 'string' && CODE.test(code) ? code : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+const bodyOf = (target: DrillTarget, run: number): unknown =>
+  target.draw === undefined ? (target.body ?? {}) : target.draw(run);
+
+/** Lanes over `targets`, round-robin, until `stop()`. */
+export function startDrillLoad(options: DrillLoadOptions): DrillLoad {
+  const lanes = finiteCount('startDrillLoad', 'lanes', options.lanes ?? 4, 1);
+  const pauseMs = finiteCount('startDrillLoad', 'pauseMs', options.pauseMs ?? 50);
+  const boundMs = finiteCount('startDrillLoad', 'requestBoundMs', options.requestBoundMs, 1);
+  const send = options.fetch ?? globalThis.fetch;
+  const now = options.now ?? (() => performance.now());
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const samples: DrillSample[] = [];
+  const stopping = new AbortController();
+  let sequence = 0;
+
+  const once = async (target: DrillTarget): Promise<void> => {
+    sequence += 1;
+    const run = sequence;
+    const label = `${target.method} ${target.path}`;
+    const headers: Record<string, string> = {
+      'x-forwarded-for': `10.${String((sequence >> 16) & 255)}.${String((sequence >> 8) & 255)}.${String(sequence & 255)}`,
+    };
+    if (target.method === 'POST') {
+      headers['content-type'] = 'application/json';
+      headers['idempotency-key'] = `drill-${String(sequence)}`;
+    }
+    const startedMs = now();
+    try {
+      const response = await send(`${options.base}${target.path}`, {
+        method: target.method,
+        headers,
+        ...(target.method === 'POST' ? { body: JSON.stringify(bodyOf(target, run)) } : {}),
+        signal: AbortSignal.timeout(boundMs),
+      });
+      const body = await response.text();
+      samples.push({
+        target: label,
+        startedMs,
+        endedMs: now(),
+        status: response.status,
+        code: problemCode(response.headers.get('content-type'), body),
+      });
+    } catch {
+      samples.push({ target: label, startedMs, endedMs: now(), status: null, code: undefined });
+    }
+  };
+
+  const lane = async (index: number): Promise<void> => {
+    for (let turn = index; !stopping.signal.aborted; turn += 1) {
+      const target = options.targets[turn % options.targets.length];
+      if (target === undefined) return;
+      await once(target);
+      if (!stopping.signal.aborted) await sleep(pauseMs);
+    }
+  };
+
+  const running = Array.from({ length: lanes }, (_, index) => lane(index));
+  return {
+    samples: () => samples,
+    async stop() {
+      stopping.abort();
+      await Promise.all(running);
+      return samples;
+    },
+  };
+}
+
+/** `X_INTERNAL` is the code of a throw nothing classified: a failure, but not a coded one. */
+const UNCLASSIFIED = 'X_INTERNAL';
+
+/** The samples as the verdict reads them. */
+export function requestEvidence(samples: readonly DrillSample[]): DrillRequestEvidence {
+  const failedByCode = new Map<string, number>();
+  let ok = 0;
+  let uncoded = 0;
+  let unanswered = 0;
+  let maxMs = 0;
+  for (const sample of samples) {
+    if (sample.status === null) {
+      unanswered += 1;
+      continue;
+    }
+    maxMs = Math.max(maxMs, Math.round(sample.endedMs - sample.startedMs));
+    if (sample.status >= 200 && sample.status < 300) ok += 1;
+    else if (sample.code === undefined || sample.code === UNCLASSIFIED) uncoded += 1;
+    else failedByCode.set(sample.code, (failedByCode.get(sample.code) ?? 0) + 1);
+  }
+  return { ok, failedByCode, uncoded, unanswered, maxMs };
+}
+
+/** ms from `clearedMs` to the end of the first 2xx that STARTED after it; `null` for none. */
+export function recoveryAfter(samples: readonly DrillSample[], clearedMs: number): number | null {
+  let first: number | null = null;
+  for (const sample of samples) {
+    const answered = sample.status !== null && sample.status >= 200 && sample.status < 300;
+    if (!answered || sample.startedMs < clearedMs) continue;
+    if (first === null || sample.endedMs < first) first = sample.endedMs;
+  }
+  return first === null ? null : Math.round(first - clearedMs);
+}

package/src/drill-probe.ts

@@ -0,0 +1,68 @@
+// One status probe and a sampler over it: how a drill asks a role's `/readyz` what it says while a
+// dependency fails. Through `node:http`, never `fetch`, so the probe answers the same inside
+// `bun test` — where the preload seals `fetch` — as under `x drill`.
+
+// why: `globalThis.fetch` is sealed inside `bun test`; `node:http` is not, and loopback is all it reaches.
+import { get } from 'node:http';
+import { finiteCount } from '@ultimat3/core';
+
+/** One GET: the status, or `null` when nothing answered within `boundMs`. Never a throw. */
+export function probeStatus(url: string, boundMs: number): Promise<number | null> {
+  const bound = finiteCount('probeStatus', 'boundMs', boundMs, 1);
+  return new Promise((resolve) => {
+    const request = get(url, (response) => {
+      response.resume();
+      resolve(response.statusCode ?? null);
+    });
+    request.on('error', () => resolve(null));
+    request.setTimeout(bound, () => {
+      request.destroy();
+      resolve(null);
+    });
+  });
+}
+
+export interface StatusSample {
+  /** ms since the sampler's `zero`. */
+  readonly atMs: number;
+  readonly status: number | null;
+}
+
+export interface StatusSampler {
+  stop(): Promise<readonly StatusSample[]>;
+}
+
+export interface StatusSamplerOptions {
+  readonly url: string;
+  readonly everyMs: number;
+  readonly boundMs: number;
+  /** The instant `atMs` counts from — the fault's start. */
+  readonly zero: number;
+  readonly now?: (() => number) | undefined;
+  readonly sleep?: ((ms: number) => Promise<void>) | undefined;
+  readonly probe?: ((url: string, boundMs: number) => Promise<number | null>) | undefined;
+}
+
+/** Probe `url` every `everyMs` until `stop()`; one probe at a time, so a hung one is one sample. */
+export function sampleStatus(options: StatusSamplerOptions): StatusSampler {
+  const everyMs = finiteCount('sampleStatus', 'everyMs', options.everyMs, 1);
+  const now = options.now ?? (() => performance.now());
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const probe = options.probe ?? probeStatus;
+  const samples: StatusSample[] = [];
+  let stopped = false;
+  const loop = (async () => {
+    while (!stopped) {
+      const atMs = Math.round(now() - options.zero);
+      samples.push({ atMs, status: await probe(options.url, options.boundMs) });
+      if (!stopped) await sleep(everyMs);
+    }
+  })();
+  return {
+    async stop() {
+      stopped = true;
+      await loop;
+      return samples;
+    },
+  };
+}