@ultimat3/testing 29.4.0 → 30.0.0

package/src/drill-run.ts

@@ -0,0 +1,144 @@
+// One drill, played: load on the web role, a `/readyz` sampler, the fault — proxy phases or a
+// signal — then recovery, then the evidence folded and judged. The fleet is a parameter, so the
+// sequence is proven against an in-process stand-in and `x drill` hands in the real one.
+
+import { finiteCount } from '@ultimat3/core';
+import type { DrillFleet } from './drill-fleet';
+import { exitWithin } from './drill-fleet';
+import type { DrillJobCounts, DrillSql } from './drill-jobs';
+import { drillJobCounts } from './drill-jobs';
+import type { DrillSample, DrillTarget } from './drill-load';
+import { recoveryAfter, requestEvidence, startDrillLoad } from './drill-load';
+import { sampleStatus } from './drill-probe';
+import type { DrillSyncLane } from './drill-sync';
+import type { DrillBudgets, DrillEvidence, DrillVerdict } from './drill-verdict';
+import { judgeDrill } from './drill-verdict';
+import type { Drill } from './drills';
+
+export interface DrillRunOptions {
+  readonly drill: Drill;
+  readonly fleet: DrillFleet;
+  readonly budgets: DrillBudgets & { readonly phaseMs: number };
+  readonly targets: readonly DrillTarget[];
+  /** The direct line to the scratch database, for the jobs evidence. */
+  readonly sql: DrillSql;
+  /** Load before the fault, so a drill that never had a 2xx says so rather than blaming the fault. */
+  readonly baselineMs?: number | undefined;
+  readonly sampleEveryMs?: number | undefined;
+  readonly lanes?: number | undefined;
+  /**
+   * The sync-client lane, started with the fault and stopped after recovery — what `sync-resumes`
+   * reads. Absent, that property is red on no evidence, never green on none.
+   */
+  readonly sync?: (() => DrillSyncLane) | undefined;
+  readonly now?: (() => number) | undefined;
+  readonly sleep?: ((ms: number) => Promise<void>) | undefined;
+}
+
+export interface DrillReport {
+  readonly name: string;
+  readonly verdict: DrillVerdict;
+  readonly evidence: DrillEvidence;
+  readonly budgets: DrillBudgets;
+  /** Answers before the fault began: a drill whose baseline failed is reporting the app, not it. */
+  readonly baseline: { readonly ok: number; readonly failed: number };
+}
+
+interface Played {
+  readonly clearedMs: number | undefined;
+  readonly exit: DrillEvidence['exit'];
+}
+
+async function play(
+  options: DrillRunOptions,
+  now: () => number,
+  sleep: (ms: number) => Promise<void>,
+): Promise<Played> {
+  const { drill, fleet, budgets } = options;
+  const fault = drill.fault;
+  if (fault.kind === 'service') {
+    const proxy = fleet.proxies.get(fault.service);
+    if (proxy === undefined) return { clearedMs: undefined, exit: undefined };
+    for (const mode of fault.phases(budgets)) {
+      await proxy.set(mode);
+      await sleep(budgets.phaseMs);
+    }
+    await proxy.set({ kind: 'pass' });
+    return { clearedMs: now(), exit: undefined };
+  }
+  const role = fleet.roles.get(fault.role);
+  if (role === undefined) return { clearedMs: undefined, exit: undefined };
+  const signalled = now();
+  role.signal(fault.signal);
+  const code = await exitWithin(role, budgets.drainMs);
+  const exit = { role: fault.role, afterMs: code === null ? null : Math.round(now() - signalled) };
+  // A worker comes back, as its supervisor would bring it: the run it held is reclaimed by it.
+  if (fault.role === 'web') return { clearedMs: undefined, exit };
+  await role.respawn();
+  return { clearedMs: now(), exit };
+}
+
+const failedOf = (samples: readonly DrillSample[]): number =>
+  samples.filter((sample) => sample.status === null || sample.status >= 300).length;
+
+/** Play `drill` on `fleet` and judge it. Never throws for a property that did not hold. */
+export async function runDrill(options: DrillRunOptions): Promise<DrillReport> {
+  const now = options.now ?? (() => performance.now());
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const { budgets, fleet, drill } = options;
+  const [started] = await options.sql.rows('select now() as t', []);
+  const since = started?.['t'];
+  const load = startDrillLoad({
+    base: fleet.base,
+    targets: options.targets,
+    lanes: options.lanes,
+    requestBoundMs: budgets.deadlineMs * 2,
+    now,
+    sleep,
+  });
+  await sleep(finiteCount('runDrill', 'baselineMs', options.baselineMs ?? budgets.promptMs * 2));
+  const baselineSamples = [...load.samples()];
+  const zero = now();
+  const web = fleet.roles.get('web');
+  const sampler =
+    web === undefined
+      ? undefined
+      : sampleStatus({
+          url: web.readyz,
+          everyMs: finiteCount('runDrill', 'sampleEveryMs', options.sampleEveryMs ?? 250, 1),
+          boundMs: budgets.promptMs,
+          zero,
+          now,
+          sleep,
+        });
+  const syncLane = options.sync?.();
+  const played = await play(options, now, sleep);
+  // Recovery is given its whole budget to show, and the load keeps measuring through it.
+  if (played.clearedMs !== undefined) await sleep(budgets.recoveryMs);
+  const samples = await load.stop();
+  const readyz = await sampler?.stop();
+  const sync = await syncLane?.stop();
+  const jobs: DrillJobCounts | undefined =
+    since instanceof Date
+      ? await drillJobCounts({ sql: options.sql, since, settleMs: budgets.drainMs, now, sleep })
+      : undefined;
+  const evidence: DrillEvidence = {
+    requests: requestEvidence(samples),
+    readyz,
+    recoveryMs:
+      played.clearedMs === undefined ? undefined : recoveryAfter(samples, played.clearedMs),
+    jobs,
+    exit: played.exit,
+    sync,
+  };
+  return {
+    name: drill.name,
+    verdict: judgeDrill(drill.holds, evidence, budgets),
+    evidence,
+    budgets,
+    baseline: {
+      ok: baselineSamples.length - failedOf(baselineSamples),
+      failed: failedOf(baselineSamples),
+    },
+  };
+}

package/src/drill-services.ts

@@ -0,0 +1,70 @@
+// Which real services a drill runs against, read from the `TEST_*` URLs `docker/test-services.env`
+// writes and from what the app declares. A service the app uses must be there for EVERY drill — the
+// role would refuse to boot without it — and the faulted one must be one the app actually uses.
+
+import { DrillServiceMissingError } from './drill-errors';
+import type { DrillServiceUrls } from './drill-fleet';
+import type { Drill } from './drills';
+import { drillService } from './drills';
+
+/** The variables the test services are published under (`docker/test-services.env`). */
+export const DRILL_SERVICE_ENV = {
+  postgres: 'TEST_DATABASE_URL',
+  redis: 'TEST_REDIS_URL',
+  nats: 'TEST_NATS_URL',
+} as const;
+
+/** What the app's config says it uses, beyond the Postgres every drill needs. */
+export interface DrillDeclarations {
+  /** `cache.tiers` names `redis`. */
+  readonly redis: boolean;
+  /** `realtime.transport` is `nats`, read from the variable `realtime.urlEnv` names. */
+  readonly nats: { readonly env: string } | undefined;
+}
+
+type Env = Readonly<Record<string, string | undefined>>;
+
+const read = (env: Env, name: string, drill: Drill): string => {
+  const value = env[name]?.trim();
+  if (value === undefined || value === '') {
+    throw new DrillServiceMissingError({
+      drill: drill.name,
+      gap: { kind: 'unset', variable: name },
+    });
+  }
+  return value;
+};
+
+/** The URLs `drill` runs on, or `X_DRILL_SERVICE_MISSING` naming the one gap. */
+export function drillServices(
+  drill: Drill,
+  env: Env,
+  declared: DrillDeclarations,
+): DrillServiceUrls {
+  const faulted = drillService(drill);
+  if (faulted === 'redis' && !declared.redis) {
+    throw new DrillServiceMissingError({
+      drill: drill.name,
+      gap: {
+        kind: 'undeclared',
+        declaration: "cache: { tiers: ['request-memo', 'lru', 'redis'] }",
+      },
+    });
+  }
+  if (faulted === 'nats' && declared.nats === undefined) {
+    throw new DrillServiceMissingError({
+      drill: drill.name,
+      gap: {
+        kind: 'undeclared',
+        declaration: "realtime: { transport: 'nats', urlEnv: 'NATS_URL' }",
+      },
+    });
+  }
+  const postgres = read(env, DRILL_SERVICE_ENV.postgres, drill);
+  const redis = declared.redis ? read(env, DRILL_SERVICE_ENV.redis, drill) : undefined;
+  const nats =
+    declared.nats === undefined
+      ? undefined
+      : { url: read(env, DRILL_SERVICE_ENV.nats, drill), env: declared.nats.env };
+  return { postgres, redis, nats };
+}

package/src/drill-sync.ts

@@ -0,0 +1,108 @@
+// A drill's sync-client lane: one socket on the sync node, opened with the protocol's `hello`,
+// holding the subscriptions it is handed and re-dialling whenever it drops, as a page's socket
+// does. It counts what `sync-resumes` judges — whether the client is back once the fault cleared,
+// and every `replay-gap` the node reported — and never decides anything itself.
+
+import { finiteCount } from '@ultimat3/core';
+import type { DrillEvidence } from './drill-verdict';
+
+export type DrillSyncEvidence = NonNullable<DrillEvidence['sync']>;
+
+/** The part of a `WebSocket` the lane uses; the real one satisfies it, a test hands in its own. */
+export interface DrillSyncSocket {
+  send(data: string): void;
+  close(): void;
+  addEventListener(type: 'open' | 'close' | 'error', listener: () => void): void;
+  addEventListener(type: 'message', listener: (event: { readonly data: unknown }) => void): void;
+}
+
+export interface DrillSyncOptions {
+  /** `ws://…/_x/sync` on the sync role. */
+  readonly url: string;
+  /** `@ultimat3/realtime`'s `PROTOCOL_VERSION`, handed in: this file loads no realtime module. */
+  readonly protocolVersion: number;
+  /** Each a `subscribe` frame's `target`, re-sent on every `hello`, as a page resubscribes. */
+  readonly subscriptions?: readonly unknown[] | undefined;
+  /** Between a drop and the next dial. */
+  readonly redialMs?: number | undefined;
+  readonly connect?: ((url: string) => DrillSyncSocket) | undefined;
+  readonly sleep?: ((ms: number) => Promise<void>) | undefined;
+}
+
+export interface DrillSyncLane {
+  /** Stop re-dialling, close the socket, and answer what the lane saw. */
+  stop(): Promise<DrillSyncEvidence>;
+}
+
+const typeOf = (data: unknown): string | undefined => {
+  if (typeof data !== 'string') return undefined;
+  try {
+    const frame: unknown = JSON.parse(data);
+    const type =
+      typeof frame === 'object' && frame !== null ? Reflect.get(frame, 'type') : undefined;
+    return typeof type === 'string' ? type : undefined;
+  } catch {
+    return undefined;
+  }
+};
+
+/** Dial `url` and keep a socket on it until `stop()`. */
+export function startDrillSync(options: DrillSyncOptions): DrillSyncLane {
+  const redialMs = finiteCount('startDrillSync', 'redialMs', options.redialMs ?? 250);
+  const connect = options.connect ?? ((url: string): DrillSyncSocket => new WebSocket(url));
+  const sleep = options.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const v = finiteCount('startDrillSync', 'protocolVersion', options.protocolVersion, 1);
+  let stopping = false;
+  let gaps = 0;
+  /** The current socket has had the node's `hello`: the client is connected and subscribed. */
+  let live = false;
+  let socket: DrillSyncSocket | undefined;
+
+  const dial = (): void => {
+    if (stopping) return;
+    live = false;
+    const current = connect(options.url);
+    socket = current;
+    current.addEventListener('open', () => {
+      current.send(
+        JSON.stringify({ type: 'hello', v, buildId: '', sessionId: null, actorId: null }),
+      );
+    });
+    current.addEventListener('message', (event) => {
+      const type = typeOf(event.data);
+      if (type === 'replay-gap') gaps += 1;
+      if (type !== 'hello') return;
+      live = true;
+      for (const [index, target] of (options.subscriptions ?? []).entries()) {
+        current.send(
+          JSON.stringify({
+            type: 'subscribe',
+            v,
+            op: 'add',
+            sid: `drill-${String(index)}`,
+            target,
+          }),
+        );
+      }
+    });
+    let gone = false;
+    const drop = (): void => {
+      if (gone || socket !== current) return;
+      gone = true;
+      live = false;
+      if (!stopping) void sleep(redialMs).then(dial);
+    };
+    current.addEventListener('close', drop);
+    current.addEventListener('error', drop);
+  };
+
+  dial();
+  return {
+    async stop() {
+      stopping = true;
+      const reconnected = live;
+      socket?.close();
+      return { reconnected, gaps };
+    },
+  };
+}

package/src/drill-verdict.ts

@@ -0,0 +1,256 @@
+// A drill's verdict, over evidence only: what the load saw, what `/readyz` answered, what the jobs
+// tables hold, how long recovery took. Pure, so every property is proven red on evidence that
+// violates it without a service running. A property with no evidence is red too: a drill that
+// measured nothing is the vacuous pass this file exists to refuse.
+
+/** Every property a drill may state. The table (`drills.ts`) picks; this file judges. */
+export type DrillProperty =
+  | 'requests-coded'
+  | 'requests-bounded'
+  | 'requests-prompt'
+  | 'requests-ok'
+  | 'recovers'
+  | 'readyz-flips'
+  | 'jobs-terminal'
+  | 'jobs-once'
+  | 'role-exits'
+  | 'sync-resumes';
+
+/** The thresholds, each derived from the app's config (`drillBudgets`), never typed per drill. */
+export interface DrillBudgets {
+  /** The request deadline (`requestTimeoutMs`), or `--budget-ms`. A request past it is red. */
+  readonly deadlineMs: number;
+  /** One bounded delay: a degraded dependency may cost a read this much, and no more. */
+  readonly promptMs: number;
+  /** From the fault cleared to the first answered request, with no restart. */
+  readonly recoveryMs: number;
+  /** How long `/readyz` may take to say 503 once a dependency is gone. */
+  readonly readinessMs: number;
+  /** `drain.deadlineMs` plus the readiness grace: a signalled role is gone inside it. */
+  readonly drainMs: number;
+  /** `health.readiness`: `'process'` keeps `/readyz` 200 through a dependency outage, by design. */
+  readonly readiness: 'dependencies' | 'process';
+  /** `--budget-ms`, when given: printed in the verdict, because it replaced the derived deadline. */
+  readonly override?: number | undefined;
+}
+
+export interface DrillRequestEvidence {
+  readonly ok: number;
+  /** Non-2xx answers carrying a framework `X_*` code, by code. */
+  readonly failedByCode: ReadonlyMap<string, number>;
+  /** Non-2xx answers with no code, or `X_INTERNAL` — a failure the framework did not classify. */
+  readonly uncoded: number;
+  /** Requests nothing answered: the connection refused, reset or the harness's own bound hit. */
+  readonly unanswered: number;
+  /** The slowest answered request, in ms. */
+  readonly maxMs: number;
+}
+
+export interface DrillEvidence {
+  readonly requests: DrillRequestEvidence | undefined;
+  /** Each `/readyz` sample: ms since the fault began, and the status (`null`: nothing answered). */
+  readonly readyz: readonly { readonly atMs: number; readonly status: number | null }[] | undefined;
+  /** From the fault cleared to the first 2xx; `null` when none came. */
+  readonly recoveryMs: number | null | undefined;
+  readonly jobs:
+    | { readonly ran: number; readonly stuck: number; readonly doubled: number }
+    | undefined;
+  /** The signalled role: ms from the signal to its exit, `null` when it never exited. */
+  readonly exit: { readonly role: string; readonly afterMs: number | null } | undefined;
+  readonly sync: { readonly reconnected: boolean; readonly gaps: number } | undefined;
+}
+
+export interface DrillFailure {
+  readonly property: DrillProperty;
+  readonly expected: string;
+  readonly observed: string;
+}
+
+export interface DrillVerdict {
+  readonly ok: boolean;
+  readonly failures: readonly DrillFailure[];
+}
+
+type Judge = (evidence: DrillEvidence, budgets: DrillBudgets) => DrillFailure | undefined;
+
+const NO_EVIDENCE = 'no evidence: the drill did not measure it';
+
+/**
+ * How late past the deadline an answer may reach the load and still be one the deadline gave: the
+ * load times from before `fetch`, so the 504 a deadline fires arrives a beat after it (measured:
+ * 30,002 ms on a 30,000 ms deadline). A request nothing bounded runs to the load's own bound.
+ */
+export const DRILL_ANSWER_SLACK_MS = 500;
+
+const fail = (property: DrillProperty, expected: string, observed: string): DrillFailure => ({
+  property,
+  expected,
+  observed,
+});
+
+const codes = (requests: DrillRequestEvidence): string =>
+  [...requests.failedByCode].map(([code, count]) => `${code}×${String(count)}`).join(', ') ||
+  'none';
+
+const failedCount = (requests: DrillRequestEvidence): number =>
+  requests.uncoded + [...requests.failedByCode.values()].reduce((sum, count) => sum + count, 0);
+
+const JUDGES: ReadonlyMap<DrillProperty, Judge> = new Map<DrillProperty, Judge>([
+  [
+    'requests-coded',
+    ({ requests }) => {
+      const expected = 'every failed request answered with an X_* code';
+      if (requests === undefined) return fail('requests-coded', expected, NO_EVIDENCE);
+      if (requests.ok + requests.unanswered + failedCount(requests) === 0) {
+        return fail('requests-coded', expected, 'no request was sent');
+      }
+      return requests.uncoded === 0
+        ? undefined
+        : fail('requests-coded', expected, `${String(requests.uncoded)} uncoded 5xx or X_INTERNAL`);
+    },
+  ],
+  [
+    'requests-bounded',
+    ({ requests }, budgets) => {
+      const expected = `every request answered within the ${String(budgets.deadlineMs)} ms deadline`;
+      if (requests === undefined) return fail('requests-bounded', expected, NO_EVIDENCE);
+      if (requests.unanswered > 0) {
+        return fail('requests-bounded', expected, `${String(requests.unanswered)} never answered`);
+      }
+      return requests.maxMs <= budgets.deadlineMs + DRILL_ANSWER_SLACK_MS
+        ? undefined
+        : fail('requests-bounded', expected, `slowest ${String(requests.maxMs)} ms`);
+    },
+  ],
+  [
+    'requests-prompt',
+    ({ requests }, budgets) => {
+      const expected = `no request held past one bounded delay, ${String(budgets.promptMs)} ms`;
+      if (requests === undefined) return fail('requests-prompt', expected, NO_EVIDENCE);
+      return requests.maxMs <= budgets.promptMs && requests.unanswered === 0
+        ? undefined
+        : fail(
+            'requests-prompt',
+            expected,
+            `slowest ${String(requests.maxMs)} ms, ${String(requests.unanswered)} unanswered`,
+          );
+    },
+  ],
+  [
+    'requests-ok',
+    ({ requests }) => {
+      const expected = 'every request accepted was answered 2xx';
+      if (requests === undefined) return fail('requests-ok', expected, NO_EVIDENCE);
+      const failed = failedCount(requests);
+      if (requests.ok === 0) return fail('requests-ok', expected, 'no request was answered');
+      return failed === 0
+        ? undefined
+        : fail('requests-ok', expected, `${String(failed)} non-2xx: ${codes(requests)}`);
+    },
+  ],
+  [
+    'recovers',
+    ({ recoveryMs }, budgets) => {
+      const expected = `a 2xx within ${String(budgets.recoveryMs)} ms of the fault clearing, no restart`;
+      if (recoveryMs === undefined) return fail('recovers', expected, NO_EVIDENCE);
+      if (recoveryMs === null) return fail('recovers', expected, 'no 2xx after the fault cleared');
+      return recoveryMs <= budgets.recoveryMs
+        ? undefined
+        : fail('recovers', expected, `first 2xx after ${String(recoveryMs)} ms`);
+    },
+  ],
+  ['readyz-flips', judgeReadyz],
+  [
+    'jobs-terminal',
+    ({ jobs }) => {
+      const expected = 'every job the drill saw reached a terminal state';
+      if (jobs === undefined) return fail('jobs-terminal', expected, NO_EVIDENCE);
+      if (jobs.ran === 0) return fail('jobs-terminal', expected, 'no job ran during the drill');
+      return jobs.stuck === 0
+        ? undefined
+        : fail('jobs-terminal', expected, `${String(jobs.stuck)} of ${String(jobs.ran)} stuck`);
+    },
+  ],
+  [
+    'jobs-once',
+    ({ jobs }) => {
+      const expected = 'no job run twice for one idempotency key';
+      if (jobs === undefined) return fail('jobs-once', expected, NO_EVIDENCE);
+      return jobs.doubled === 0
+        ? undefined
+        : fail('jobs-once', expected, `${String(jobs.doubled)} key(s) ran twice`);
+    },
+  ],
+  [
+    'role-exits',
+    ({ exit }, budgets) => {
+      const expected = `the signalled role exits within its drain, ${String(budgets.drainMs)} ms`;
+      if (exit === undefined) return fail('role-exits', expected, NO_EVIDENCE);
+      if (exit.afterMs === null) return fail('role-exits', expected, `${exit.role} never exited`);
+      return exit.afterMs <= budgets.drainMs
+        ? undefined
+        : fail('role-exits', expected, `${exit.role} exited after ${String(exit.afterMs)} ms`);
+    },
+  ],
+  [
+    'sync-resumes',
+    ({ sync }) => {
+      const expected = 'sync clients reconnected and resynced with no sequence gap';
+      if (sync === undefined) return fail('sync-resumes', expected, NO_EVIDENCE);
+      if (!sync.reconnected) return fail('sync-resumes', expected, 'a client never reconnected');
+      return sync.gaps === 0
+        ? undefined
+        : fail('sync-resumes', expected, `${String(sync.gaps)} sequence gap(s)`);
+    },
+  ],
+]);
+
+/**
+ * `dependencies`: 503 within `readinessMs` of the fault, and 200 again by the last sample.
+ * `process`: 200 throughout — the app declared that one dependency must not empty the fleet.
+ */
+function judgeReadyz(evidence: DrillEvidence, budgets: DrillBudgets): DrillFailure | undefined {
+  const samples = evidence.readyz;
+  const flips = budgets.readiness === 'dependencies';
+  const expected = flips
+    ? `/readyz 503 within ${String(budgets.readinessMs)} ms of the fault, then 200 again`
+    : '/readyz 200 throughout (health.readiness is process)';
+  if (samples === undefined || samples.length === 0) {
+    return fail('readyz-flips', expected, NO_EVIDENCE);
+  }
+  const last = samples[samples.length - 1];
+  const statuses = samples.map((sample) => String(sample.status ?? 'none')).join(',');
+  if (!flips) {
+    return samples.every((sample) => sample.status === 200)
+      ? undefined
+      : fail('readyz-flips', expected, `statuses ${statuses}`);
+  }
+  const down = samples.find((sample) => sample.status === 503);
+  if (down === undefined || down.atMs > budgets.readinessMs) {
+    return fail(
+      'readyz-flips',
+      expected,
+      `first 503 at ${down === undefined ? 'never' : `${String(down.atMs)} ms`}`,
+    );
+  }
+  return last?.status === 200
+    ? undefined
+    : fail('readyz-flips', expected, `last sample ${String(last?.status ?? 'none')}`);
+}
+
+/** Every stated property, judged; `ok` only when none failed and at least one was stated. */
+export function judgeDrill(
+  holds: readonly DrillProperty[],
+  evidence: DrillEvidence,
+  budgets: DrillBudgets,
+): DrillVerdict {
+  const failures: DrillFailure[] = [];
+  for (const property of holds) {
+    const failure = JUDGES.get(property)?.(evidence, budgets);
+    if (failure !== undefined) failures.push(failure);
+  }
+  return { ok: holds.length > 0 && failures.length === 0, failures };
+}
+
+/** The properties this file knows how to judge — what the drill table is held to. */
+export const DRILL_PROPERTIES: readonly DrillProperty[] = [...JUDGES.keys()];