@ultimat3/testing 29.4.0 → 30.1.0

package/src/job-sim-effects.ts

@@ -0,0 +1,142 @@
+// The effect log a simulated run is judged by: row writes (the entity row observer), mail, calls
+// out and enqueues, each attributed to the worker whose body made it. Reads are not effects, and
+// neither is what the process reports. An effect a worker makes after it died is dropped. Every
+// seam it borrows is chained onto what was installed — the reporter set aside — and handed back.
+
+import type { AsyncContext } from '@ultimat3/core';
+import {
+  asyncContext,
+  configureErrorReporting,
+  installedErrorReporter,
+  noopErrorReporter,
+} from '@ultimat3/core';
+import type { RowObserver } from '@ultimat3/entity';
+import type { MailDriver } from '@ultimat3/mail';
+
+export interface SimEffect {
+  /** `row:posts:insert`, `mail:receipt→a@x.example`, `network:POST api.example/v1/charges`. */
+  readonly key: string;
+  /** The worker whose body made it; absent for the harness. */
+  readonly worker?: string;
+  /** The last step that worker's run had written when the effect happened. */
+  readonly after?: string;
+}
+
+export interface EffectLog extends AsyncDisposable {
+  /** The worker a call in flight belongs to: what `run` scoped it to. */
+  readonly scope: AsyncContext<string>;
+  record(key: string): void;
+  /** A worker's run wrote this step — what the next effect it makes is placed after. */
+  stepWritten(worker: string, step: string): void;
+  effects(): readonly SimEffect[];
+  clear(): void;
+}
+
+export interface EffectLogOptions {
+  readonly isDead: (worker: string) => boolean;
+}
+
+const hostAndPath = (url: string): string => {
+  if (!URL.canParse(url)) return url;
+  const parsed = new URL(url);
+  return `${parsed.host}${parsed.pathname}`;
+};
+
+const urlOf = (input: RequestInfo | URL): string =>
+  typeof input === 'string' ? input : input instanceof URL ? input.toString() : input.url;
+
+const methodOf = (input: RequestInfo | URL, init: RequestInit | undefined): string =>
+  (
+    init?.method ?? (typeof input === 'object' && 'method' in input ? input.method : 'GET')
+  ).toUpperCase();
+
+export async function openEffectLog(options: EffectLogOptions): Promise<EffectLog> {
+  const { setRowObserver } = await import('@ultimat3/entity');
+  const mail = await import('@ultimat3/mail');
+  const scope = asyncContext<string>('the job simulation');
+  const log: SimEffect[] = [];
+  const lastStep = new Map<string, string>();
+
+  const record = (key: string): void => {
+    const worker = scope.get();
+    if (worker !== undefined && options.isDead(worker)) return;
+    const after = worker === undefined ? undefined : lastStep.get(worker);
+    log.push({
+      key,
+      ...(worker === undefined ? {} : { worker }),
+      ...(after === undefined ? {} : { after }),
+    });
+  };
+
+  const previousRows = setRowObserver(null);
+  const rows: RowObserver = {
+    onChange: (change) => {
+      record(`row:${change.entity}:${change.op}`);
+      previousRows?.onChange(change);
+    },
+    onBulk: (change) => {
+      for (let i = 0; i < change.rows; i += 1) record(`row:${change.entity}:${change.op}`);
+      previousRows?.onBulk?.(change);
+    },
+  };
+  setRowObserver(rows);
+
+  // A send with no mail driver installed is answered by an in-memory one: the effect is the send.
+  const previousMail = mail.tryMailDriver();
+  const deliver = previousMail ?? mail.memoryMailDriver();
+  const recording: MailDriver = {
+    name: deliver.name,
+    send: async (message) => {
+      const sent = await deliver.send(message);
+      record(`mail:${message.mailId}→${message.to.join(',')}`);
+      return sent;
+    },
+  };
+  mail.setMailDriver(recording);
+
+  // Set aside, never chained: a report is the process telling its operator, not the body acting.
+  // An app's sink keeps state across runs (`errorGroupsSink` enqueues on a group's FIRST sighting),
+  // so chained, whichever simulated run reported first would carry its enqueue and every other
+  // run's count would differ — and a death planted here would land in the app's own error store.
+  const previousReporter = installedErrorReporter();
+  configureErrorReporting({ reporter: noopErrorReporter });
+
+  // Recorded on the way out, then handed to whatever `fetch` is installed — the seal decides.
+  const previousFetch = globalThis.fetch;
+  const recordingFetch = (input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
+    record(`network:${methodOf(input, init)} ${hostAndPath(urlOf(input))}`);
+    return previousFetch(input, init);
+  };
+  globalThis.fetch = Object.assign(recordingFetch, previousFetch);
+
+  return {
+    scope,
+    record,
+    stepWritten: (worker, step) => {
+      lastStep.set(worker, step);
+    },
+    effects: () => [...log],
+    clear: () => {
+      log.length = 0;
+      lastStep.clear();
+    },
+    [Symbol.asyncDispose]: async (): Promise<void> => {
+      // Each restore runs whatever the one before it did: a throw must not strand a seam.
+      try {
+        globalThis.fetch = previousFetch;
+      } finally {
+        configureErrorReporting({ reporter: previousReporter });
+        if (previousMail === undefined) mail.resetMailDriver();
+        else mail.setMailDriver(previousMail);
+        setRowObserver(previousRows);
+      }
+    },
+  };
+}
+
+/** How many times each key happened. */
+export function effectCounts(effects: readonly SimEffect[]): ReadonlyMap<string, number> {
+  const counts = new Map<string, number>();
+  for (const effect of effects) counts.set(effect.key, (counts.get(effect.key) ?? 0) + 1);
+  return counts;
+}

package/src/job-sim-errors.ts

@@ -0,0 +1,72 @@
+// The job proofs' refusals: a killed run that diverged from a clean one, a seed that broke the
+// app's invariant, a seed that would not replay. Apart from ./errors, which owns the codes and the
+// one registration, because that catalogue is at its ceiling. Every fix replays the exact case.
+
+import {
+  renderCauseValue,
+  renderFixLiteral,
+  renderFixShellArg,
+  UltimateError,
+} from '@ultimat3/core';
+import './errors';
+
+export interface CrashDivergedInput {
+  readonly job: string;
+  /** The boundary the run was killed at: `charge`, `send (started)`, `ack`. */
+  readonly boundary: string;
+  readonly effect: string;
+  readonly clean: number;
+  readonly killed: number;
+  /** The last step written before the effect, in the clean run. */
+  readonly after?: string;
+}
+
+export class JobCrashDivergedError extends UltimateError {
+  constructor(input: CrashDivergedInput) {
+    const where =
+      input.after === undefined ? 'before the first step' : `after step "${input.after}"`;
+    super({
+      code: 'X_JOB_CRASH_DIVERGED',
+      cause: `job ${renderCauseValue(input.job)} killed at ${renderCauseValue(input.boundary)} made ${renderCauseValue(input.effect)} ${input.killed}× against ${input.clean}× in a clean run: it runs outside every step, ${where}, so the retry makes it again`,
+      fix: `step.run(${renderFixLiteral(`${input.boundary}-effect`, '<name>')}, async () => …)   # in job(${renderFixLiteral(input.job, '<job>')}): wrap the ${renderFixLiteral(input.effect, '<effect>')} made ${where}; key an external call on \`\${jobId}:<step>\`; then: x test job --filter <the test file>`,
+      meta: { ...input },
+    });
+  }
+}
+
+export interface SimInvariantInput {
+  readonly seed: number;
+  readonly faults: readonly string[];
+  readonly reason: string;
+}
+
+export class JobSimInvariantError extends UltimateError {
+  constructor(input: SimInvariantInput) {
+    const faults = input.faults.length === 0 ? 'no fault' : input.faults.join(', ');
+    super({
+      code: 'X_JOB_SIM_INVARIANT',
+      cause: `seed ${input.seed} broke the invariant (${renderCauseValue(input.reason)}) under ${renderCauseValue(faults)}`,
+      fix: `x test job --seed ${renderFixShellArg(String(input.seed), '<n>')} --filter <the test file>   # replays that one seed and its faults`,
+      meta: { seed: input.seed, faults: [...input.faults] },
+    });
+  }
+}
+
+export interface SimNondeterministicInput {
+  readonly seed: number;
+  /** The index of the first event the two traces disagree on. */
+  readonly at: number;
+  readonly first: string;
+  readonly second: string;
+}
+
+export class JobSimNondeterministicError extends UltimateError {
+  constructor(input: SimNondeterministicInput) {
+    super({
+      code: 'X_JOB_SIM_NONDETERMINISTIC',
+      cause: `seed ${input.seed} ran twice and the traces differ at event ${input.at}: ${renderCauseValue(input.first)} then ${renderCauseValue(input.second)} — a body reads the wall clock, real I/O or state an earlier run left`,
+      fix: `x test job --seed ${renderFixShellArg(String(input.seed), '<n>')} --filter <the test file>   # read time through Date (frozen) and ids through the job's input`,
+      meta: { ...input },
+    });
+  }
+}

package/src/job-sim-schedule.ts

@@ -0,0 +1,154 @@
+// The seeded picker: every driver call a simulated worker makes parks at a yield point, and one
+// seeded RNG decides which parked call goes next and which fault, if any, lands on it. One thread,
+// so the interleaving is the seed's alone; every pick and every fault is a line of the trace.
+
+import { assert } from '@ultimat3/core';
+import { advanceClock, seededRandom } from './determinism';
+import type { SimDecision, SimOp } from './job-sim-driver';
+import { SEND_AND_ANSWER } from './job-sim-driver';
+import type { SimWorld } from './job-sim-world';
+import { SIM_VISIBILITY_MS } from './job-sim-world';
+
+export const JOB_FAULT_KINDS = [
+  'kill',
+  'answer-lost',
+  'lease-lapse',
+  'duplicate-claim',
+  'clock-jump',
+] as const;
+
+export type JobFaultKind = (typeof JOB_FAULT_KINDS)[number];
+
+/** Every fault kind: what `simulateJobs` plants unless told fewer. */
+export const ALL_FAULTS: readonly JobFaultKind[] = JOB_FAULT_KINDS;
+
+/** Event-loop turns with nothing parked before a run is a body waiting on something unowned. */
+const MAX_IDLE_TURNS = 10_000;
+
+/** Calls whose answer can be lost after they landed: the ones that change something. */
+const WRITES: ReadonlySet<SimOp['call']> = new Set([
+  'claim',
+  'ack',
+  'nack',
+  'heartbeat',
+  'enqueue',
+  'enqueueMany',
+  'put',
+]);
+
+interface Parked {
+  readonly op: SimOp;
+  readonly resolve: (decision: SimDecision) => void;
+}
+
+export interface SimSchedulerOptions {
+  readonly seed: number;
+  readonly faults: readonly JobFaultKind[];
+  /** The chance a pick carries a fault, in [0, 1]. */
+  readonly faultRate: number;
+}
+
+export interface SimScheduler {
+  /** The world's `decide`: park until picked, then answer the fault the pick drew. */
+  decide(op: SimOp): Promise<SimDecision>;
+  /** Release parked calls one at a time until every task has settled. */
+  run(world: SimWorld, tasks: readonly Promise<unknown>[]): Promise<void>;
+  readonly trace: string[];
+  /** The faults planted, in order — what a broken invariant is reported with. */
+  readonly planted: string[];
+}
+
+const turn = (): Promise<void> => new Promise((resolve) => setImmediate(resolve));
+
+export function simScheduler(options: SimSchedulerOptions): SimScheduler {
+  // Its own stream, apart from the world's `Math.random`: a body's draws never shift a pick.
+  const rng = seededRandom((options.seed ^ 0x9e3779b9) >>> 0);
+  const pickOf = (length: number): number => Math.min(length - 1, Math.floor(rng() * length));
+  const parked: Parked[] = [];
+  const trace: string[] = [];
+  const planted: string[] = [];
+
+  const labelOf = (world: SimWorld, op: SimOp): string =>
+    [
+      op.worker,
+      op.call,
+      op.jobId === undefined ? undefined : world.name(op.jobId),
+      op.runId === undefined ? undefined : world.name(op.runId),
+      op.step,
+      op.status,
+      op.names?.join(','),
+    ]
+      .filter((part) => part !== undefined)
+      .join(' ');
+
+  const plant = (line: string): void => {
+    planted.push(line);
+    trace.push(`fault ${line}`);
+  };
+
+  /** Draw the pick's fault, landing the clock ones here; answer what the call itself suffers. */
+  const fault = async (world: SimWorld, op: SimOp, label: string): Promise<SimDecision> => {
+    if (options.faults.length === 0 || rng() >= options.faultRate) return SEND_AND_ANSWER;
+    const kind = options.faults[pickOf(options.faults.length)] ?? 'kill';
+    if (kind === 'kill') {
+      const landed = rng() < 0.5;
+      plant(`kill ${label} ${landed ? 'after it landed' : 'before it was sent'}`);
+      return landed ? { before: 'send', after: 'kill' } : { before: 'kill', after: 'answer' };
+    }
+    if (kind === 'answer-lost') {
+      if (!WRITES.has(op.call)) return SEND_AND_ANSWER;
+      plant(`answer-lost ${label}`);
+      return { before: 'send', after: 'lose' };
+    }
+    if (kind === 'lease-lapse') {
+      plant(`lease-lapse at ${label}`);
+      advanceClock(SIM_VISIBILITY_MS + 1);
+      return SEND_AND_ANSWER;
+    }
+    if (kind === 'clock-jump') {
+      const ms = 1 + Math.floor(rng() * (SIM_VISIBILITY_MS / 2));
+      plant(`clock-jump ${ms}ms at ${label}`);
+      advanceClock(ms);
+      return SEND_AND_ANSWER;
+    }
+    const running = (await world.rows()).filter((row) => row.state === 'running');
+    const victim = running[pickOf(running.length)];
+    if (victim === undefined) return SEND_AND_ANSWER;
+    plant(`duplicate-claim ${world.name(victim.id)} at ${label}`);
+    // The row's lease, cut to now: the next claim by anyone re-delivers a job still running here.
+    await world.sim.base.heartbeat(victim.id, { visibilityTimeoutMs: 0 });
+    return SEND_AND_ANSWER;
+  };
+
+  return {
+    trace,
+    planted,
+    decide: (op) => new Promise<SimDecision>((resolve) => parked.push({ op, resolve })),
+    async run(world, tasks) {
+      let settled = false;
+      const all = Promise.allSettled(tasks).then(() => {
+        settled = true;
+      });
+      for (let idle = 0; ; ) {
+        await turn();
+        const index = parked.length === 0 ? -1 : pickOf(parked.length);
+        const next = index === -1 ? undefined : parked.splice(index, 1)[0];
+        if (next === undefined) {
+          if (settled) break;
+          idle += 1;
+          assert(
+            idle < MAX_IDLE_TURNS,
+            `simulateJobs waited ${MAX_IDLE_TURNS} event-loop turns with no driver call parked: a job body awaits a timer, a socket or a promise the simulation does not own`,
+            'x test job --filter <the test file>   # replace the wait with step.sleep, and the I/O with a fixture the simulation installs',
+          );
+          continue;
+        }
+        idle = 0;
+        const label = labelOf(world, next.op);
+        trace.push(`pick ${label}`);
+        next.resolve(await fault(world, next.op, label));
+      }
+      await all;
+    },
+  };
+}

package/src/job-sim-world.ts

@@ -0,0 +1,198 @@
+// The world one simulated run happens in: one memory queue behind per-worker views, the effect
+// log, a fresh event bus, renewals on the frozen clock, the clock and `Math.random` put back to the
+// same instant and seed at the start — so one run is a function of its decisions and nothing else.
+// Ids are minted by the queue and differ run to run; the effect log names them by ordinal instead.
+
+import type { Ctx } from '@ultimat3/core';
+import { ctxOf } from '@ultimat3/core';
+import type {
+  EnqueueResult,
+  EventBus,
+  JobDriver,
+  JobExecution,
+  JobHandle,
+  JobRecord,
+} from '@ultimat3/jobs';
+import {
+  captureDeterminism,
+  installDeterminism,
+  restoreCapturedDeterminism,
+  seededRandom,
+} from './determinism';
+import { frozenScheduler } from './frozen-scheduler';
+import type { SimDecision, SimDriver, SimOp } from './job-sim-driver';
+import { isSimFault, simDriver } from './job-sim-driver';
+import type { EffectLog, SimEffect } from './job-sim-effects';
+import { openEffectLog } from './job-sim-effects';
+
+/** A lease short enough that a lapse is one clock jump, long enough that no renewal misses it. */
+export const SIM_VISIBILITY_MS = 30_000;
+/** Renewals every millisecond of test time: any move renews every running job once. */
+const RENEW_MS = 1;
+const CLAIM_LIMIT = 64;
+const LIVE: ReadonlySet<string> = new Set(['ready', 'delayed', 'running', 'suspended']);
+
+export interface SimWorldOptions {
+  /** Seeds `Math.random` for the run: a body's jitter and draws are the seed's. */
+  readonly seed: number;
+  /** What every worker runs its bodies under. Default: an anonymous worker. */
+  readonly context?: () => Ctx;
+  decide(op: SimOp): SimDecision | Promise<SimDecision>;
+  /** Told every landed call, after the world has read it. */
+  landed?(op: SimOp, answer: unknown): void;
+}
+
+export interface SimWorld extends AsyncDisposable {
+  readonly sim: SimDriver;
+  readonly log: EffectLog;
+  /** Filed by the harness, not a worker: no yield, no effect. */
+  enqueue<I>(handle: JobHandle<I>, input: I): Promise<EnqueueResult>;
+  /** One claim pass by `worker`, a real `jobWorker().tick()`. A planted fault ends it quietly. */
+  tick(worker: string): Promise<readonly JobExecution[]>;
+  rows(): Promise<readonly JobRecord[]>;
+  /** The instant the next live row is claimable — a lease lapsing or a `runAt` — if any. */
+  nextDue(): Promise<number | undefined>;
+  /** `job#1`, `run#1`: the ordinal a queue id is reported under. */
+  name(id: string): string;
+  /** The effect log, ids replaced by their ordinals. */
+  effects(): readonly SimEffect[];
+}
+
+export async function openSimWorld(options: SimWorldOptions): Promise<SimWorld> {
+  const jobs = await import('@ultimat3/jobs');
+  const clock = captureDeterminism();
+  if (!clock.installed) installDeterminism();
+  Math.random = seededRandom(options.seed);
+  const ordinals = new Map<string, string>();
+  const counters = new Map<string, number>();
+  const known = (id: string | undefined, kind: 'job' | 'run'): void => {
+    if (id === undefined || ordinals.has(id)) return;
+    const n = (counters.get(kind) ?? 0) + 1;
+    counters.set(kind, n);
+    ordinals.set(id, `${kind}#${n}`);
+  };
+  const remember = (answer: unknown): void => {
+    for (const one of Array.isArray(answer) ? answer : [answer]) {
+      if (typeof one !== 'object' || one === null) continue;
+      const row = one as { readonly id?: unknown; readonly runId?: unknown };
+      if (typeof row.id === 'string') known(row.id, 'job');
+      if (typeof row.runId === 'string') known(row.runId, 'run');
+    }
+  };
+
+  let log: EffectLog | undefined;
+  const base = jobs.memoryJobDriver();
+  const sim = simDriver(base, {
+    decide: (op) => options.decide(op),
+    landed: (op, answer) => {
+      remember(answer);
+      if (op.call === 'put' && op.step !== undefined) log?.stepWritten(op.worker, op.step);
+      if ((op.call === 'enqueue' || op.call === 'enqueueMany') && log !== undefined) {
+        const results = (Array.isArray(answer) ? answer : [answer]) as readonly EnqueueResult[];
+        results.forEach((result, i) => {
+          if (!result.deduped) log?.record(`enqueue:${op.names?.[i] ?? '?'}`);
+        });
+      }
+      options.landed?.(op, answer);
+    },
+  });
+  log = await openEffectLog({ isDead: sim.isDead });
+  const effectLog = log;
+
+  // A body's own enqueue goes through the view of the worker running it: a dead one's is refused.
+  const ambient = (): JobDriver => sim.viewFor(effectLog.scope.get() ?? 'harness');
+  const ambientDriver: JobDriver = new Proxy(base, {
+    get: (_target, key) => Reflect.get(ambient(), key),
+  });
+  const previousDriver = jobs.jobDriver();
+  jobs.setJobDriver(ambientDriver);
+  const previousBus: EventBus = jobs.eventBus();
+  jobs.setEventBus(jobs.memoryEventBus());
+  const renewals = frozenScheduler();
+  const anonymous = ctxOf({ role: 'worker' });
+  const context = options.context ?? (() => anonymous);
+
+  const queues = (): readonly string[] => [
+    ...new Set([jobs.DEFAULT_QUEUE, ...jobs.registeredJobs().map((handle) => handle.queue)]),
+  ];
+
+  const rows = async (): Promise<readonly JobRecord[]> => {
+    const all: JobRecord[] = [];
+    for (let after: string | undefined; ; ) {
+      const page =
+        (await base.introspect?.list({
+          limit: jobs.MAX_JOB_PAGE,
+          ...(after === undefined ? {} : { after }),
+        })) ?? [];
+      all.push(...page);
+      const last = page[page.length - 1];
+      if (last === undefined || page.length < jobs.MAX_JOB_PAGE) return all;
+      after = jobs.jobCursor(last);
+    }
+  };
+
+  const normalise = (key: string): string => {
+    let out = key;
+    for (const [id, ordinal] of ordinals) out = out.split(id).join(ordinal);
+    return out;
+  };
+
+  return {
+    sim,
+    log: effectLog,
+    enqueue: async (handle, input) => {
+      const result = await base.enqueue({
+        name: handle.name,
+        queue: handle.queue,
+        input,
+        idempotencyKey: handle.idempotencyKeyFor(input),
+        maxAttempts: jobs.enqueuedAttempts(handle.retry, handle.compensates),
+      });
+      remember(result);
+      return result;
+    },
+    tick: (worker) =>
+      effectLog.scope.run(worker, async () => {
+        const one = jobs.jobWorker({
+          driver: sim.viewFor(worker),
+          workerId: worker,
+          queues: queues(),
+          concurrency: CLAIM_LIMIT,
+          visibilityTimeoutMs: SIM_VISIBILITY_MS,
+          heartbeatIntervalMs: RENEW_MS,
+          schedule: renewals.schedule,
+          pollIntervalMs: 0,
+          drainOnShutdown: false,
+          context,
+        });
+        try {
+          return await one.tick();
+        } catch (error) {
+          // A claim refused by a planted fault is the pass's end; anything else is a real failure.
+          if (isSimFault(error)) return [];
+          throw error;
+        }
+      }),
+    rows,
+    nextDue: async () => {
+      const due = (await rows())
+        .filter((row) => LIVE.has(row.state))
+        .map((row) => (row.state === 'running' ? (row.visibleAt ?? row.runAt) : row.runAt));
+      return due.length === 0 ? undefined : Math.min(...due);
+    },
+    name: (id) => ordinals.get(id) ?? id,
+    effects: () => effectLog.effects().map((effect) => ({ ...effect, key: normalise(effect.key) })),
+    [Symbol.asyncDispose]: async (): Promise<void> => {
+      try {
+        await effectLog[Symbol.asyncDispose]();
+      } finally {
+        renewals[Symbol.dispose]();
+        jobs.setEventBus(previousBus);
+        if (previousDriver === undefined) jobs.resetJobDriver();
+        else jobs.setJobDriver(previousDriver);
+        await base.close();
+        restoreCapturedDeterminism(clock);
+      }
+    },
+  };
+}