@ultimat3/testing 29.3.2 → 30.0.0

package/src/exercise-report.ts

@@ -0,0 +1,282 @@
+// What one primitive did in the exercise pass, and the room it did it in: the network sealed with
+// every allow and mock cleared, the mail fixture, the clock frozen, statements and committed rows
+// observed. Every seam is process-global, so the room hands back exactly what it found.
+
+import { isSelfOrigin, peerAddressClass } from '@ultimat3/core';
+import type { StatementObserver } from '@ultimat3/db';
+import {
+  expectedSeqScanReason,
+  setStatementObserver,
+  statementFingerprint,
+  statementObserver,
+} from '@ultimat3/db';
+import type { RowObserver } from '@ultimat3/entity';
+import { captureDeterminism, installDeterminism, restoreCapturedDeterminism } from './determinism';
+import { testMail } from './fixture-mail';
+import type { PlanProof, SentStatement } from './plan-sweep';
+import type { ScaleProof, StatementTally } from './scale-sweep';
+import {
+  captureNetwork,
+  isNetworkSealed,
+  requestLog,
+  resetNetwork,
+  restoreCapturedNetwork,
+  sealNetwork,
+  unsealNetwork,
+} from './sealed-network';
+
+export type ExerciseEffect =
+  | { readonly kind: 'row'; readonly entity: string; readonly op: string; readonly rows: number }
+  | { readonly kind: 'mail'; readonly mail: string; readonly count: number }
+  /** `refused`: sealed, never sent. A call to this machine is the only one that is not. */
+  | {
+      readonly kind: 'network';
+      readonly method: string;
+      readonly host: string;
+      readonly refused: boolean;
+    }
+  | { readonly kind: 'enqueue'; readonly job: string; readonly count: number };
+
+export type ExerciseOutcome = 'reached' | 'refused' | 'unreached' | 'off';
+
+/** B's call before A's positive control, and after it. */
+export type TenantLeg = 'before' | 'after';
+
+export interface TenantFinding {
+  readonly code: 'X_TENANT_LEAK' | 'X_TENANT_WRITE_CROSSED';
+  readonly leg: TenantLeg;
+  /** LEAK: where A's canary or id was found, e.g. 'output.items[0].url' or 'effect.mail[0].to' */
+  readonly path?: string;
+  /** WRITE_CROSSED: the table and the primary key of A's changed row */
+  readonly table?: string;
+  readonly id?: string;
+}
+
+/** The tenant proof's verdict on one primitive: B attacked A's rows twice, around A's own call. */
+export type TenantProof =
+  | { readonly verdict: 'proven' }
+  /** The policy admits anonymous: read half skipped, write half kept and clean. */
+  | { readonly verdict: 'public'; readonly reason: string }
+  /** A's positive control was not reached, so B's clean answer proves nothing. */
+  | { readonly verdict: 'unproven'; readonly reason: string }
+  | { readonly verdict: 'crossed'; readonly findings: readonly TenantFinding[] };
+
+/** One effect a run killed at `boundary` made a different number of times than a clean run. */
+export interface CrashFinding {
+  /** `charge`, `charge (started)`, `unwinding`, `compensate:charge`, `ack`. */
+  readonly boundary: string;
+  /** `forced-fail`: a `compensates: true` job made to fail for good, then unwound. */
+  readonly mode: 'clean' | 'forced-fail';
+  /** `row:posts:insert`, `mail:receipt→a@x.example`, `network:POST api.example/v1`, `enqueue:x`. */
+  readonly effect: string;
+  readonly clean: number;
+  readonly killed: number;
+  /** The last step the clean run had written before the effect; absent, before the first. */
+  readonly after?: string;
+}
+
+/** The crash proof's verdict on a reached job: killed at every step boundary (`crashProof`). */
+export type CrashProof =
+  | { readonly verdict: 'held'; readonly boundaries: number; readonly reruns: number }
+  | {
+      readonly verdict: 'diverged';
+      readonly boundaries: number;
+      /** Lost answers that reran a step: at-least-once, counted, never red. */
+      readonly reruns: number;
+      readonly findings: readonly CrashFinding[];
+    }
+  /** The proof itself could not finish: reported, never red. */
+  | { readonly verdict: 'unproven'; readonly reason: string };
+
+export interface ExerciseRecord {
+  readonly kind: 'action' | 'query' | 'job';
+  readonly name: string;
+  readonly outcome: ExerciseOutcome;
+  readonly code?: string;
+  readonly reason?: string;
+  readonly statements: number;
+  readonly effects: readonly ExerciseEffect[];
+  readonly ms: number;
+  /** For an `unreached` one: the line to paste into `defineExercise({ … })`. */
+  readonly override?: string;
+  /** On every record that ran — absent only on `off` and `refused`. The fields above are A's. */
+  readonly tenant?: TenantProof;
+  /** On a reached job only: its control's input, killed at every step boundary. */
+  readonly crash?: CrashProof;
+  /** Every statement it sent, planned with no index but the declared ones (`plan-sweep.ts`). */
+  readonly plans?: PlanProof;
+  /** On a reached primitive: its statement shapes at 2 and at 8 rows a table (`scale-sweep.ts`). */
+  readonly scale?: ScaleProof;
+}
+
+/** What an effect carried, for the tenant proof's read half: searched, never reported as is. */
+export interface EffectPayload {
+  /** `effect.mail[0]`, `effect.network[1]`, `effect.enqueue[0]`. */
+  readonly path: string;
+  readonly value: unknown;
+}
+
+const IDENTIFIER = /^[A-Za-z_$][\w$]*$/;
+
+/** `publishPost: { actor, input: (drawn) => ({ ...drawn, postId: '<…>' }) },` — valid TS to paste. */
+export function overrideLine(name: string, idFields: readonly string[]): string {
+  const key = IDENTIFIER.test(name) ? name : JSON.stringify(name);
+  const bound = idFields
+    .map((field) => `, ${IDENTIFIER.test(field) ? field : JSON.stringify(field)}: '<a real id>'`)
+    .join('');
+  return `${key}: { actor: <an actor its policy permits>, input: (drawn) => ({ ...drawn${bound} }) },`;
+}
+
+const hostOf = (url: string): string => (URL.canParse(url) ? new URL(url).host : url);
+
+/** `sealed-network.ts`'s own exemption: this machine and a server this process booted. */
+const local = (url: string): boolean => {
+  if (isSelfOrigin(url)) return true;
+  if (!URL.canParse(url)) return false;
+  const name = new URL(url).hostname.replace(/^\[|\]$/g, '');
+  if (name.toLowerCase() === 'localhost') return true;
+  const kind = peerAddressClass(name);
+  return kind === 'loopback' || kind === 'unspecified';
+};
+
+const tally = (keys: readonly string[]): ReadonlyMap<string, number> => {
+  const counts = new Map<string, number>();
+  for (const key of keys) counts.set(key, (counts.get(key) ?? 0) + 1);
+  return counts;
+};
+
+/** What happened between `begin()` and `end()`. */
+export interface Observed {
+  readonly statements: number;
+  readonly effects: readonly ExerciseEffect[];
+  /** Calls out the seal refused: each makes the primitive `unreached`, whatever it answered. */
+  readonly refusedHosts: readonly string[];
+  /** Each mail's addresses and text, each call out's url and body — a sealed one included. */
+  readonly payloads: readonly EffectPayload[];
+  /** Each statement shape and how often, the `expectedQueryLoop` ones told apart. */
+  readonly shapes: readonly StatementTally[];
+}
+
+export interface ExerciseRoom extends AsyncDisposable {
+  /** `subject`: the primitive the statements until `end()` are sent for — what the plan sweep reads. */
+  begin(subject?: string): void;
+  end(): Observed;
+  /** Every distinct statement text sent between a `begin(subject)` and its `end()`, so far. */
+  sent(): ReadonlyMap<string, SentStatement>;
+}
+
+export async function openExerciseRoom(): Promise<ExerciseRoom> {
+  const { setRowObserver } = await import('@ultimat3/entity');
+  const clock = captureDeterminism();
+  if (!clock.installed) installDeterminism();
+  const network = captureNetwork();
+  const sealedBefore = isNetworkSealed();
+  // Every allow and mock cleared: a test's `mockFetch` is not the app's, and a call out here is
+  // a refusal by definition.
+  resetNetwork();
+  sealNetwork();
+  const mail = await testMail();
+  let statements = 0;
+  let logFrom = 0;
+  let subject: string | undefined;
+  const rows: string[] = [];
+  const shapes = new Map<string, { count: number; unexpected: number; expected?: string }>();
+  const sent = new Map<string, { primitives: Set<string>; scan?: string }>();
+  const previousStatements = statementObserver();
+  const counting: StatementObserver = {
+    onStatement: (event) => {
+      statements += 1;
+      const fingerprint = statementFingerprint(event);
+      const shape = shapes.get(fingerprint) ?? { count: 0, unexpected: 0 };
+      shape.count += 1;
+      if (event.expected === undefined) shape.unexpected += 1;
+      else shape.expected = event.expected;
+      shapes.set(fingerprint, shape);
+      if (subject === undefined) return;
+      // Whitespace collapsed, so one builder's two indentations are one statement to plan.
+      const text = event.text.replace(/\s+/g, ' ').trim();
+      const one = sent.get(text) ?? { primitives: new Set<string>() };
+      one.primitives.add(subject);
+      // Read here, in the sender's own async context: the scope is the statement's, not the room's.
+      const scan = expectedSeqScanReason();
+      if (scan !== undefined) one.scan = scan;
+      sent.set(text, one);
+    },
+  };
+  setStatementObserver(counting);
+  const rowObserver: RowObserver = {
+    onChange: (change) => rows.push(`${change.entity}\u0000${change.op}`),
+    onBulk: (change) => {
+      for (let i = 0; i < change.rows; i += 1) rows.push(`${change.entity}\u0000${change.op}`);
+    },
+  };
+  const previousRows = setRowObserver(rowObserver);
+
+  return {
+    begin: (name) => {
+      subject = name;
+      statements = 0;
+      shapes.clear();
+      rows.length = 0;
+      mail.clear();
+      logFrom = requestLog().length;
+    },
+    end: (): Observed => {
+      const calls = requestLog().slice(logFrom);
+      const effects: ExerciseEffect[] = [
+        ...[...tally(rows)].map(([key, count]): ExerciseEffect => {
+          const [entity = '', op = ''] = key.split('\u0000');
+          return { kind: 'row', entity, op, rows: count };
+        }),
+        ...[...tally(mail.outbox().map((sent) => sent.message.mailId))].map(
+          ([id, count]): ExerciseEffect => ({ kind: 'mail', mail: id, count }),
+        ),
+        ...calls.map(
+          (call): ExerciseEffect => ({
+            kind: 'network',
+            method: call.method,
+            host: hostOf(call.url),
+            refused: !local(call.url),
+          }),
+        ),
+      ];
+      const refusedHosts = calls.filter((call) => !local(call.url)).map((c) => hostOf(c.url));
+      const payloads: EffectPayload[] = [
+        ...mail.outbox().map(({ message }, i) => ({
+          path: `effect.mail[${i}]`,
+          value: {
+            to: message.to,
+            cc: message.cc,
+            bcc: message.bcc,
+            replyTo: message.replyTo,
+            subject: message.subject,
+            text: message.text,
+            html: message.html,
+            unsubscribeUrl: message.unsubscribeUrl,
+          },
+        })),
+        // Headers left out: a bearer token is not A's row, and a header is no row's text.
+        ...calls.map((call, i) => ({
+          path: `effect.network[${i}]`,
+          value: { url: call.url, body: call.body },
+        })),
+      ];
+      subject = undefined;
+      const tallies = [...shapes].map(([fingerprint, shape]) => ({ fingerprint, ...shape }));
+      return { statements, effects, refusedHosts, payloads, shapes: tallies };
+    },
+    sent: () => sent,
+    [Symbol.asyncDispose]: async (): Promise<void> => {
+      // Each restore runs whatever the one before it did: a throw must not strand a seam.
+      try {
+        mail[Symbol.dispose]();
+      } finally {
+        setRowObserver(previousRows);
+        setStatementObserver(previousStatements);
+        restoreCapturedNetwork(network);
+        if (!sealedBefore) unsealNetwork();
+        restoreCapturedDeterminism(clock);
+      }
+    },
+  };
+}

package/src/exercise-request.ts

@@ -0,0 +1,38 @@
+// The context an action or a query runs in during the exercise pass: an HTTP request's, built the
+// way the pipeline builds one, because `surface: 'http'` is the call the pass makes. A bare `Ctx`
+// has no inbound headers, so a session action's `useRequestContext()` answered X_NO_REQUEST, a 500.
+
+import type { Actor } from '@ultimat3/core';
+import { anonymousActor, runWithContext, useContext, withChildContext } from '@ultimat3/core';
+
+/** Where the pass's requests say they came from: no host the app serves, and never sent. */
+const EXERCISE_ORIGIN = 'https://exercise.test';
+
+/**
+ * `fn` inside a fresh request as `actor`: no cookie, no header, a POST to the primitive's own name,
+ * nothing spending a rate bucket. The actor is ON the request, so the caller passes `invoke` no
+ * `actor` — its impersonation path rebuilds a plain child context and drops the request's fields.
+ * The child is derived first for the services an actor binds, then the request's fields go over
+ * it, as `withAnonymousRequest` (`@ultimat3/http`) does.
+ */
+export async function withExerciseRequest<T>(
+  name: string,
+  actor: Actor | null,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const http = await import('@ultimat3/http');
+  const request = http.requestContext({
+    url: new URL(`/${encodeURIComponent(name)}`, EXERCISE_ORIGIN),
+    method: 'POST',
+    role: 'web',
+    config: http.defineHttpConfig({ rateLimit: { scope: 'process' } }),
+    requestHeaders: new Headers(),
+  });
+  return runWithContext(http.asCtx(request), () =>
+    withChildContext({ actor: actor ?? anonymousActor() }, () => {
+      const child = useContext();
+      const scope = { ...child, ...request, actor: child.actor };
+      return runWithContext(http.asCtx(scope), fn);
+    }),
+  );
+}

package/src/exercise-sweeps.ts

@@ -0,0 +1,100 @@
+// What runs after every primitive's own call: the scale sweep (each reached one again, at 2 and at 8
+// rows a table) and the plan sweep (every statement the pass sent, planned on the migrated
+// database). Both verdicts land on the record of the primitive that sent the statements.
+
+import type { Actor } from '@ultimat3/core';
+import type { Policy } from '@ultimat3/policy';
+import type { StandardSchemaV1 } from '@ultimat3/schema';
+import type { ExerciseOverrides } from './define-exercise';
+import type { ActorRows, RoleGrants } from './exercise-actor';
+import { asSeededRow, chooseActor } from './exercise-actor';
+import type { Sampler } from './exercise-input';
+import { bindIds, drawInput, seededIds } from './exercise-input';
+import { overrideInput } from './exercise-override-input';
+import type { ExerciseRecord } from './exercise-report';
+import { sweepPlans } from './plan-sweep';
+import type { ScaleSweepOptions } from './scale-sweep-run';
+import { scaleSweep } from './scale-sweep-run';
+import type { SeedGraph } from './seed-graph-plan';
+import type { SeedEntity } from './seed-graph-shape';
+
+/** The pass's options, as far as choosing who calls and with what goes. */
+interface CallOptions {
+  readonly entities: readonly SeedEntity[];
+  readonly overrides?: ExerciseOverrides;
+  readonly roles: RoleGrants;
+}
+
+/** The seeded rows of the entity `defineExercise(…, { actor })` named, or none. */
+export const actorRowsOf = (options: CallOptions, graph: SeedGraph): ActorRows | undefined => {
+  const name = options.overrides?.actorEntity;
+  const entity = options.entities.find((one) => one.$name === name);
+  if (entity === undefined) return undefined;
+  return { entity, rows: graph.inserts.filter((row) => row.entity === entity.$name) };
+};
+
+/** The actor of `chosen`, seated as a row of the app's actor entity in `tenant` when it names one. */
+export const seated = (actor: Actor | null, tenant: string, rows: ActorRows | undefined) =>
+  rows === undefined ? actor : asSeededRow(actor, tenant, rows);
+
+/** A primitive as a sized call needs it: its schema and its policy. */
+interface Callable {
+  readonly name: string;
+  readonly input: StandardSchemaV1;
+  readonly policy: Policy | undefined;
+  readonly permissions: readonly string[];
+}
+
+/** One call of `subject` in `tenant` of a sized graph: the control's actor and first input. */
+export async function sizedCall(
+  subject: Callable,
+  options: CallOptions,
+  sampler: Sampler,
+  at: { readonly graph: SeedGraph; readonly tenant: string; readonly self: boolean },
+): Promise<{ readonly actor: Actor | null; readonly input: unknown } | undefined> {
+  const { graph, tenant, self } = at;
+  const override = options.overrides?.entries.get(subject.name);
+  const chosen = chooseActor(subject.policy, subject.permissions, options.roles, tenant, override);
+  if (chosen.kind === 'refused') return undefined;
+  const rows = actorRowsOf(options, graph);
+  const actor = seated(chosen.actor, tenant, rows);
+  const scope = {
+    tenant,
+    actorId: actor?.id,
+    subject: subject.name,
+    actorEntity: rows?.entity.$name,
+  };
+  const lookup = seededIds(graph, options.entities, { ...scope, self });
+  const drawn = bindIds(drawInput(sampler, subject.input, 0, 0), lookup);
+  const bindInput = overrideInput(subject.name, override);
+  return { actor, input: bindInput === undefined ? drawn : await bindInput(drawn, actor) };
+}
+
+/**
+ * The scale sweep inside the room, the room closed, then the plan sweep — on the installed client
+ * only: with none, the pass ran on a driver that sends no SQL and there is no plan to read.
+ */
+export async function attachSweeps(
+  records: readonly ExerciseRecord[],
+  options: ScaleSweepOptions,
+): Promise<readonly ExerciseRecord[]> {
+  let scale: Awaited<ReturnType<typeof scaleSweep>>;
+  try {
+    scale = await scaleSweep(options);
+  } finally {
+    await options.room[Symbol.asyncDispose]();
+  }
+  const db = await import('@ultimat3/db');
+  const plans = db.dbClientInstalled()
+    ? await sweepPlans(db.db(), options.room.sent(), options.entities)
+    : new Map();
+  return records.map((record) => {
+    const proof = plans.get(record.name);
+    const sized = scale.get(record.name);
+    return {
+      ...record,
+      ...(proof === undefined ? {} : { plans: proof }),
+      ...(sized === undefined ? {} : { scale: sized }),
+    };
+  });
+}