@devosurf/tesser-testing 0.1.0-alpha.0

package/src/engine.ts

@@ -0,0 +1,571 @@
+// The in-process TestEngine (ADR-0008): journal-of-results semantics identical to the
+// durable server engine — occurrence-disambiguated steps, real retry policies on a
+// time-skipping clock, undo-in-reverse on terminal failure, connector calls captured —
+// with connectors auto-mocked. Milliseconds per run; failures machine-actionable.
+
+import { AsyncLocalStorage } from "node:async_hooks";
+import type {
+  AutomationDef,
+  Ctx,
+  EventDefinition,
+  HarnessRunRequest,
+  HarnessRunResult,
+  Logger,
+  NormalizedModelRequest,
+  NormalizedModelResponse,
+  Schema,
+  StepOpts,
+  WebhookRequest,
+} from "@devosurf/tesser-sdk";
+import { isRetryableError, isTerminalError } from "@devosurf/tesser-sdk";
+import {
+  buildConnectorClient,
+  buildHarnesses,
+  buildOperators,
+  decodeJournal,
+  encodeJournal,
+  isRetrySafe,
+  nextRetryDelayMs,
+  parseDuration,
+  resolveRetryPolicy,
+  validateSchema,
+  type JsonValue,
+} from "@devosurf/tesser-sdk/internal";
+import type { AnyAction, ConnectorInstance } from "@devosurf/tesser-sdk/connector";
+import { createSpy, recordCall, type Spy } from "./spy.js";
+import { sampleFromSchema } from "./sample.js";
+
+export class TestConfigError extends Error {
+  constructor(
+    message: string,
+    readonly hint?: string,
+  ) {
+    super(hint ? `${message}\n  hint: ${hint}` : message);
+    this.name = "TestConfigError";
+  }
+}
+
+export interface JournalEntry {
+  kind: "step" | "signal" | "sleep";
+  name: string;
+  occurrence: number;
+  status: "completed" | "failed" | "timed-out";
+  attempts: number;
+  result?: unknown;
+  error?: SerializedError;
+  /** Step names this entry's undo was executed for (undo bookkeeping lives run-level). */
+}
+
+export interface SerializedError {
+  name: string;
+  message: string;
+  retryable: boolean;
+  terminal: boolean;
+}
+
+export interface MachineFailure {
+  automation: string;
+  status: "failed";
+  error: SerializedError;
+  failedStep?: { name: string; occurrence: number; attempts: number; error: SerializedError };
+  steps: Array<{ name: string; occurrence: number; status: string; attempts: number }>;
+  connectorCalls: Array<{ step?: string; action?: string; input: unknown }>;
+  suggestion?: string;
+}
+
+export type StepMock = unknown | ((input: unknown, info: { action: string; connection: string }) => unknown);
+export type ScriptedModelTurn =
+  | NormalizedModelResponse
+  | ((request: NormalizedModelRequest, info: { operatorKey: string; modelKey: string; turn: number }) => NormalizedModelResponse | Promise<NormalizedModelResponse>);
+export type ScriptedHarnessRun =
+  | HarnessRunResult<unknown>
+  | ((request: HarnessRunRequest<unknown>, info: { harnessKey: string; invocation: number }) => HarnessRunResult<unknown> | Promise<HarnessRunResult<unknown>>);
+
+export interface RunOptions {
+  input?: unknown;
+  /** Mock connector-call results by the *step name* they happen inside. The step's own
+   *  code still runs; only the external call is replaced. */
+  mocks?: Record<string, StepMock>;
+  /** Replay seeding (ADR-0002 recovery semantics): steps listed here return their
+   *  recorded results WITHOUT executing — exactly how the durable engine resumes.
+   *  Used by `tesser replay` fixtures. */
+  journal?: Array<{ name: string; occurrence?: number; result: unknown }>;
+  /** Mock whole connections by action path: { slack: { chat: { postMessage: fn } } }. */
+  connections?: Record<string, unknown>;
+  secrets?: Record<string, string>;
+  /** Scripted model responses by Operator key and Model key. Operators never call live models in tests. */
+  models?: Record<string, Record<string, ScriptedModelTurn | ScriptedModelTurn[]>>;
+  /** Scripted Harness results by Harness key. Harnesses never run live subprocesses in tests. */
+  harnesses?: Record<string, ScriptedHarnessRun | ScriptedHarnessRun[]>;
+  /** Resolutions for ctx.waitForSignal, by name; value, per-occurrence array, or fn. */
+  signals?: Record<string, unknown | unknown[] | ((occurrence: number) => unknown)>;
+  request?: WebhookRequest;
+}
+
+export interface TestRunResult<TOutput = unknown> {
+  status: "completed" | "failed";
+  result?: TOutput;
+  error?: SerializedError & { step?: string };
+  /** Step results by name — the value for a single occurrence, an array for repeats. */
+  steps: Record<string, unknown>;
+  journal: JournalEntry[];
+  /** Spies by step name AND dotted action path; safe to read for never-called names. */
+  calls: Record<string, Spy>;
+  emitted: Array<{ event: string; payload: unknown }>;
+  slept: string[];
+  undone: string[];
+  logs: Array<{ level: "info" | "warn" | "error"; msg: string; meta?: Record<string, unknown> }>;
+  failure(): MachineFailure | null;
+}
+
+function serializeError(err: unknown): SerializedError {
+  const e = err instanceof Error ? err : new Error(String(err));
+  return {
+    name: e.name,
+    message: e.message,
+    retryable: isRetryableError(err),
+    terminal: isTerminalError(err),
+  };
+}
+
+interface ActiveStep {
+  name: string;
+  sawUnsafeWrite: boolean;
+}
+
+export async function executeAutomation<TOutput>(
+  def: AutomationDef<any, TOutput, any, any, any, any, any>,
+  opts: RunOptions = {},
+): Promise<TestRunResult<TOutput>> {
+  const journal: JournalEntry[] = [];
+  const journalByKey = new Map<string, JournalEntry>();
+  for (const seeded of opts.journal ?? []) {
+    const occurrence = seeded.occurrence ?? 1;
+    const entry: JournalEntry = {
+      kind: "step",
+      name: seeded.name,
+      occurrence,
+      status: "completed",
+      attempts: 1,
+      result: encodeJournal(seeded.result),
+    };
+    journalByKey.set(`step:${seeded.name}#${occurrence}`, entry);
+    journal.push(entry);
+  }
+  const emitted: TestRunResult["emitted"] = [];
+  const slept: string[] = [];
+  const undone: string[] = [];
+  const logs: TestRunResult["logs"] = [];
+  const connectorCalls: MachineFailure["connectorCalls"] = [];
+  const undoStack: Array<{ name: string; undo: () => unknown | Promise<unknown> }> = [];
+
+  const spies = new Map<string, Spy>();
+  const spyFor = (key: string): Spy => {
+    let s = spies.get(key);
+    if (!s) {
+      s = createSpy(key);
+      spies.set(key, s);
+    }
+    return s;
+  };
+  for (const key of Object.keys(opts.mocks ?? {})) spyFor(key);
+
+  const calls: Record<string, Spy> = new Proxy({} as Record<string, Spy>, {
+    get: (_t, prop) => (typeof prop === "string" ? spyFor(prop) : undefined),
+    has: () => true,
+    ownKeys: () => [...spies.keys()],
+    getOwnPropertyDescriptor: () => ({ enumerable: true, configurable: true }),
+  });
+
+  const activeStep = new AsyncLocalStorage<ActiveStep>();
+
+  const logger: Logger = {
+    info: (msg, meta) => logs.push({ level: "info", msg, ...(meta ? { meta } : {}) }),
+    warn: (msg, meta) => logs.push({ level: "warn", msg, ...(meta ? { meta } : {}) }),
+    error: (msg, meta) => logs.push({ level: "error", msg, ...(meta ? { meta } : {}) }),
+  };
+
+  // ---- connections: real connector defs, mocked transport ----
+  const connections: Record<string, unknown> = {};
+  for (const [connKey, connector] of Object.entries((def.connections ?? {}) as Record<string, ConnectorInstance<any, any>>)) {
+    connections[connKey] = buildConnectorClient(connector, async (path, actionDef, rawInput) => {
+      const step = activeStep.getStore();
+      const actionPath = path.join(".");
+      const fullPath = `${connKey}.${actionPath}`;
+      if (!step) {
+        throw new TestConfigError(
+          `connector call ${fullPath} happened outside ctx.step()`,
+          `side effects must live inside a step (ADR-0002) — wrap the call: ctx.step("name", () => ctx.connections.${fullPath}(...))`,
+        );
+      }
+      const input = await validateSchema(
+        (actionDef as AnyAction).input,
+        rawInput ?? {},
+        `${fullPath} input`,
+      );
+
+      if (!isRetrySafe(actionDef as AnyAction, connector.__connector.idempotencyHeader !== undefined)) {
+        step.sawUnsafeWrite = true;
+      }
+
+      const resolve = async (): Promise<{ value: unknown; validate: boolean }> => {
+        // 1) whole-connection mock by action path
+        const connMockRoot = opts.connections?.[connKey];
+        if (connMockRoot !== undefined) {
+          let node: unknown = connMockRoot;
+          for (const seg of path) node = (node as Record<string, unknown> | undefined)?.[seg];
+          if (node !== undefined) {
+            return {
+              value: typeof node === "function" ? await (node as (i: unknown) => unknown)(input) : node,
+              validate: false,
+            };
+          }
+        }
+        // 2) step-name mock
+        const stepMock = opts.mocks?.[step.name];
+        if (stepMock !== undefined) {
+          return {
+            value:
+              typeof stepMock === "function"
+                ? await (stepMock as (i: unknown, info: { action: string; connection: string }) => unknown)(
+                    input,
+                    { action: actionPath, connection: connKey },
+                  )
+                : stepMock,
+            validate: false,
+          };
+        }
+        // 3) connector-declared sample
+        const sample = connector.__connector.samples?.[actionPath];
+        if (sample !== undefined) return { value: sample, validate: true };
+        // 4) derive from the action's output schema
+        const derived = await sampleFromSchema((actionDef as AnyAction).output);
+        if (derived !== undefined) return { value: derived, validate: true };
+        throw new TestConfigError(
+          `no mock for connector call ${fullPath} in step "${step.name}"`,
+          `provide mocks: { "${step.name}": () => <result> } or connections: { ${connKey}: { ${actionPath
+            .split(".")
+            .join(": { ")}: () => <result> ${"}".repeat(actionPath.split(".").length)} }`,
+        );
+      };
+
+      try {
+        const { value, validate } = await resolve();
+        const result = validate
+          ? await validateSchema((actionDef as AnyAction).output, value, `${fullPath} sample output`)
+          : value;
+        const record = { args: [input], step: step.name, action: fullPath, result };
+        recordCall(spyFor(step.name), record);
+        recordCall(spyFor(fullPath), record);
+        connectorCalls.push({ step: step.name, action: fullPath, input });
+        return result;
+      } catch (err) {
+        const record = { args: [input], step: step.name, action: fullPath, error: String(err) };
+        recordCall(spyFor(step.name), record);
+        recordCall(spyFor(fullPath), record);
+        connectorCalls.push({ step: step.name, action: fullPath, input });
+        throw err;
+      }
+    });
+  }
+
+  // ---- secrets ----
+  const secretNames = Object.keys((def.secrets ?? {}) as Record<string, unknown>);
+  const secrets: Record<string, string> = {};
+  for (const name of secretNames) {
+    secrets[name] = opts.secrets?.[name] ?? `test-secret-${name}`;
+  }
+
+  // ---- occurrence counters (reset per handler invocation; here: single invocation) ----
+  const stepOccurrence = new Map<string, number>();
+  const signalOccurrence = new Map<string, number>();
+  const defaultRetry = resolveRetryPolicy(def.retry);
+
+  const modelTurns = new Map<string, number>();
+  const harnessInvocations = new Map<string, number>();
+
+  const ctx: Ctx<any, any, any, any> = {
+    async step<T>(name: string, fn: () => Promise<T> | T, stepOpts?: StepOpts<T>): Promise<T> {
+      if (typeof name !== "string" || name.length === 0 || typeof fn !== "function") {
+        throw new TestConfigError(`ctx.step: expected (name, fn) — got name=${JSON.stringify(name)}`);
+      }
+      const occ = (stepOccurrence.get(name) ?? 0) + 1;
+      stepOccurrence.set(name, occ);
+      const key = `step:${name}#${occ}`;
+      const cached = journalByKey.get(key);
+      if (cached?.status === "completed") {
+        const value = decodeJournal(cached.result as JsonValue) as T;
+        if (stepOpts?.undo) undoStack.push({ name, undo: () => stepOpts.undo!(value) });
+        return value;
+      }
+
+      const policy = resolveRetryPolicy(stepOpts?.retry, defaultRetry);
+      const timeoutMs = stepOpts?.timeout !== undefined ? parseDuration(stepOpts.timeout, "step timeout") : undefined;
+      const entry: JournalEntry = { kind: "step", name, occurrence: occ, status: "failed", attempts: 0 };
+      journal.push(entry);
+      journalByKey.set(key, entry);
+
+      let lastError: unknown;
+      for (let attempt = 1; attempt <= policy.maxAttempts; attempt++) {
+        entry.attempts = attempt;
+        const state: ActiveStep = { name, sawUnsafeWrite: false };
+        try {
+          let resultPromise = Promise.resolve(activeStep.run(state, () => fn()));
+          if (timeoutMs !== undefined) {
+            // Time-skipping: a step that *would* time out only does so if it actually
+            // hangs on something unmocked — we reject after the wall-clock budget.
+            resultPromise = Promise.race([
+              resultPromise,
+              new Promise<never>((_r, reject) =>
+                setTimeout(
+                  () => reject(new TestConfigError(`step "${name}" exceeded its ${stepOpts?.timeout} timeout`)),
+                  Math.min(timeoutMs, 5_000),
+                ).unref?.(),
+              ),
+            ]);
+          }
+          const result = await resultPromise;
+          const encoded = encodeJournal(result); // throws NotSerializableError → terminal
+          entry.status = "completed";
+          entry.result = encoded;
+          if (stepOpts?.undo) undoStack.push({ name, undo: () => stepOpts.undo!(result) });
+          return result;
+        } catch (err) {
+          lastError = err;
+          entry.error = serializeError(err);
+          if (isTerminalError(err)) break;
+          if (state.sawUnsafeWrite && stepOpts?.retry === undefined) {
+            // Derived retry-safety (ADR-0012): a non-idempotent write never silently
+            // double-fires. Explicit per-step retry opts override.
+            logger.warn(
+              `step "${name}" performed a non-retry-safe write and will not auto-retry — pass StepOpts.retry to opt in`,
+            );
+            break;
+          }
+          const delay = nextRetryDelayMs(policy, attempt, isRetryableError(err) ? err.retryAfterMs : undefined);
+          if (delay === null) break;
+          slept.push(`retry:${name}#${occ}@${attempt}`);
+          // time-skipping clock: no real wait
+        }
+      }
+      entry.status = "failed";
+      throw lastError;
+    },
+
+    connections: connections as never,
+    secrets: secrets as never,
+    operators: {} as never,
+    harnesses: {} as never,
+
+    async sleep(duration: string): Promise<void> {
+      parseDuration(duration, "ctx.sleep"); // validate eagerly, real engine persists it
+      slept.push(duration);
+    },
+
+    async waitForSignal<T>(name: string, sOpts: { schema: Schema<T>; timeout?: string }): Promise<T | null> {
+      const occ = (signalOccurrence.get(name) ?? 0) + 1;
+      signalOccurrence.set(name, occ);
+      const provided = opts.signals?.[name];
+      let value: unknown;
+      if (typeof provided === "function") value = (provided as (o: number) => unknown)(occ);
+      else if (Array.isArray(provided)) value = provided[occ - 1];
+      else value = provided;
+
+      const entry: JournalEntry = { kind: "signal", name, occurrence: occ, status: "completed", attempts: 1 };
+      journal.push(entry);
+      if (value === undefined) {
+        if (sOpts.timeout !== undefined) {
+          parseDuration(sOpts.timeout, "waitForSignal timeout");
+          entry.status = "timed-out";
+          return null; // clock skips straight to the timeout
+        }
+        throw new TestConfigError(
+          `run is suspended waiting for signal "${name}" (occurrence ${occ}) with no timeout`,
+          `provide signals: { "${name}": <payload> } in t.run(...)`,
+        );
+      }
+      const validated = await validateSchema(sOpts.schema, value, `signal "${name}" payload`);
+      entry.result = encodeJournal(validated as unknown);
+      return validated;
+    },
+
+    async emit<T>(event: EventDefinition<T>, payload: T): Promise<void> {
+      const validated = await validateSchema(event.schema, payload, `event "${event.name}" payload`);
+      emitted.push({ event: event.name, payload: validated });
+    },
+
+    ...(opts.request !== undefined || (def.trigger as { kind?: string }).kind === "webhook"
+      ? {
+          request:
+            opts.request ??
+            ({
+              headers: { "content-type": "application/json" },
+              query: {},
+              rawBody: new TextEncoder().encode(JSON.stringify(opts.input ?? {})),
+            } satisfies WebhookRequest),
+        }
+      : {}),
+
+    logger,
+    run: { id: `test_${Math.random().toString(36).slice(2, 10)}`, attempt: 1, automationId: def.id },
+  };
+
+  (ctx as { operators: unknown }).operators = buildOperators(def, ctx, async ({ operatorKey, modelKey, request }) => {
+    const key = `${operatorKey}:${modelKey}`;
+    const turn = (modelTurns.get(key) ?? 0) + 1;
+    modelTurns.set(key, turn);
+    const script = opts.models?.[operatorKey]?.[modelKey];
+    if (script === undefined) {
+      throw new TestConfigError(
+        `unscripted model turn for operator "${operatorKey}" model "${modelKey}" (turn ${turn})`,
+        `provide models: { ${operatorKey}: { ${modelKey}: [{ output: <typed output>, usage: { inputTokens, outputTokens } }] } }`,
+      );
+    }
+    const picked = Array.isArray(script) ? script[turn - 1] : script;
+    if (picked === undefined) {
+      throw new TestConfigError(`no scripted model response for operator "${operatorKey}" model "${modelKey}" turn ${turn}`);
+    }
+    const response = typeof picked === "function" ? await picked(request, { operatorKey, modelKey, turn }) : picked;
+    recordCall(spyFor(`operator.${operatorKey}.model`), { args: [request], result: response });
+    recordCall(spyFor(`operator.${operatorKey}.model.${turn}`), { args: [request], result: response });
+    return response;
+  });
+
+  (ctx as { harnesses: unknown }).harnesses = buildHarnesses(def, ctx, async ({ harnessKey, request }) => {
+    const step = activeStep.getStore();
+    if (!step) {
+      throw new TestConfigError(
+        `harness "${harnessKey}" ran outside ctx.step()`,
+        `Harnesses are durable Step runners — wrap the call: ctx.step("name", () => ctx.harnesses.${harnessKey}.run(...))`,
+      );
+    }
+    const invocation = (harnessInvocations.get(harnessKey) ?? 0) + 1;
+    harnessInvocations.set(harnessKey, invocation);
+    const script = opts.harnesses?.[harnessKey];
+    if (script === undefined) {
+      throw new TestConfigError(
+        `unscripted Harness run for "${harnessKey}"`,
+        `provide harnesses: { ${harnessKey}: { output: <typed output>, status: "completed", artifacts: [], adapter: "test" } }`,
+      );
+    }
+    const picked = Array.isArray(script) ? script[invocation - 1] : script;
+    if (picked === undefined) throw new TestConfigError(`no scripted Harness result for "${harnessKey}" invocation ${invocation}`);
+    const result = typeof picked === "function" ? await picked(request, { harnessKey, invocation }) : picked;
+    recordCall(spyFor(`harness.${harnessKey}`), { args: [request], step: step.name, result });
+    return result;
+  });
+
+  // ---- resolve + validate input ----
+  let input = opts.input;
+  const trigger = def.trigger as {
+    kind: string;
+    input?: Schema<unknown>;
+    event?: EventDefinition<unknown>;
+    connectorId?: string;
+    triggerId?: string;
+  };
+  let inputSchema: Schema<unknown> | undefined =
+    def.input ?? (trigger.kind === "webhook" ? trigger.input : trigger.kind === "event" ? trigger.event?.schema : undefined);
+
+  if (trigger.kind === "connector" && trigger.connectorId !== undefined) {
+    // A connector trigger's payload contract is its declaration's output schema; the
+    // connector may also ship an explicit `trigger:<id>` sample.
+    const connector = Object.values(
+      (def.connections ?? {}) as Record<string, ConnectorInstance<any, any>>,
+    ).find((c) => c.id === trigger.connectorId);
+    const decl = connector?.__connector.triggers?.[trigger.triggerId ?? ""];
+    if (decl) {
+      inputSchema = def.input ?? (decl.output as Schema<unknown>);
+      if (input === undefined) {
+        input = connector?.__connector.samples?.[`trigger:${trigger.triggerId}`];
+      }
+    }
+  }
+
+  if (input === undefined && inputSchema && trigger.kind !== "schedule") {
+    input = await sampleFromSchema(inputSchema);
+  }
+  if (inputSchema && input !== undefined) {
+    input = await validateSchema(inputSchema, input, `automation "${def.id}" input`);
+  }
+
+  const finish = (
+    status: "completed" | "failed",
+    result?: TOutput,
+    error?: TestRunResult["error"],
+  ): TestRunResult<TOutput> => {
+    const stepsByName: Record<string, unknown[]> = {};
+    for (const e of journal) {
+      if (e.kind !== "step" || e.status !== "completed") continue;
+      (stepsByName[e.name] ??= []).push(decodeJournal(e.result as JsonValue));
+    }
+    const steps: Record<string, unknown> = {};
+    for (const [name, results] of Object.entries(stepsByName)) {
+      steps[name] = results.length === 1 ? results[0] : results;
+    }
+    const failedEntry = journal.find((e) => e.kind === "step" && e.status === "failed");
+    return {
+      status,
+      ...(result !== undefined ? { result } : {}),
+      ...(error !== undefined ? { error } : {}),
+      steps,
+      journal,
+      calls,
+      emitted,
+      slept,
+      undone,
+      logs,
+      failure: () =>
+        status === "completed"
+          ? null
+          : {
+              automation: def.id,
+              status: "failed",
+              error: error as SerializedError,
+              ...(failedEntry
+                ? {
+                    failedStep: {
+                      name: failedEntry.name,
+                      occurrence: failedEntry.occurrence,
+                      attempts: failedEntry.attempts,
+                      error: failedEntry.error as SerializedError,
+                    },
+                  }
+                : {}),
+              steps: journal
+                .filter((e) => e.kind === "step")
+                .map((e) => ({ name: e.name, occurrence: e.occurrence, status: e.status, attempts: e.attempts })),
+              connectorCalls,
+              ...(failedEntry?.error?.terminal === false
+                ? {}
+                : failedEntry
+                  ? { suggestion: `step "${failedEntry.name}" failed terminally — fix the input or mark the error retryable` }
+                  : {}),
+            },
+    };
+  };
+
+  try {
+    let output: unknown = await def.run(input as never, ctx as never);
+    if (def.output) {
+      output = await validateSchema(def.output, output, `automation "${def.id}" output`);
+    }
+    return finish("completed", output as TOutput);
+  } catch (err) {
+    // Terminal failure → undo completed steps in reverse (ADR-0002), best-effort.
+    for (const item of [...undoStack].reverse()) {
+      try {
+        await item.undo();
+        undone.push(item.name);
+      } catch (undoErr) {
+        logger.error(`undo for step "${item.name}" failed: ${String(undoErr)}`);
+        undone.push(`${item.name} (failed)`);
+      }
+    }
+    const failedStep = journal.find((e) => e.kind === "step" && e.status === "failed");
+    return finish("failed", undefined, {
+      ...serializeError(err),
+      ...(failedStep ? { step: failedStep.name } : {}),
+    });
+  }
+}

package/src/index.ts

@@ -0,0 +1,48 @@
+// @devosurf/tesser-testing — the agent's fast pass/fail loop (ADR-0008). In-process engine with
+// journal-of-results semantics, time-skipping, auto-mocked connectors, machine-actionable
+// failures. Apache-2.0; never imports the AGPL server.
+
+import type { AutomationDef } from "@devosurf/tesser-sdk";
+import { executeAutomation, type RunOptions, type TestRunResult } from "./engine.js";
+
+export interface AutomationTest<TOutput> {
+  run(opts?: RunOptions): Promise<TestRunResult<TOutput>>;
+}
+
+/** Create a test handle for one automation. `run` executes it in-process on the
+ *  time-skipping clock with connectors mocked (explicitly via `mocks`/`connections`,
+ *  or auto-mocked from connector samples / output schemas). */
+export function createTest<TOutput>(opts: {
+  automation: AutomationDef<any, TOutput, any, any, any, any, any>;
+}): AutomationTest<TOutput> {
+  if (!opts?.automation?.id || typeof opts.automation.run !== "function") {
+    throw new TypeError("createTest: pass { automation } — the default export of an automation file");
+  }
+  return {
+    run: (runOpts?: RunOptions) => executeAutomation(opts.automation, runOpts ?? {}),
+  };
+}
+
+export {
+  executeAutomation,
+  TestConfigError,
+  type RunOptions,
+  type TestRunResult,
+  type JournalEntry,
+  type MachineFailure,
+  type SerializedError,
+  type StepMock,
+  type ScriptedModelTurn,
+  type ScriptedHarnessRun,
+} from "./engine.js";
+export { smokeTest, type SmokeOutcome } from "./smoke.js";
+export {
+  sampleFromSchema,
+  edgeCasesFromSchema,
+  sampleFromJsonSchema,
+  toJsonSchemaAsync,
+  type EdgeCase,
+} from "./sample.js";
+export { Cassette, hashInput, type CassetteEntry } from "./cassette.js";
+export { createSpy, type Spy, type CapturedCall } from "./spy.js";
+export { invokeAction, fakeFetch, type InvokeActionOptions } from "./invoke-action.js";