@mercury-fw/cli 0.29.4 → 0.31.0

package/src/app/commands.ts

@@ -5,12 +5,16 @@
  * and validated before any of this runs (`program.ts`); what runs a command is
  * injected (`AppDeps`), which is how the tests see the exact calls.
  */
-import { readFileSync } from "node:fs";
+import { chmodSync, cpSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
-import { join, resolve } from "node:path";
+import { join, relative, resolve } from "node:path";
 import { CATALOG, type CliCredentials } from "../catalog.ts";
 import { packCredentials, readServiceAccountKey, setEnvVar } from "./credentials.ts";
 import type { App } from "./find-app.ts";
+import { LOCAL_PACKS_DIR, packageNameOf, withLocalOverrides, withoutLocalOverrides } from "./local-packages.ts";
+import { findTests, loadTest } from "../e2e/load.ts";
+import { runE2e } from "../e2e/runner.ts";
+import { openReplSession } from "../e2e/session.ts";
 
 export type AppDeps = {
   /** Runs `argv` in `cwd` on the user's terminal (stdin, stdout, stderr) and returns its exit code. */
@@ -48,6 +52,16 @@ export const RESET_TARGETS = {
 } as const;
 export type ResetTarget = keyof typeof RESET_TARGETS;
 
+/** How long one e2e turn may take: a local model on a long, tool-heavy turn is slow. */
+const E2E_TURN_TIMEOUT_MS = 10 * 60_000;
+
+/** Runs `argv` in `cwd`, returning its exit code and its output (stdout and stderr together). */
+async function captureCode(argv: string[], cwd: string): Promise<{ code: number; output: string }> {
+  const proc = Bun.spawn(argv, { cwd, stdin: "ignore", stdout: "pipe", stderr: "pipe" });
+  const [stdout, stderr, code] = await Promise.all([new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited]);
+  return { code, output: stdout + stderr };
+}
+
 /** The compose calls that build and start the app; `recreate` restarts containers even when nothing changed. */
 function startCalls(noCache: boolean, recreate: boolean): string[][] {
   const up = [...COMPOSE, "up", "-d", ...(noCache ? [] : ["--build"]), ...(recreate ? ["--force-recreate"] : [])];
@@ -174,8 +188,77 @@ export function appCommands(app: App, deps: AppDeps) {
       deps.print(`Delete ${source} now, the env file holds the key. mfw start applies it to a running app.`);
       return 0;
     },
+    /** Makes the app install the packages in `from`'s tarballs (`bun pm
+     * pack`) instead of the registry's: copied into `.packs/` (which the
+     * image copies too), overridden in the manifest, then installed.
+     * Everything is checked before anything is written. */
+    localPackages: async (from: string) => {
+      const source = resolve(from);
+      const target = join(app.dir, LOCAL_PACKS_DIR);
+      if (source === target) throw new Error(`${source} is the app's own .packs/: give the folder the tarballs were packed into.`);
+      if (!existsSync(source)) throw new Error(`${source} doesn't exist.`);
+      const files = readdirSync(source).filter((f) => f.endsWith(".tgz")).sort();
+      if (files.length === 0) throw new Error(`No .tgz in ${source}: pack the packages there first (bun pm pack).`);
+      const packs = await Promise.all(files.map(async (file) => ({ name: await packageNameOf(join(source, file)), file })));
+      const manifest = withLocalOverrides(readManifest(), packs);
+      rmSync(target, { recursive: true, force: true });
+      mkdirSync(target);
+      for (const { file } of packs) cpSync(join(source, file), join(target, file));
+      writeManifest(manifest);
+      deps.print(`${packs.length} local packages in ${target}: ${packs.map((p) => p.name).join(", ")}.`);
+      return deps.run(["bun", "install"], { cwd: app.dir });
+    },
+    /** Runs e2e tests (`tests`, or the app's `e2e/*.e2e.ts`) against the app's
+     * REPL in its container, keeping the turns and checks in
+     * `e2e/results/<time>/`; returns 1 when a case didn't pass. */
+    e2e: async (tests: string[], { repeat }: { repeat?: number }) => {
+      const files = findTests(tests, { appDir: app.dir, cwd: process.cwd() });
+      const loaded = await Promise.all(files.map(async (file) => ({ file: relative(process.cwd(), file) || file, test: await loadTest(file) })));
+      const results = join(app.dir, "e2e", "results", new Date().toISOString().replace(/[:.]/g, "-"));
+      mkdirSync(results, { recursive: true });
+      // The container's user writes the dumps here; on a Linux host it isn't
+      // the folder's owner.
+      chmodSync(results, 0o777);
+      let sessions = 0;
+      return runE2e(loaded, repeat === undefined ? {} : { repeat }, {
+        openSession: async () =>
+          openReplSession({
+            argv: [...COMPOSE, "run", "--rm", "-T", "-v", `${results}:/e2e`, SERVICE, "bun", "run", "repl"],
+            cwd: app.dir,
+            hostDir: results,
+            replDir: "/e2e",
+            name: `run-${++sessions}`,
+            timeoutMs: E2E_TURN_TIMEOUT_MS,
+          }),
+        cli: (command) => captureCode([...COMPOSE, "run", "--rm", "-T", "--no-deps", SERVICE, "sh", "-c", command], app.dir),
+        appPackages: (readManifest() as { dependencies?: Record<string, string> }).dependencies ?? {},
+        print: deps.print,
+        now: Date.now,
+        writeReport: async (report) => {
+          writeFileSync(join(results, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
+          deps.print(`Turns and checks in ${results}`);
+        },
+      });
+    },
+    /** Undoes `localPackages`: the app installs from the registry again. */
+    localPackagesOff: async () => {
+      writeManifest(withoutLocalOverrides(readManifest()));
+      rmSync(join(app.dir, LOCAL_PACKS_DIR), { recursive: true, force: true });
+      deps.print("Local packages removed: installing from the registry.");
+      return deps.run(["bun", "install"], { cwd: app.dir });
+    },
   };
 
+  /** The app's manifest, as an object. */
+  function readManifest(): Record<string, unknown> {
+    return JSON.parse(readFileSync(join(app.dir, "package.json"), "utf-8")) as Record<string, unknown>;
+  }
+
+  /** Writes the app's manifest, two-space indented like the one `mfw create` writes. */
+  function writeManifest(manifest: Record<string, unknown>): void {
+    writeFileSync(join(app.dir, "package.json"), `${JSON.stringify(manifest, null, 2)}\n`);
+  }
+
   /** Stops `service`, runs `steps`, starts it again; stops at the first
    * failure and, once the service is stopped, says it's down and how to bring
    * it back. Returns the exit code. */

package/src/app/local-packages.ts

@@ -0,0 +1,51 @@
+/**
+ * What `mfw local-packages` writes: the app's `package.json` with `overrides`
+ * pointing packages at local tarballs (`bun pm pack`) copied into `.packs/`,
+ * which the image copies before `bun install`. Overrides, and not the
+ * dependencies themselves, because a packed package names its siblings by
+ * version, which the registry has too: only an override sends those
+ * transitive dependencies to the tarballs as well.
+ */
+
+/** The tarballs' folder inside the app, as the Dockerfile copies it. */
+export const LOCAL_PACKS_DIR = ".packs";
+
+/** The prefix of every override this command writes, how it tells them from the user's own. */
+const LOCAL_OVERRIDE = `file:./${LOCAL_PACKS_DIR}/`;
+
+type Manifest = Record<string, unknown> & { overrides?: Record<string, string> };
+
+/** The user's own overrides in `pkg`: every one this command didn't write. */
+function ownOverrides(pkg: Manifest): Record<string, string> {
+  return Object.fromEntries(Object.entries(pkg.overrides ?? {}).filter(([, spec]) => !spec.startsWith(LOCAL_OVERRIDE)));
+}
+
+/** `pkg` with each of `packs` (a package name and its tarball's file name in
+ * `.packs/`) as an override, in place of any left by an earlier run. */
+export function withLocalOverrides(pkg: Manifest, packs: Array<{ name: string; file: string }>): Manifest {
+  const local = Object.fromEntries(packs.map((p) => [p.name, `${LOCAL_OVERRIDE}${p.file}`]));
+  return { ...pkg, overrides: { ...ownOverrides(pkg), ...local } };
+}
+
+/** `pkg` without the overrides this command writes; without `overrides` at
+ * all when none of the user's own is left. */
+export function withoutLocalOverrides(pkg: Manifest): Manifest {
+  const { overrides: _, ...rest } = pkg;
+  const own = ownOverrides(pkg);
+  return Object.keys(own).length > 0 ? { ...rest, overrides: own } : rest;
+}
+
+/** The package name in a `bun pm pack` tarball, read from its
+ * `package/package.json`. */
+export async function packageNameOf(tarball: string): Promise<string> {
+  const proc = Bun.spawn(["tar", "-xOzf", tarball, "package/package.json"], { stdout: "pipe", stderr: "pipe" });
+  const [manifest, code] = await Promise.all([new Response(proc.stdout).text(), proc.exited]);
+  try {
+    if (code !== 0) throw new Error(`tar exited with ${code}`);
+    const name = (JSON.parse(manifest) as { name?: unknown }).name;
+    if (typeof name !== "string") throw new Error("no name in its package.json");
+    return name;
+  } catch (err) {
+    throw new Error(`${tarball} isn't a package tarball: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}

package/src/args.ts

@@ -15,7 +15,13 @@ export type CreateArgs = {
   role?: string;
   channels?: string[];
   plugins?: string[];
+  /** The repository's origin, taken as typed. */
+  gitRemote?: string;
   yes: boolean;
+  /** Run `bun install` after writing (`--no-install` turns it off). */
+  install: boolean;
+  /** Create the repository with a first commit (`--no-git` turns it off). */
+  git: boolean;
 };
 
 /** `create`'s options as commander hands them over. */
@@ -25,7 +31,10 @@ export type CreateOptions = {
   role?: string;
   channels?: string;
   plugins?: string;
+  gitRemote?: string;
   yes?: boolean;
+  install?: boolean;
+  git?: boolean;
 };
 
 /** A comma-separated list, trimmed, empty items dropped. */
@@ -37,11 +46,13 @@ const list = (value: string): string[] =>
 
 /** The answers in `folder` and `opts`, with only what was actually given. */
 export function toCreateArgs(folder: string, opts: CreateOptions): CreateArgs {
-  const args: CreateArgs = { dir: folder, yes: opts.yes ?? false };
+  const args: CreateArgs = { dir: folder, yes: opts.yes ?? false, install: opts.install ?? true, git: opts.git ?? true };
   if (opts.name !== undefined) args.name = opts.name;
   if (opts.assistantName !== undefined) args.assistantName = opts.assistantName;
   if (opts.role !== undefined) args.role = opts.role;
   if (opts.channels !== undefined) args.channels = list(opts.channels);
   if (opts.plugins !== undefined) args.plugins = list(opts.plugins);
+  // Blank is none, as in the wizard.
+  if (opts.gitRemote !== undefined && opts.gitRemote.trim() !== "") args.gitRemote = opts.gitRemote.trim();
   return args;
 }

package/src/e2e/define.ts

@@ -0,0 +1,71 @@
+/**
+ * The shape of an e2e test, as a test file imports it
+ * (`import { e2e } from "@mercury-fw/cli/e2e"`): the plugins and channels the
+ * app must have, and cases of turns sent to the app's real model through its
+ * REPL, each with checks on the calls it made and the answer it gave.
+ * `mfw e2e` runs them (`runner.ts`).
+ */
+import type { Call, TurnData } from "./dump.ts";
+
+export type { Call } from "./dump.ts";
+
+/** One turn as a check sees it: its calls, its answer, how long it took. */
+export type Turn = TurnData & { seconds: number };
+
+/** A case's run: every turn in order, and the last one. */
+export type Run = { turns: Turn[]; last: Turn };
+
+/** Runs `command` in the app's container (`sh -c`), outside the model:
+ * preparing data, reading what a turn changed, cleaning up. */
+export type Cli = (command: string) => Promise<{ code: number; output: string }>;
+
+/** What `before`, `after` and `check` can reach besides the run. */
+export type Context = { cli: Cli };
+
+/** The checks a case makes. Call helpers look at every turn's calls, answer
+ * helpers at the last answer; each records a named check and never throws,
+ * so a case reports every failure at once. */
+export type Expect = {
+  /** At least one call to `tool`, matching `match` when given. */
+  call(tool: string, match?: (call: Call) => boolean, label?: string): void;
+  /** Every call to `tool` matches `match` (and there is at least one). */
+  everyCall(tool: string, match: (call: Call) => boolean, label?: string): void;
+  /** No call failed or went without a result. */
+  noFailedCalls(label?: string): void;
+  /** The number of calls, to every tool or to `tool`, within the bounds. */
+  callCount(bounds: { min?: number; max?: number }, tool?: string, label?: string): void;
+  /** The answer contains `pattern`, or matches it. */
+  answer(pattern: string | RegExp, label?: string): void;
+  /** The answer doesn't contain `pattern`, or doesn't match it. */
+  answerNot(pattern: string | RegExp, label?: string): void;
+  /** Anything else. */
+  that(label: string, condition: boolean): void;
+};
+
+/** One case: the turns sent, in one REPL session, and the checks on them. */
+export type E2eCase = {
+  name: string;
+  /** A message, or a function of the turn before (a follow-up, a confirmation token). */
+  turns: Array<string | ((previous: Turn) => string)>;
+  /** How many times to run it (the model isn't deterministic); default 1. */
+  repeat?: number;
+  /** How many runs must pass; default every one. */
+  minPasses?: number;
+  before?: (ctx: Context) => unknown;
+  after?: (ctx: Context) => unknown;
+  check: (run: Run, expect: Expect, ctx: Context) => unknown;
+};
+
+/** An e2e test: what the app must have, and its cases. */
+export type E2eTest = {
+  /** Catalog ids of the tool plugins the app must have (`jira`, …). */
+  plugins?: string[];
+  /** Catalog ids of the channels the app must have (`http`, …). */
+  channels?: string[];
+  cases: E2eCase[];
+};
+
+/** Declares an e2e test: the identity, there for the types. */
+export function e2e(test: E2eTest): E2eTest {
+  return test;
+}

package/src/e2e/dump.ts

@@ -0,0 +1,48 @@
+/**
+ * A turn read out of the file the REPL's `/dump` writes: the AI SDK's step
+ * results for the last turn, whose `content` parts are the tool calls, their
+ * results (or errors) and the model's text. Read as data, not through the
+ * SDK's types: only the parts listed here matter.
+ */
+
+/** One tool call as a check sees it. `ok` is false when the call failed or
+ * never got a result; a result without an `ok` of its own counts as worked.
+ * `pending` is an irreversible command staged for confirmation, which worked
+ * (its output has the token) though it reports `ok: false`. */
+export type Call = { tool: string; input: unknown; output: unknown; ok: boolean; pending: boolean };
+
+/** What a turn did: its tool calls in order, and its final text. */
+export type TurnData = { calls: Call[]; answer: string };
+
+type Part = { type?: unknown; toolCallId?: unknown; toolName?: unknown; input?: unknown; output?: unknown; error?: unknown; text?: unknown };
+
+/** The calls and the answer in `dump`, the parsed content of a `/dump` file. */
+export function turnFromDump(dump: unknown): TurnData {
+  if (!Array.isArray(dump)) throw new Error("the file isn't a /dump of steps");
+  const parts: Part[][] = dump.map((step) =>
+    Array.isArray((step as { content?: unknown } | null)?.content) ? ((step as { content: Part[] }).content) : [],
+  );
+
+  const calls: Array<Call & { id: unknown }> = [];
+  for (const part of parts.flat()) {
+    if (part.type === "tool-call") {
+      calls.push({ id: part.toolCallId, tool: String(part.toolName), input: part.input, output: undefined, ok: false, pending: false });
+      continue;
+    }
+    const call = calls.find((c) => c.id === part.toolCallId);
+    if (call === undefined) continue;
+    if (part.type === "tool-result") {
+      call.output = part.output;
+      const result = part.output as { ok?: unknown; pendingConfirmation?: unknown } | null;
+      call.pending = result?.pendingConfirmation === true;
+      call.ok = call.pending || (typeof result?.ok === "boolean" ? result.ok : true);
+    } else if (part.type === "tool-error") {
+      call.output = { error: part.error };
+      call.ok = false;
+    }
+  }
+
+  const last = parts.at(-1) ?? [];
+  const answer = last.filter((p) => p.type === "text" && typeof p.text === "string").map((p) => p.text as string).join("");
+  return { calls: calls.map(({ id: _, ...call }) => call), answer };
+}

package/src/e2e/expect.ts

@@ -0,0 +1,59 @@
+/**
+ * The `expect` a case's `check` receives: helpers that record named checks
+ * against a run, passed or failed with a detail, and never throw.
+ */
+import type { Call } from "./dump.ts";
+import type { Expect, Run } from "./define.ts";
+
+/** One check's outcome; `detail` says what was found when it failed. */
+export type Check = { label: string; ok: boolean; detail?: string };
+
+/** A call, one line, for a failure's detail. */
+const describe = (c: Call) => `${c.tool} ${JSON.stringify(c.input)}`;
+
+/** `pattern`, as a label says it. */
+const said = (pattern: string | RegExp) => (typeof pattern === "string" ? `contains "${pattern}"` : `matches ${pattern}`);
+const saidNot = (pattern: string | RegExp) => (typeof pattern === "string" ? `doesn't contain "${pattern}"` : `doesn't match ${pattern}`);
+const found = (text: string, pattern: string | RegExp) => (typeof pattern === "string" ? text.includes(pattern) : pattern.test(text));
+
+/** The checks' bounds, as a label says them. */
+function boundsLabel({ min, max }: { min?: number; max?: number }, what: string): string {
+  if (min !== undefined && max !== undefined) return `${min} to ${max} ${what}`;
+  if (max !== undefined) return `at most ${max} ${what}`;
+  return `at least ${min ?? 0} ${what}`;
+}
+
+/** An `expect` bound to `run`, and the checks it recorded so far. */
+export function createExpect(run: Run): { expect: Expect; checks: () => Check[] } {
+  const checks: Check[] = [];
+  const record = (label: string, ok: boolean, detail?: string) =>
+    void checks.push(ok || detail === undefined ? { label, ok } : { label, ok, detail });
+  const calls = run.turns.flatMap((t) => t.calls);
+  const of = (tool: string) => calls.filter((c) => c.tool === tool);
+
+  const expect: Expect = {
+    call: (tool, match, label) => {
+      const ok = of(tool).some((c) => match?.(c) ?? true);
+      record(label ?? `a ${tool} call`, ok, `calls: ${calls.map((c) => c.tool).join(", ") || "none"}`);
+    },
+    everyCall: (tool, match, label) => {
+      const mine = of(tool);
+      const bad = mine.find((c) => !match(c));
+      record(label ?? `every ${tool} call matches`, mine.length > 0 && bad === undefined, bad ? describe(bad) : `no ${tool} call`);
+    },
+    noFailedCalls: (label) => {
+      const failed = calls.filter((c) => !c.ok);
+      record(label ?? "no failed calls", failed.length === 0, failed.map(describe).join("; "));
+    },
+    callCount: (bounds, tool, label) => {
+      const n = (tool === undefined ? calls : of(tool)).length;
+      const ok = n >= (bounds.min ?? 0) && n <= (bounds.max ?? Infinity);
+      const what = tool === undefined ? "calls" : `${tool} call${bounds.max === 1 || bounds.min === 1 ? "" : "s"}`;
+      record(label ?? boundsLabel(bounds, what), ok, String(n));
+    },
+    answer: (pattern, label) => record(label ?? `answer ${said(pattern)}`, found(run.last.answer, pattern), run.last.answer),
+    answerNot: (pattern, label) => record(label ?? `answer ${saidNot(pattern)}`, !found(run.last.answer, pattern), run.last.answer),
+    that: (label, condition) => record(label, condition),
+  };
+  return { expect, checks: () => [...checks] };
+}

package/src/e2e/load.ts

@@ -0,0 +1,40 @@
+/**
+ * Which e2e test files `mfw e2e` runs, and loading one: Bun imports the
+ * TypeScript file as it is, and its default export is checked to look like
+ * a test before anything starts.
+ */
+import { existsSync, readdirSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+import type { E2eTest } from "./define.ts";
+
+/** The files to run: `named` resolved from `cwd`, or, when none is named,
+ * the app's `e2e/*.e2e.ts` in name order. */
+export function findTests(named: string[], { appDir, cwd }: { appDir: string; cwd: string }): string[] {
+  if (named.length > 0) return named.map((f) => resolve(cwd, f));
+  const folder = join(appDir, "e2e");
+  const found = existsSync(folder) ? readdirSync(folder).filter((f) => f.endsWith(".e2e.ts")).sort() : [];
+  if (found.length === 0) throw new Error(`No tests: none named, and no *.e2e.ts in ${folder}`);
+  return found.map((f) => join(folder, f));
+}
+
+/** The test `file` exports as default; throws naming the file when it isn't one. */
+export async function loadTest(file: string): Promise<E2eTest> {
+  const loaded = ((await import(pathToFileURL(file).href)) as { default?: unknown }).default as Partial<E2eTest> | undefined;
+  if (loaded === undefined || loaded === null || !Array.isArray(loaded.cases)) {
+    throw new Error(`${file} doesn't export a test as default (export default e2e({ … }))`);
+  }
+  for (const [i, c] of loaded.cases.entries()) {
+    const which = `${file}: case ${i + 1} ("${c?.name ?? "?"}")`;
+    if (typeof c?.name !== "string") throw new Error(`${which} has no name`);
+    if (!Array.isArray(c.turns) || c.turns.length === 0) throw new Error(`${which} has no turns`);
+    if (typeof c.check !== "function") throw new Error(`${which} has no check`);
+    const whole = (n: unknown) => typeof n === "number" && Number.isInteger(n) && n >= 1;
+    if (c.repeat !== undefined && !whole(c.repeat)) throw new Error(`${which} repeat takes a positive whole number`);
+    if (c.minPasses !== undefined && !whole(c.minPasses)) throw new Error(`${which} minPasses takes a positive whole number`);
+    if (c.minPasses !== undefined && c.minPasses > (c.repeat ?? 1)) {
+      throw new Error(`${which} minPasses (${c.minPasses}) is more than repeat (${c.repeat ?? 1})`);
+    }
+  }
+  return loaded as E2eTest;
+}

package/src/e2e/runner.ts

@@ -0,0 +1,153 @@
+/**
+ * Runs e2e tests against an app: for each case (as many times as it
+ * repeats), a fresh REPL session gets the case's turns, the checks run on
+ * what each turn did, and every check is printed; the exit code says whether
+ * every case passed enough runs. The session, the container commands and the
+ * report's destination are injected (`session.ts` and `commands.ts` hold the
+ * real ones), so the tests drive the runner with scripted turns.
+ */
+import { CATALOG } from "../catalog.ts";
+import type { Context, E2eCase, E2eTest, Run, Turn } from "./define.ts";
+import { turnFromDump } from "./dump.ts";
+import { createExpect, type Check } from "./expect.ts";
+
+/** A REPL session in the app: `turn` sends one line and resolves with what
+ * `/dump` wrote for it and what the REPL printed meanwhile. */
+export type Session = {
+  turn: (line: string) => Promise<{ dump: unknown; output: string }>;
+  close: () => Promise<void>;
+};
+
+export type RunnerDeps = {
+  openSession: () => Promise<Session>;
+  cli: Context["cli"];
+  /** The app's dependencies, to check the test's plugins and channels against. */
+  appPackages: Record<string, string>;
+  print: (line: string) => void;
+  /** Milliseconds, for each turn's duration. */
+  now: () => number;
+  writeReport: (report: CaseReport[]) => Promise<void>;
+};
+
+/** One run of a case, as the report keeps it. */
+export type RunReport = { ok: boolean; turns: Turn[]; checks: Check[] };
+export type CaseReport = { file: string; case: string; passed: boolean; runs: RunReport[] };
+
+/** The prompt the REPL prints after a reply, with its context-usage suffix. */
+const PROMPT = /(\[[^\]\n]*\] )?> $/;
+
+/** A turn's printed output as an answer: no styling, no trailing prompt. */
+function answerFromOutput(output: string): string {
+  return output.replace(/\u001b\[[0-9;]*m/g, "").replace(PROMPT, "").trim();
+}
+
+/** The test's plugins and channels the app doesn't depend on, as a message, or undefined. */
+function missing(test: E2eTest, packages: Record<string, string>): string | undefined {
+  const wanted = [
+    ...(test.plugins ?? []).map((id) => ({ id, kind: "tool" as const, word: "plugin" })),
+    ...(test.channels ?? []).map((id) => ({ id, kind: "channel" as const, word: "channel" })),
+  ];
+  const lacking = wanted.flatMap(({ id, kind, word }) => {
+    const pkg = CATALOG.find((e) => e.kind === kind && e.id === id)?.package;
+    if (pkg === undefined) return [`${word} ${id} (not in the catalog)`];
+    return packages[pkg] === undefined ? [`${word} ${id} (${pkg})`] : [];
+  });
+  return lacking.length > 0 ? `the app lacks what the test needs: ${lacking.join(", ")}` : undefined;
+}
+
+/** Runs the tests in `tests` (each with the file it came from); returns the
+ * exit code: 0 when every case passed enough runs. `repeat` overrides each
+ * case's own. */
+export async function runE2e(
+  tests: Array<{ file: string; test: E2eTest }>,
+  opts: { repeat?: number },
+  deps: RunnerDeps,
+): Promise<number> {
+  const report: CaseReport[] = [];
+  let code = 0;
+  for (const { file, test } of tests) {
+    deps.print(file);
+    const lacking = missing(test, deps.appPackages);
+    if (lacking !== undefined) {
+      deps.print(`  ${lacking}`);
+      code = 1;
+      continue;
+    }
+    for (const c of test.cases) {
+      const result = await runCase(c, opts.repeat, deps);
+      report.push({ file, case: c.name, ...result });
+      if (!result.passed) code = 1;
+    }
+  }
+  await deps.writeReport(report);
+  return code;
+}
+
+/** Runs one case as many times as it repeats, printing each run's checks and the verdict. */
+async function runCase(c: E2eCase, repeatOverride: number | undefined, deps: RunnerDeps): Promise<{ passed: boolean; runs: RunReport[] }> {
+  const repeat = repeatOverride ?? c.repeat ?? 1;
+  const minPasses = Math.min(c.minPasses ?? repeat, repeat);
+  deps.print(`  ${c.name}`);
+  const runs: RunReport[] = [];
+  for (let i = 1; i <= repeat; i++) {
+    if (repeat > 1) deps.print(`    run ${i}/${repeat}`);
+    const run = await runOnce(c, deps);
+    const indent = repeat > 1 ? "      " : "    ";
+    for (const check of run.checks) {
+      deps.print(`${indent}${check.ok ? "✓" : "✗"} ${check.label}${check.ok || check.detail === undefined ? "" : `: ${check.detail}`}`);
+    }
+    runs.push(run);
+  }
+  const passes = runs.filter((r) => r.ok).length;
+  const passed = passes >= minPasses;
+  deps.print(
+    repeat > 1
+      ? `  ${c.name}: ${passes}/${repeat} runs passed (needs ${minPasses})${passed ? "" : " FAILED"}`
+      : `  ${c.name}: ${passed ? "passed" : "FAILED"}`,
+  );
+  return { passed, runs };
+}
+
+/** One run of a case in a fresh session: `before`, the turns, the checks, `after`. */
+async function runOnce(c: E2eCase, deps: RunnerDeps): Promise<RunReport> {
+  const ctx: Context = { cli: deps.cli };
+  const turns: Turn[] = [];
+  let checks: Check[] = [];
+  try {
+    await c.before?.(ctx);
+    const session = await deps.openSession();
+    try {
+      for (const next of c.turns) {
+        const line = typeof next === "string" ? next : next(turns.at(-1)!);
+        if (line.includes("\n")) throw new Error("a turn must be one line (the REPL reads one line per turn)");
+        const start = deps.now();
+        const { dump, output } = await session.turn(line);
+        const seconds = (deps.now() - start) / 1000;
+        const data = turnFromDump(dump);
+        const answer = data.calls.length === 0 && data.answer === "" ? answerFromOutput(output) : data.answer;
+        turns.push({ calls: data.calls, answer, seconds });
+      }
+    } finally {
+      await session.close();
+    }
+    const run: Run = { turns, last: turns.at(-1)! };
+    const recorder = createExpect(run);
+    try {
+      await c.check(run, recorder.expect, ctx);
+      checks = recorder.checks();
+      // A case that checks nothing proves nothing.
+      if (checks.length === 0) checks = [{ label: "the case made no checks", ok: false }];
+    } catch (err) {
+      checks = [...recorder.checks(), { label: `check threw: ${err instanceof Error ? err.message : String(err)}`, ok: false }];
+    }
+  } catch (err) {
+    checks = [{ label: `run failed: ${err instanceof Error ? err.message : String(err)}`, ok: false }];
+  } finally {
+    try {
+      await c.after?.(ctx);
+    } catch (err) {
+      checks = [...checks, { label: `after threw: ${err instanceof Error ? err.message : String(err)}`, ok: false }];
+    }
+  }
+  return { ok: checks.every((ch) => ch.ok), turns, checks };
+}