debutly 0.0.0-stage → 0.2.0

package/src/import.ts

@@ -0,0 +1,108 @@
+// Load finished runs into the store for the dashboard.
+//
+//   npm run debutly:import -- --store <url> --dir runs/2026-10-02 --app acme [--at 2026-10-02T10:02Z]
+//   npm run debutly:import -- --store <url> --github acme/app --app acme [--since 2026-09-28]
+//
+// --github reads that repo's Debutly workflow runs through the `gh` CLI, downloads each
+// one's report artifact and imports it; runs already in the store (same CI page) are skipped.
+import { execFile } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { parseArgs, promisify } from "node:util";
+import { eq } from "drizzle-orm";
+import { chromium, type Browser } from "playwright";
+import { openDb } from "./store/db";
+import { importRunFolder } from "./store/import-run";
+import { runs } from "./store/schema";
+
+const run = promisify(execFile);
+const TRIGGERS: Record<string, string> = { schedule: "nightly", pull_request: "pr", workflow_dispatch: "manual" };
+// The report artifact's name; runs from before 2026-10-03, when Debutly was Wind Tunnel, use the second.
+const REPORT_ARTIFACTS = ["debutly-report", "wind-tunnel-report"];
+
+/** Downloads a CI run's report into dir; false when the run has none. */
+async function downloadReport(repo: string, runId: number, dir: string): Promise<boolean> {
+  for (const name of REPORT_ARTIFACTS) {
+    try {
+      await run("gh", ["run", "download", String(runId), "--repo", repo, "-n", name, "-D", dir], { maxBuffer: 1 << 24 });
+      return true;
+    } catch {
+      // No artifact by this name; try the next.
+    }
+  }
+  return false;
+}
+
+const { values } = parseArgs({
+  options: {
+    store: { type: "string" },
+    dir: { type: "string" },
+    github: { type: "string" },
+    workflow: { type: "string", default: "throwaway-env.yml" },
+    app: { type: "string" },
+    at: { type: "string" },
+    "ci-url": { type: "string" },
+    trigger: { type: "string" },
+    since: { type: "string" },
+    limit: { type: "string", default: "30" },
+  },
+});
+
+if (!values.store || (!values.dir && !values.github)) {
+  console.error("Usage: npm run debutly:import -- --store <url> (--dir <run folder> | --github owner/repo) [--app name] [--since YYYY-MM-DD]");
+  process.exit(1);
+}
+
+// Screenshots go in as compact JPEGs, converted by drawing each PNG in a browser.
+async function jpegConverter(browser: Browser) {
+  const page = await browser.newPage({ viewport: { width: 1600, height: 1200 } });
+  return async (png: Buffer) => {
+    await page.setContent(`<style>body{margin:0}</style><img src="data:image/png;base64,${png.toString("base64")}">`);
+    const img = page.locator("img");
+    await img.evaluate((el: HTMLImageElement) => el.decode());
+    return img.screenshot({ type: "jpeg", quality: 50 });
+  };
+}
+
+const { db, close } = await openDb(values.store);
+const browser = await chromium.launch();
+try {
+  const toJpeg = await jpegConverter(browser);
+  if (values.dir) {
+    const id = await importRunFolder(db, values.dir, {
+      app: values.app,
+      trigger: values.trigger,
+      ciUrl: values["ci-url"],
+      at: values.at ? new Date(values.at) : undefined,
+      toJpeg,
+    });
+    console.log(`Imported ${values.dir} as run ${id}`);
+  } else {
+    const repo = values.github!;
+    const app = values.app ?? repo.split("/").pop()!;
+    const { stdout } = await run("gh", ["run", "list", "--repo", repo, "--workflow", values.workflow!, "--status", "success", "--limit", values.limit!, "--json", "databaseId,event,createdAt,url"], { maxBuffer: 1 << 24 });
+    const listed = (JSON.parse(stdout) as { databaseId: number; event: string; createdAt: string; url: string }[])
+      .filter((r) => !values.since || r.createdAt >= values.since)
+      .reverse();
+    for (const r of listed) {
+      const [seen] = await db.select({ id: runs.id }).from(runs).where(eq(runs.ciUrl, r.url));
+      if (seen) continue;
+      const dir = await mkdtemp(join(tmpdir(), "debutly-import-"));
+      // Runs that skipped the agents (e.g. pull requests without the run-agents label) have no report.
+      if (!(await downloadReport(repo, r.databaseId, dir))) {
+        await rm(dir, { recursive: true, force: true });
+        continue;
+      }
+      try {
+        const id = await importRunFolder(db, dir, { app, trigger: TRIGGERS[r.event] ?? r.event, ciUrl: r.url, at: new Date(r.createdAt), toJpeg });
+        console.log(`Imported ${repo} ${r.createdAt} (${r.event}) as run ${id}`);
+      } finally {
+        await rm(dir, { recursive: true, force: true });
+      }
+    }
+  }
+} finally {
+  await browser.close();
+  await close();
+}

package/src/inbox/mailpit.ts

@@ -0,0 +1,46 @@
+import type { EmailMessage, Inbox } from "./types";
+
+const MAX_MESSAGES = 3;
+
+const decodeEntities = (s: string) => s.replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'");
+
+function extractLinks(text: string, html: string): string[] {
+  const fromText = text.match(/https?:\/\/[^\s<>"')]+/g) ?? [];
+  const fromHtml = [...html.matchAll(/href\s*=\s*["']([^"']+)["']/gi)].map((m) => m[1]);
+  return [...new Set([...fromText, ...fromHtml].map(decodeEntities).filter((l) => /^https?:\/\//.test(l)))];
+}
+
+/** Reads the local Mailpit mail catcher that `supabase start` runs, used as the agents' inboxes. */
+export class MailpitInbox implements Inbox {
+  private readonly base: string;
+  private readonly headers: Record<string, string> = {};
+
+  /** A mailbox with a login is given as https://user:password@host; fetch won't take that, so it becomes a header. */
+  constructor(
+    baseUrl: string,
+    private readonly fetchFn: typeof fetch = fetch,
+  ) {
+    const url = new URL(baseUrl);
+    if (url.username) {
+      const login = `${decodeURIComponent(url.username)}:${decodeURIComponent(url.password)}`;
+      this.headers.authorization = `Basic ${Buffer.from(login).toString("base64")}`;
+      url.username = "";
+      url.password = "";
+    }
+    this.base = url.href.replace(/\/+$/, "");
+  }
+
+  async latest(address: string): Promise<EmailMessage[]> {
+    const search = await this.fetchFn(`${this.base}/api/v1/search?query=${encodeURIComponent(`to:"${address}"`)}`, { headers: this.headers });
+    if (!search.ok) throw new Error(`mailbox search failed: HTTP ${search.status}`);
+    const { messages = [] } = (await search.json()) as { messages?: { ID: string }[] };
+    const out: EmailMessage[] = [];
+    for (const { ID } of messages.slice(0, MAX_MESSAGES)) {
+      const res = await this.fetchFn(`${this.base}/api/v1/message/${ID}`, { headers: this.headers });
+      if (!res.ok) continue;
+      const m = (await res.json()) as { Subject?: string; Text?: string; HTML?: string };
+      out.push({ subject: m.Subject ?? "", text: (m.Text ?? "").slice(0, 1500), links: extractLinks(m.Text ?? "", m.HTML ?? "") });
+    }
+    return out;
+  }
+}

package/src/inbox/types.ts

@@ -0,0 +1,10 @@
+export interface EmailMessage {
+  subject: string;
+  text: string;
+  links: string[];
+}
+
+export interface Inbox {
+  /** Most recent messages for an address, newest first. */
+  latest(address: string): Promise<EmailMessage[]>;
+}

package/src/init.ts

@@ -0,0 +1,184 @@
+// `debutly init`: a few questions, then a debutly.json (and, if wanted, a weekly GitHub workflow)
+// so that `npx debutly` alone runs the agents.
+import { existsSync, mkdirSync, readFileSync, writeFileSync, appendFileSync } from "node:fs";
+import { join } from "node:path";
+import { createInterface } from "node:readline/promises";
+import { CONFIG_FILE, type DebutlyConfig } from "./config";
+import { looksLikeProduction } from "./doctor";
+
+export const GOALS = [
+  "Sign up and get to the main screen",
+  "Sign up, choose a plan and pay (test mode)",
+  "Sign up and invite a teammate",
+] as const;
+
+/** Customers who invite teammates are teams: an owner signs up, teammates accept by email. */
+export const teamGoal = (goal: string) => /\b(invite|teammate|team|colleague|workspace|seat)s?\b/i.test(goal);
+
+export interface Answers {
+  url: string;
+  goal: string;
+  icp: string;
+  team: boolean;
+  mailpit?: string;
+}
+
+export function buildConfig(a: Answers): DebutlyConfig {
+  return {
+    $schema: "https://debutly.dev/schema.json",
+    url: a.url,
+    goal: a.goal,
+    icp: a.icp,
+    ...(a.team ? { unit: "business", employees: 2 } : {}),
+    tier: "quick",
+    maxCost: 1,
+    ...(a.mailpit ? { mailpit: a.mailpit } : {}),
+  };
+}
+
+/** How the workflow starts the app, guessed from its package.json scripts. */
+export function startCommand(scripts: Record<string, string> | undefined, url: string): string[] {
+  const port = new URL(url).port || "3000";
+  if (scripts?.build && scripts?.start) return ["npm ci", "npm run build", `PORT=${port} npm start &`];
+  if (scripts?.dev) return ["npm ci", `PORT=${port} npm run dev &`];
+  return ["# Start a test copy of your app here"];
+}
+
+export function workflowYaml(start: string[]): string {
+  return [
+    "# Debutly's AI agents use a test copy of your app like new customers, every Monday.",
+    "# Settings are in debutly.json. Add your model key as the repository secret DEBUTLY_API_KEY.",
+    "name: debutly",
+    "on:",
+    '  schedule: [{ cron: "0 4 * * 1" }]',
+    "  workflow_dispatch:",
+    "jobs:",
+    "  agents:",
+    "    runs-on: ubuntu-latest",
+    "    timeout-minutes: 45",
+    "    steps:",
+    "      - uses: actions/checkout@v4",
+    "      # Starts your app for the agents. Adjust it if the app needs a database or test keys.",
+    "      - run: |",
+    ...start.map((line) => `          ${line}`),
+    "      - uses: sroay/debutly@v0",
+    "        with:",
+    "          api-key: ${{ secrets.DEBUTLY_API_KEY }}",
+    "",
+  ].join("\n");
+}
+
+/** Adds the lines .gitignore lacks: the key file and Debutly's local reports. */
+export function ignoreLines(current: string): string[] {
+  const have = new Set(current.split(/\r?\n/).map((l) => l.trim()));
+  return [".env", ".debutly/"].filter((l) => !have.has(l) && !have.has(`/${l}`));
+}
+
+const visibleText = (html: string) =>
+  html
+    .replace(/<(script|style|noscript)[\s\S]*?<\/\1>/gi, " ")
+    .replace(/<[^>]+>/g, " ")
+    .replace(/&[a-z#0-9]+;/gi, " ")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 4000);
+
+/** One sentence on who the app is for, read off its home page by the judge model. */
+async function guessCustomers(url: string): Promise<string | undefined> {
+  if (!process.env.DEBUTLY_API_KEY) return undefined;
+  try {
+    const res = await fetch(url, { signal: AbortSignal.timeout(10_000) });
+    const text = visibleText(await res.text());
+    if (text.length < 80) return undefined;
+    const { createLlms } = await import("./llm/factory");
+    const { judge } = createLlms(process.env);
+    const reply = await judge.complete({
+      system: "You describe a product's ideal customers in one plain sentence, under 20 words, like: Owners of small marketing agencies, 2 to 15 people.",
+      messages: [{ role: "user", content: `Who are the ideal customers of this product? Its home page says:\n\n${text}` }],
+      maxTokens: 200,
+    });
+    return reply.replace(/\s+/g, " ").trim().replace(/^["']|["']$/g, "") || undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export async function init(cwd: string = process.cwd(), input: NodeJS.ReadableStream & { isTTY?: boolean } = process.stdin): Promise<number> {
+  const rl = createInterface({ input, output: process.stdout });
+  // Lines are buffered, so answers piped in by a script aren't lost between questions; at the end
+  // of the input every question takes its default.
+  const lines = rl[Symbol.asyncIterator]();
+  const ask = async (question: string, fallback = "") => {
+    process.stdout.write(fallback ? `${question} [${fallback}] ` : `${question} `);
+    const { value, done } = await lines.next();
+    if (done || !input.isTTY) process.stdout.write("\n");
+    return (done ? "" : String(value).trim()) || fallback;
+  };
+  try {
+    console.log("Debutly sends AI agents through a test copy of your app, like your first customers.\n");
+    if (existsSync(join(cwd, CONFIG_FILE)) && !/^y/i.test(await ask(`${CONFIG_FILE} exists. Replace it? (y/N)`, "N"))) return 0;
+
+    const url = await ask("Where is the test copy of your app running?", "http://localhost:3000");
+    if (looksLikeProduction(url)) {
+      console.log("  Careful: agents create accounts, submit forms and pay in test mode. Use a local, staging or throwaway copy, never production.");
+    }
+
+    console.log("\nWhat should the agents try to do?");
+    GOALS.forEach((g, i) => console.log(`  ${i + 1}. ${g}`));
+    console.log("  4. Something else");
+    // A goal typed here instead of a number is the goal.
+    const pick = await ask("Choose 1-4, or type the goal:", "1");
+    const goal = GOALS[Number(pick) - 1] ?? (/^\d*$/.test(pick) ? await ask("Describe it in a sentence:") : pick);
+
+    const guess = await guessCustomers(url);
+    const icp = await ask("\nWho are your customers? One sentence, e.g. who they are and how big.", guess);
+    if (!icp) {
+      console.log("Debutly needs to know who the customers are; run init again when you know.");
+      return 1;
+    }
+    const team = /^y/i.test(await ask("Do they use it as a team, inviting teammates? (y/N)", teamGoal(goal) ? "y" : "N"));
+    const mailpit =
+      (await ask(
+        "If sign-up sends a confirmation email: the address of a local mail catcher (Mailpit, e.g. from `supabase start`), or Enter to skip.",
+        team ? "http://127.0.0.1:54324" : "",
+      )) || undefined;
+
+    const config = buildConfig({ url, goal, icp, team, mailpit });
+    writeFileSync(join(cwd, CONFIG_FILE), `${JSON.stringify(config, null, 2)}\n`);
+    console.log(`\nWrote ${CONFIG_FILE}.`);
+
+    const gitignore = join(cwd, ".gitignore");
+    if (existsSync(join(cwd, ".git")) || existsSync(gitignore)) {
+      const add = ignoreLines(existsSync(gitignore) ? readFileSync(gitignore, "utf8") : "");
+      if (add.length) {
+        appendFileSync(gitignore, `\n# Debutly: your model key, and local reports\n${add.join("\n")}\n`);
+        console.log(`Added ${add.join(" and ")} to .gitignore.`);
+      }
+    }
+
+    if (/^y/i.test(await ask("\nAdd a GitHub Actions workflow that runs the agents every Monday? (y/N)", "N"))) {
+      const path = join(cwd, ".github", "workflows", "debutly.yml");
+      if (existsSync(path)) console.log(".github/workflows/debutly.yml exists; left as it is.");
+      else {
+        let scripts: Record<string, string> | undefined;
+        try {
+          scripts = JSON.parse(readFileSync(join(cwd, "package.json"), "utf8")).scripts;
+        } catch {
+          // not a Node app, or no package.json: the workflow says where to start the app
+        }
+        mkdirSync(join(cwd, ".github", "workflows"), { recursive: true });
+        writeFileSync(path, workflowYaml(startCommand(scripts, url)));
+        console.log("Wrote .github/workflows/debutly.yml. Add the repository secret DEBUTLY_API_KEY (Settings → Secrets and variables → Actions).");
+      }
+    }
+
+    console.log(
+      process.env.DEBUTLY_API_KEY
+        ? "\nNext: npx debutly doctor, then npx debutly"
+        : "\nNext: create a key at https://openrouter.ai/keys, put DEBUTLY_API_KEY=… in .env here, then run npx debutly doctor and npx debutly",
+    );
+    return 0;
+  } finally {
+    rl.close();
+  }
+}

package/src/llm/anthropic.ts

@@ -0,0 +1,41 @@
+import Anthropic from "@anthropic-ai/sdk";
+import { OutOfCreditsError, type LlmClient, type LlmRequest } from "./types";
+
+export type MessagesClient = Pick<Anthropic, "messages">;
+
+const LOW_BALANCE = /credit balance is too low/i;
+
+export class AnthropicLlm implements LlmClient {
+  private outOfCredits?: OutOfCreditsError;
+
+  constructor(
+    private readonly client: MessagesClient,
+    readonly model: string,
+  ) {}
+
+  static fromApiKey(apiKey: string, model: string): AnthropicLlm {
+    return new AnthropicLlm(new Anthropic({ apiKey }), model);
+  }
+
+  async complete(req: LlmRequest): Promise<string> {
+    if (this.outOfCredits) throw this.outOfCredits;
+    let response: Anthropic.Message;
+    try {
+      response = await this.client.messages.create({
+        model: this.model,
+        max_tokens: req.maxTokens ?? 16000,
+        system: req.system,
+        messages: req.messages,
+      });
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      if (LOW_BALANCE.test(message)) throw (this.outOfCredits = new OutOfCreditsError(message));
+      throw err;
+    }
+    if (response.usage) req.onUsage?.({ inputTokens: response.usage.input_tokens, outputTokens: response.usage.output_tokens });
+    if (response.stop_reason === "refusal") {
+      throw new Error("LLM refused the request");
+    }
+    return response.content.flatMap((block) => (block.type === "text" ? [block.text] : [])).join("");
+  }
+}

package/src/llm/credits.ts

@@ -0,0 +1,35 @@
+import type { CostEstimate } from "../cost/estimate";
+import { isOpenRouter } from "./openrouter";
+
+/**
+ * What an OpenRouter key may still spend, in USD, from GET /key. Null when the key has no limit,
+ * the provider isn't OpenRouter, or the lookup fails: the account's own balance needs a
+ * management key, so a key limit is the only balance an ordinary key can see.
+ */
+export async function keyCreditsLeft(baseUrl: string, apiKey: string, fetchFn: typeof fetch = fetch): Promise<number | null> {
+  if (!isOpenRouter(baseUrl)) return null;
+  try {
+    const res = await fetchFn(`${baseUrl.replace(/\/+$/, "")}/key`, { headers: { authorization: `Bearer ${apiKey}` } });
+    if (!res.ok) return null;
+    const { data } = (await res.json()) as { data?: { limit_remaining?: number | null } };
+    return typeof data?.limit_remaining === "number" ? data.limit_remaining : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Before a run: refuse one the key can't pay for even at the low estimate (agents would stop
+ * part-way, 2026-10-21), and warn when the high estimate is more than is left.
+ */
+export function creditCheck(left: number | null, estimate: CostEstimate | null): { refuse?: string; warn?: string } {
+  if (left === null || !estimate) return {};
+  const usd = (n: number) => `$${n.toFixed(2)}`;
+  if (left < estimate.low) {
+    return { refuse: `the API key has ${usd(left)} left and this run needs at least ${usd(estimate.low)}. Top up, or raise the key's limit, then run again.` };
+  }
+  if (left < estimate.high) {
+    return { warn: `the API key has ${usd(left)} left and this run could cost up to ${usd(estimate.high)}; agents may stop part-way.` };
+  }
+  return {};
+}

package/src/llm/factory.ts

@@ -0,0 +1,65 @@
+import { AnthropicLlm } from "./anthropic";
+import { OpenAiCompatibleLlm } from "./openai-compatible";
+import { isOpenRouter } from "./openrouter";
+import type { LlmClient } from "./types";
+
+export interface LlmPair {
+  nav: LlmClient;
+  judge: LlmClient;
+}
+
+/** DEBUTLY_NAV_ROUTING / DEBUTLY_JUDGE_ROUTING: OpenRouter provider preferences as JSON, e.g. {"order":["google-ai-studio/flex"]}. */
+export function parseRouting(json: string | undefined, name = "DEBUTLY_NAV_ROUTING"): Record<string, unknown> | undefined {
+  if (!json?.trim()) return undefined;
+  let value: unknown;
+  try {
+    value = JSON.parse(json);
+  } catch {
+    throw new Error(`${name} is not valid JSON: ${json}`);
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`${name} must be a JSON object`);
+  return value as Record<string, unknown>;
+}
+
+// Agents send the pages they read to the model provider. On OpenRouter, Debutly asks only for
+// providers that neither keep prompts (zero data retention) nor use them for training. The
+// recommended models are served that way at their usual price (checked 2026-10-23).
+// DEBUTLY_ZERO_RETENTION=off allows any provider.
+const NO_RETENTION = { zdr: true, data_collection: "deny" } as const;
+
+export function routing(env: Record<string, string | undefined>, baseUrl: string, own: Record<string, unknown> | undefined): Record<string, unknown> | undefined {
+  const zeroRetention = isOpenRouter(baseUrl) && !/^(off|false|0|no)$/i.test(env.DEBUTLY_ZERO_RETENTION ?? "");
+  if (!zeroRetention) return own;
+  return { ...NO_RETENTION, ...own };
+}
+
+export function createLlms(env: Record<string, string | undefined>): LlmPair {
+  const apiKey = env.DEBUTLY_API_KEY;
+  if (!apiKey) throw new Error("DEBUTLY_API_KEY is required (bring your own key)");
+  const provider = env.DEBUTLY_PROVIDER ?? "anthropic";
+
+  if (provider === "anthropic") {
+    return {
+      nav: AnthropicLlm.fromApiKey(apiKey, env.DEBUTLY_NAV_MODEL ?? "claude-haiku-4-5"),
+      judge: AnthropicLlm.fromApiKey(apiKey, env.DEBUTLY_JUDGE_MODEL ?? "claude-opus-5"),
+    };
+  }
+
+  if (provider === "openai-compatible") {
+    const baseUrl = env.DEBUTLY_BASE_URL;
+    const navModel = env.DEBUTLY_NAV_MODEL;
+    const judgeModel = env.DEBUTLY_JUDGE_MODEL;
+    if (!baseUrl || !navModel || !judgeModel) {
+      throw new Error("openai-compatible requires DEBUTLY_BASE_URL, DEBUTLY_NAV_MODEL and DEBUTLY_JUDGE_MODEL");
+    }
+    const reasoningEffort = env.DEBUTLY_REASONING_EFFORT || undefined;
+    const navProvider = routing(env, baseUrl, parseRouting(env.DEBUTLY_NAV_ROUTING));
+    const judgeProvider = routing(env, baseUrl, parseRouting(env.DEBUTLY_JUDGE_ROUTING, "DEBUTLY_JUDGE_ROUTING"));
+    return {
+      nav: new OpenAiCompatibleLlm({ baseUrl, apiKey, model: navModel, reasoningEffort, provider: navProvider }),
+      judge: new OpenAiCompatibleLlm({ baseUrl, apiKey, model: judgeModel, provider: judgeProvider }),
+    };
+  }
+
+  throw new Error(`Unknown DEBUTLY_PROVIDER "${provider}"`);
+}

package/src/llm/meter.ts

@@ -0,0 +1,39 @@
+import type { LlmUsage } from "./types";
+
+/** USD per million tokens. */
+export interface ModelPrice {
+  inputPerMTok: number;
+  outputPerMTok: number;
+}
+
+/** Running token count for a session or a run. */
+export class Meter {
+  private inputTokens = 0;
+  private outputTokens = 0;
+  private requests = 0;
+  private charged = 0;
+  private unchargedRequests = 0;
+
+  add = (u: LlmUsage): void => {
+    this.inputTokens += u.inputTokens;
+    this.outputTokens += u.outputTokens;
+    this.requests++;
+    if (u.costUsd === undefined) this.unchargedRequests++;
+    else this.charged += u.costUsd;
+  };
+
+  get total(): LlmUsage {
+    return { inputTokens: this.inputTokens, outputTokens: this.outputTokens };
+  }
+
+  /**
+   * What the provider charged when it reported every request's charge: that reflects caching,
+   * the provider tier (e.g. Gemini flex) and routing. Otherwise an estimate from the list price;
+   * undefined when that is unknown too.
+   */
+  costUsd(price: ModelPrice | null | undefined): number | undefined {
+    if (this.requests > 0 && this.unchargedRequests === 0) return this.charged;
+    if (!price) return undefined;
+    return (this.inputTokens * price.inputPerMTok + this.outputTokens * price.outputPerMTok) / 1_000_000;
+  }
+}

package/src/llm/openai-compatible.ts

@@ -0,0 +1,90 @@
+import { isOpenRouter } from "./openrouter";
+import { OutOfCreditsError, type LlmClient, type LlmRequest } from "./types";
+
+export interface OpenAiCompatibleOptions {
+  baseUrl: string;
+  apiKey: string;
+  model: string;
+  fetchFn?: typeof fetch;
+  sleep?: (ms: number) => Promise<void>;
+  /** OpenRouter's unified reasoning control. Clicking through pages rarely needs deep thinking. */
+  reasoningEffort?: string;
+  /**
+   * OpenRouter provider routing, passed through: e.g. {"order":["google-ai-studio/flex"]} for
+   * Gemini's half-price tier, or {"sort":"throughput"} for the fastest host of an open model.
+   */
+  provider?: Record<string, unknown>;
+}
+
+const MAX_ATTEMPTS = 4;
+const MAX_WAIT_MS = 30_000;
+const isRetryable = (status: number) => status === 429 || status >= 500;
+// OpenRouter holds back credit for a reply of max_tokens. On a low balance it refuses with 402 and
+// says what it can still cover ("You requested up to 16000 tokens, but can only afford 7001"); a
+// reply that size is still plenty for most requests (an agent's is ~150 tokens), so ask again.
+const AFFORDABLE = /can only afford (\d+)/;
+const MIN_AFFORDABLE_TOKENS = 1024;
+
+export class OpenAiCompatibleLlm implements LlmClient {
+  private outOfCredits?: OutOfCreditsError;
+
+  constructor(private readonly opts: OpenAiCompatibleOptions) {}
+
+  get reasoningEffort(): string | undefined {
+    return this.opts.reasoningEffort;
+  }
+
+  async complete(req: LlmRequest): Promise<string> {
+    if (this.outOfCredits) throw this.outOfCredits;
+    const fetchFn = this.opts.fetchFn ?? fetch;
+    const openRouter = isOpenRouter(this.opts.baseUrl);
+    const sleep = this.opts.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
+    let maxTokens = req.maxTokens ?? 4096;
+    const body = () =>
+      JSON.stringify({
+        model: this.opts.model,
+        max_tokens: maxTokens,
+        messages: [{ role: "system", content: req.system }, ...req.messages],
+        ...(this.opts.reasoningEffort ? { reasoning: { effort: this.opts.reasoningEffort } } : {}),
+        ...(this.opts.provider ? { provider: this.opts.provider } : {}),
+        // OpenRouter then reports what each request actually cost.
+        ...(openRouter ? { usage: { include: true } } : {}),
+      });
+
+    for (let attempt = 1; ; attempt++) {
+      const res = await fetchFn(`${this.opts.baseUrl.replace(/\/+$/, "")}/chat/completions`, {
+        method: "POST",
+        headers: { "content-type": "application/json", authorization: `Bearer ${this.opts.apiKey}` },
+        body: body(),
+      });
+      if (res.ok) {
+        const data = (await res.json()) as {
+          choices?: { message?: { content?: string | null } }[];
+          usage?: { prompt_tokens?: number; completion_tokens?: number; cost?: number };
+        };
+        if (data.usage) {
+          const { prompt_tokens = 0, completion_tokens = 0, cost } = data.usage;
+          req.onUsage?.({ inputTokens: prompt_tokens, outputTokens: completion_tokens, ...(typeof cost === "number" ? { costUsd: cost } : {}) });
+        }
+        return data.choices?.[0]?.message?.content ?? "";
+      }
+      const text = await res.text();
+      // OpenRouter: the account (or the key's limit) can't cover this request.
+      if (res.status === 402) {
+        const affordable = Number(text.match(AFFORDABLE)?.[1]);
+        if (affordable >= MIN_AFFORDABLE_TOKENS && affordable < maxTokens) {
+          maxTokens = affordable;
+          continue;
+        }
+        this.outOfCredits = new OutOfCreditsError(text);
+        throw this.outOfCredits;
+      }
+      if (!isRetryable(res.status) || attempt >= MAX_ATTEMPTS) {
+        throw new Error(`LLM HTTP ${res.status}: ${text}`);
+      }
+      const retryAfter = Number(res.headers.get("retry-after"));
+      const waitMs = retryAfter > 0 ? retryAfter * 1000 : 1000 * 2 ** attempt;
+      await sleep(Math.min(waitMs, MAX_WAIT_MS));
+    }
+  }
+}

package/src/llm/openrouter.ts

@@ -0,0 +1,9 @@
+/** Is this OpenRouter's API? By host, so https://evil.test/openrouter.ai/ is not. */
+export function isOpenRouter(url: string | undefined): boolean {
+  try {
+    const host = new URL(url ?? "").hostname.toLowerCase();
+    return host === "openrouter.ai" || host.endsWith(".openrouter.ai");
+  } catch {
+    return false;
+  }
+}

package/src/llm/scripted.ts

@@ -0,0 +1,18 @@
+import type { LlmClient, LlmRequest } from "./types";
+
+const estimateTokens = (text: string) => Math.ceil(text.length / 4);
+
+export class ScriptedLlm implements LlmClient {
+  readonly calls: LlmRequest[] = [];
+
+  constructor(private readonly respond: (req: LlmRequest, callIndex: number) => string) {}
+
+  async complete(req: LlmRequest): Promise<string> {
+    const index = this.calls.length;
+    this.calls.push(req);
+    const reply = this.respond(req, index);
+    // No provider to ask, so estimate: lets tests exercise metering and caps.
+    req.onUsage?.({ inputTokens: estimateTokens(req.system + req.messages.map((m) => m.content).join("")), outputTokens: estimateTokens(reply) });
+    return reply;
+  }
+}