@stringsync/splates 0.0.61 → 1.13.3

package/vendor/@stringsync/splates-server/index-raqrwzdq.js

@@ -0,0 +1,952 @@
+import {
+  fail2
+} from "./index-t9w5a8ed.js";
+import {
+  pickableAgents
+} from "./index-acbcm66f.js";
+import {
+  body
+} from "./index-caep3sde.js";
+import {
+  evalsDoneReference2
+} from "./index-0jttnv86.js";
+
+// evals/agents.ts
+import { z } from "zod";
+var NATIVE_AGENTS = ["claude", "codex", "opencode", "kimi"];
+var FORMAT = 1;
+var EVAL_AGENTS = z.object({
+  format: z.literal(FORMAT),
+  agents: z.array(z.object({
+    agent: z.enum(NATIVE_AGENTS),
+    model: z.string().min(1).optional(),
+    effort: z.string().min(1).optional()
+  }))
+});
+
+class EvalAgentsStore2 {
+  path;
+  fs;
+  constructor(path, fs) {
+    this.path = path;
+    this.fs = fs;
+  }
+  async agents() {
+    const raw = await this.fs.read(this.path).catch(() => null);
+    if (raw === null) {
+      return [];
+    }
+    let json;
+    try {
+      json = JSON.parse(raw);
+    } catch {
+      fail2("usage", `${this.path} is not JSON: fix or delete it`, {
+        path: this.path
+      });
+    }
+    return this.checked(json, `${this.path} is not eval agents`).agents;
+  }
+  async save(agents) {
+    const checked = this.checked({ format: FORMAT, agents }, "not eval agents");
+    const seen = new Set;
+    for (const pick of checked.agents) {
+      const key = JSON.stringify([pick.agent, pick.model, pick.effort]);
+      if (seen.has(key)) {
+        fail2("usage", "not eval agents: an agent names the same model and thinking level once", { path: this.path });
+      }
+      seen.add(key);
+    }
+    await this.fs.mkdir(this.path.slice(0, this.path.lastIndexOf("/")), {
+      recursive: true
+    });
+    const tmp = `${this.path}.${crypto.randomUUID()}.tmp`;
+    await this.fs.write(tmp, `${JSON.stringify(checked, null, "\t")}
+`);
+    await this.fs.rename(tmp, this.path);
+  }
+  checked(value, what) {
+    const parsed = EVAL_AGENTS.safeParse(value);
+    if (!parsed.success) {
+      const issue = parsed.error.issues[0];
+      fail2("usage", `${what}: ${issue?.path.join(".") || "the agents"}: ${issue?.message}`, { path: this.path });
+    }
+    return parsed.data;
+  }
+}
+// evals/agents-routes.ts
+class EvalAgentsRoutes2 {
+  opts;
+  store;
+  constructor(opts) {
+    this.opts = opts;
+    this.store = new EvalAgentsStore2(`${opts.home}/eval-agents.json`, opts.fs);
+  }
+  read = async () => Response.json(await this.view());
+  save = async (request) => {
+    const { agents } = await body(request);
+    await this.store.save(agents);
+    return Response.json(await this.view());
+  };
+  async view() {
+    let agents = [];
+    let problem = null;
+    try {
+      agents = await this.store.agents();
+    } catch (error) {
+      if (!(error instanceof Error)) {
+        throw error;
+      }
+      problem = error.message;
+    }
+    const roster = await pickableAgents(this.opts.fs, this.opts.home);
+    return {
+      agents,
+      problem,
+      roster: roster.filter((each) => NATIVE_AGENTS.includes(each.id))
+    };
+  }
+}
+// evals/contract.ts
+import { readFile } from "node:fs/promises";
+import { join, resolve, sep } from "node:path";
+var CONTRACT_FORMAT2 = "splates-evals/1";
+var RESULTS_FILE2 = "results.jsonl";
+function isContractBatch(saved) {
+  return typeof saved === "object" && saved !== null && saved.format === CONTRACT_FORMAT2 && Array.isArray(saved.cells);
+}
+function contractRunId2(batch, cell) {
+  return `${batch}#${cell}`;
+}
+async function contractCells(dir, id, batch) {
+  const results = await resultsOf(dir);
+  const targets = [];
+  const grid = batch.cells.map((cell) => {
+    const target = contractTarget(cell.target);
+    if (!targets.some((each) => each.key === target.key)) {
+      targets.push(target);
+    }
+    const result = results.get(cell.id);
+    return {
+      scenario: cell.case,
+      aspect: cell.group ?? "",
+      requirement: cell.about ?? "",
+      target: target.key,
+      repetition: cell.repetition ?? 0,
+      run: result === undefined ? null : runOf(contractRunId2(id, cell.id), result)
+    };
+  });
+  return { targets, grid };
+}
+async function contractDetail(dir, id, batch, cell) {
+  const planned = batch.cells.find((each) => each.id === cell);
+  const result = (await resultsOf(dir)).get(cell);
+  if (planned === undefined || result === undefined) {
+    return null;
+  }
+  const detail = await detailOf(dir, result.detail);
+  return {
+    ...runOf(contractRunId2(id, cell), result),
+    producer: batch.producer ?? null,
+    scenario: planned.case,
+    aspect: planned.group ?? "",
+    requirement: planned.about ?? "",
+    target: contractTarget(planned.target),
+    repetition: planned.repetition ?? 0,
+    input: detail.input ?? "",
+    rubric: detail.rubric ?? null,
+    checks: detail.checks ?? [],
+    judge: detail.judge ?? null,
+    reply: detail.reply ?? "",
+    transcript: detail.transcript ?? []
+  };
+}
+function contractTarget(target) {
+  const agent = target.agent ?? "";
+  const variant = target.variant ?? null;
+  return {
+    key: `${agent}/${target.model}${target.effort === undefined ? "" : `@${target.effort}`}${variant === null ? "" : ` @${variant}`}`,
+    agent,
+    model: target.model,
+    effort: target.effort ?? null,
+    skill: variant
+  };
+}
+function runOf(id, result) {
+  return {
+    id,
+    status: result.status,
+    tokens: result.tokens ?? null,
+    costUsd: result.costUsd ?? null,
+    milliseconds: result.milliseconds ?? 0,
+    ...result.status === "error" && result.reason !== undefined ? { error: result.reason } : {},
+    ...result.score === undefined ? {} : { score: result.score },
+    ...result.reason === undefined ? {} : { reason: result.reason },
+    ...result.url === undefined ? {} : { url: result.url }
+  };
+}
+async function resultsOf(dir) {
+  const text = await readFile(join(dir, RESULTS_FILE2), "utf8").catch((error) => {
+    if (error.code === "ENOENT") {
+      return "";
+    }
+    throw error;
+  });
+  const results = new Map;
+  for (const line of text.split(`
+`)) {
+    const result = parsed(line);
+    if (result !== null) {
+      results.set(result.cell, result);
+    }
+  }
+  return results;
+}
+function parsed(line) {
+  if (line.trim() === "") {
+    return null;
+  }
+  try {
+    const result = JSON.parse(line);
+    return typeof result.cell === "string" && (result.status === "pass" || result.status === "fail" || result.status === "error") ? result : null;
+  } catch {
+    return null;
+  }
+}
+async function detailOf(dir, name) {
+  if (name === undefined) {
+    return {};
+  }
+  const path = resolve(dir, name);
+  if (!path.startsWith(resolve(dir) + sep)) {
+    return {};
+  }
+  return readFile(path, "utf8").then((text) => JSON.parse(text), () => ({}));
+}
+// evals/target.ts
+function targetName2(target) {
+  return [
+    target.agent,
+    target.model,
+    target.effort,
+    target.skill === null ? null : `@${target.skill}`
+  ].filter((part) => part !== null && part !== "").join(" ");
+}
+
+// evals/done.ts
+var CUT = 240;
+var TALLY = ["pass", "fail", "error", "unreadable", "pending"];
+function evalsDonePrompt2(grid, details) {
+  const tally = TALLY.filter((status) => grid.tally[status]).map((status) => `${grid.tally[status]} ${status}`).join(" · ");
+  const cost = [
+    grid.tokens === null ? null : `${grid.tokens.toLocaleString()} tokens`,
+    grid.costUsd === null ? null : `$${grid.costUsd.toFixed(2)}`
+  ].filter((part) => part !== null);
+  const scenarios = [...new Set(grid.grid.map((cell) => cell.scenario))];
+  const table = [
+    `| Scenario | ${grid.targets.map(targetName2).join(" | ")} |`,
+    `| --- |${" --- |".repeat(grid.targets.length)}`,
+    ...scenarios.map((scenario) => `| \`${scenario}\` | ${grid.targets.map((target) => outcomes(grid.grid.filter((cell) => cell.scenario === scenario && cell.target === target.key))).join(" | ")} |`)
+  ];
+  const missed = (runs) => runs.map((run) => {
+    const held = [
+      ...run.checks.filter((check) => !check.pass).map((check) => `  - Check \`${check.id}\`: ${check.about}${check.error === undefined ? "" : ` (${cut(check.error)})`}`),
+      ...run.judge?.pass === false ? [`  - Judge: ${cut(run.judge.reason)}`] : [],
+      ...run.error === undefined ? [] : [`  - Error: ${cut(run.error)}`]
+    ];
+    return [
+      `- \`${run.scenario}\` on ${targetName2(run.target)}, repetition ${run.repetition + 1}: ${run.status}`,
+      ...held,
+      `  - Reply: ${run.reply === "" ? "(none)" : cut(JSON.stringify(run.reply))}`,
+      `  - Run: \`${run.id}\`, in full with \`splates dev eval show ${run.id}\``
+    ].join(`
+`);
+  });
+  const failed = missed(details.filter((run) => run.status !== "error"));
+  const errored = missed(details.filter((run) => run.status === "error"));
+  const sections = [
+    ...failed.length === 0 ? [] : [`Not passing:
+
+${failed.join(`
+`)}`],
+    ...errored.length === 0 ? [] : [
+      `Errors, the harness's or the setup's, never the skill's:
+
+${errored.join(`
+`)}`
+    ]
+  ];
+  return `${evalsDoneReference2(grid.id)}
+
+The run your run evals card asked for is done: ${grid.cells} ${grid.cells === 1 ? "cell" : "cells"}${tally === "" ? "" : ` · ${tally}`}${cost.length === 0 ? "" : ` · ${cost.join(" · ")}`}.
+
+${table.join(`
+`)}
+${sections.length === 0 ? `
+Every run passed.` : `
+${sections.join(`
+
+`)}`}
+`;
+}
+function outcomes(cells) {
+  if (cells.length === 0) {
+    return "not run";
+  }
+  const statuses = cells.map((cell) => cell.run?.status ?? "pending");
+  if (statuses.length === 1) {
+    return statuses[0] ?? "pending";
+  }
+  const count = (status) => statuses.filter((each) => each === status).length;
+  const graded = statuses.filter((each) => each !== "error" && each !== "pending").length;
+  return [
+    `${count("pass")}/${graded} pass`,
+    ...["error", "pending"].filter((status) => count(status) > 0).map((status) => `${count(status)} ${status}`)
+  ].join(", ");
+}
+function cut(text) {
+  return text.length <= CUT ? text : `${text.slice(0, CUT)}…`;
+}
+// evals/import.ts
+import { mkdir, writeFile } from "node:fs/promises";
+import { join as join2 } from "node:path";
+async function writeImported2(dir, imported) {
+  await mkdir(dir, { recursive: true });
+  for (const [cell, detail] of imported.details) {
+    await writeFile(join2(dir, `${cell}.json`), JSON.stringify(detail));
+  }
+  await writeFile(join2(dir, RESULTS_FILE2), imported.results.map((result) => `${JSON.stringify(result)}
+`).join(""));
+  await writeFile(join2(dir, "batch.json"), JSON.stringify(imported.batch));
+}
+// evals/inspect.ts
+function inspectBatch2(log, passAt) {
+  const samples = log.samples ?? [];
+  const cells = samples.map((sample, index) => ({
+    id: `c${index}`,
+    case: String(sample.id),
+    group: log.eval.task,
+    target: { model: log.eval.model },
+    repetition: (sample.epoch ?? 1) - 1
+  }));
+  return {
+    batch: {
+      format: CONTRACT_FORMAT2,
+      startedAt: new Date(log.stats?.started_at ?? log.eval.created ?? Date.now()).toISOString(),
+      title: `${log.eval.task} on ${log.eval.model}`,
+      cells
+    },
+    results: samples.map((sample, index) => resultOf(`c${index}`, sample, passAt)),
+    details: new Map(samples.map((sample, index) => [`c${index}`, detailOf2(sample, passAt)]))
+  };
+}
+function resultOf(cell, sample, passAt) {
+  const usage = Object.values(sample.model_usage ?? {});
+  const tokens = sum(usage.map((each) => each.total_tokens));
+  const costUsd = sum(usage.map((each) => each.total_cost));
+  const measured = {
+    ...tokens === undefined ? {} : { tokens },
+    ...costUsd === undefined ? {} : { costUsd },
+    ...typeof sample.total_time === "number" ? { milliseconds: Math.round(sample.total_time * 1000) } : {},
+    detail: `${cell}.json`
+  };
+  if (sample.error) {
+    return {
+      cell,
+      status: "error",
+      reason: sample.error.message ?? "the sample errored",
+      ...measured
+    };
+  }
+  const scores = Object.entries(sample.scores ?? {});
+  if (scores.length === 0) {
+    return {
+      cell,
+      status: "error",
+      reason: "no scorer scored it",
+      ...measured
+    };
+  }
+  const values = scores.map(([, score]) => scoreValue(score.value));
+  const known = values.filter((value) => value !== null);
+  const score = known.length === 0 ? undefined : known.reduce((total, value) => total + value, 0) / known.length;
+  return {
+    cell,
+    status: values.every((value) => value !== null && value >= passAt) ? "pass" : "fail",
+    ...score === undefined || score < 0 || score > 1 ? {} : { score },
+    ...reasonOf(scores) === "" ? {} : { reason: reasonOf(scores) },
+    ...measured
+  };
+}
+function detailOf2(sample, passAt) {
+  const target = [sample.target ?? []].flat();
+  return {
+    input: typeof sample.input === "string" ? sample.input : sample.input.map(textOf).join(`
+
+`),
+    ...target.length === 0 ? {} : { rubric: target.join(`
+`) },
+    reply: sample.output?.completion ?? "",
+    checks: Object.entries(sample.scores ?? {}).map(([name, score]) => {
+      const value = scoreValue(score.value);
+      return {
+        id: name,
+        about: score.explanation ?? score.answer ?? String(score.value),
+        pass: value !== null && value >= passAt
+      };
+    }),
+    transcript: transcriptOf(sample.messages ?? [])
+  };
+}
+function scoreValue(value) {
+  if (typeof value === "number") {
+    return value;
+  }
+  if (typeof value === "boolean") {
+    return value ? 1 : 0;
+  }
+  if (typeof value === "string") {
+    const letter = { C: 1, P: 0.5, I: 0, N: 0 }[value];
+    if (letter !== undefined) {
+      return letter;
+    }
+    const lower = value.toLowerCase();
+    if (lower === "yes" || lower === "true") {
+      return 1;
+    }
+    if (lower === "no" || lower === "false") {
+      return 0;
+    }
+    const number = Number(value);
+    return value.trim() === "" || Number.isNaN(number) ? null : number;
+  }
+  return null;
+}
+function reasonOf(scores) {
+  const reasons = scores.filter(([, score]) => score.explanation);
+  if (reasons.length === 1) {
+    return reasons[0]?.[1].explanation ?? "";
+  }
+  return reasons.map(([name, score]) => `${name}: ${score.explanation}`).join(`
+`);
+}
+function transcriptOf(messages) {
+  const outputs = new Map(messages.filter((message) => message.role === "tool" && message.tool_call_id).map((message) => [message.tool_call_id, textOf(message)]));
+  return messages.flatMap((message) => {
+    if (message.role === "tool") {
+      return [];
+    }
+    const text = textOf(message);
+    return [
+      ...text === "" ? [] : [{ kind: "text", text: `${message.role}: ${text}` }],
+      ...(message.tool_calls ?? []).map((call) => ({
+        kind: "tool",
+        name: call.function,
+        input: JSON.stringify(call.arguments),
+        output: outputs.get(call.id) ?? null
+      }))
+    ];
+  });
+}
+function textOf(message) {
+  if (typeof message.content === "string") {
+    return message.content;
+  }
+  return message.content.map((part) => part.text ?? "").filter((text) => text !== "").join(`
+`);
+}
+function sum(values) {
+  const known = values.filter((value) => typeof value === "number");
+  return known.length === 0 ? undefined : known.reduce((total, value) => total + value, 0);
+}
+// evals/launch.ts
+import { readdir } from "node:fs/promises";
+import { Dispatcher } from "../../core/events/index.js";
+function targetOf(pick) {
+  return `${pick.agent}${pick.model === undefined ? "" : `:${pick.model}`}${pick.effort === undefined ? "" : `@${pick.effort}`}`;
+}
+var TAIL = 20;
+
+class EvalLaunches2 {
+  ps;
+  dispatcher = new Dispatcher;
+  events = this.dispatcher.events;
+  launches = new Map;
+  watching = new Set;
+  record;
+  writing = Promise.resolve();
+  disposing = false;
+  constructor(ps, storage) {
+    this.ps = ps;
+    this.record = storage.document("eval-launches.json");
+  }
+  async scenarios(commands, cwd = null) {
+    if (commands.list === null) {
+      return [];
+    }
+    const { exitCode, stdout, stderr } = await this.ps.spawnCapture(["sh", "-c", commands.list], { cwd: cwd ?? commands.root });
+    if (exitCode !== 0) {
+      fail2("hook_failed", tailOf(`${stdout}${stderr}`) || `evals.list exited ${exitCode}`, {});
+    }
+    return stdout.split(`
+`).flatMap((line) => {
+      const each = parsed2(line);
+      return each === null || typeof each.case !== "string" || each.case === "" ? [] : [
+        {
+          id: each.case,
+          group: typeof each.group === "string" ? each.group : null,
+          about: typeof each.about === "string" ? each.about : null
+        }
+      ];
+    });
+  }
+  async plan(commands, spec) {
+    if (commands.plan === null) {
+      return { cells: null, usd: null, refused: false, output: "" };
+    }
+    const { exitCode, stdout, stderr } = await this.ps.spawnCapture(["sh", "-c", commands.plan], { cwd: spec.cwd ?? commands.root, env: envOf(commands, spec) });
+    const said = stdout.split(`
+`).map(parsed2).findLast((each) => each !== null);
+    const number = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
+    return {
+      cells: number(said?.cells),
+      usd: exitCode === 0 ? number(said?.usd) : null,
+      refused: exitCode !== 0,
+      output: tailOf(`${stdout}${stderr}`)
+    };
+  }
+  async start(commands, spec) {
+    if (commands.run === null) {
+      fail2("not_found", "this project names no evals.run in splates.config.ts", {});
+    }
+    const id = idOf(commands, spec.key);
+    const held = this.launches.get(id);
+    if (held !== undefined && held.status === "running") {
+      return this.read(held);
+    }
+    const known = new Set(await batchesIn(commands.results));
+    const child = this.ps.start(["sh", "-c", commands.run], {
+      cwd: spec.cwd ?? commands.root,
+      env: envOf(commands, spec)
+    });
+    const launch = {
+      id: crypto.randomUUID(),
+      key: spec.key,
+      status: "running",
+      batch: null,
+      output: "",
+      recordId: id,
+      commands,
+      spec,
+      child,
+      known,
+      stopping: false
+    };
+    this.launches.set(id, launch);
+    try {
+      await this.save(launch, true);
+    } catch (error) {
+      this.launches.delete(id);
+      launch.child.kill();
+      throw error;
+    }
+    this.running(launch);
+    const watching = this.watch(launch);
+    this.watching.add(watching);
+    watching.then(() => this.watching.delete(watching), () => {});
+    return this.read(launch);
+  }
+  async resume(commands) {
+    const current = new Map(commands.map((each) => [each.root, each]));
+    for (const [id, saved] of Object.entries(await this.record.read() ?? {})) {
+      if (saved.launch.status !== "running" || this.launches.get(id)?.status === "running") {
+        continue;
+      }
+      const commandsNow = current.get(saved.commands.root);
+      if (commandsNow?.run != null) {
+        await this.start(commandsNow, saved.spec);
+      }
+    }
+  }
+  async get(commands, key) {
+    const id = idOf(commands, key);
+    const launch = this.launches.get(id);
+    return launch === undefined ? (await this.record.read())?.[id]?.launch ?? null : this.read(launch);
+  }
+  async stop(commands, key) {
+    const launch = this.launches.get(idOf(commands, key));
+    if (launch === undefined) {
+      return null;
+    }
+    if (launch.status === "running") {
+      launch.stopping = true;
+      launch.child.kill();
+    }
+    return this.read(launch);
+  }
+  async disposeAsync() {
+    this.disposing = true;
+    for (const launch of this.launches.values()) {
+      if (launch.status === "running") {
+        launch.child.kill();
+      }
+    }
+    await Promise.all(this.watching);
+  }
+  async watch(launch) {
+    const words = [launch.child.stdout, launch.child.stderr].map(async (stream) => {
+      for await (const text of decoded(stream)) {
+        launch.output = tailOf(launch.output + text);
+      }
+    });
+    const code = await launch.child.exited;
+    await Promise.all(words);
+    await this.read(launch);
+    if (this.disposing && !launch.stopping) {
+      return;
+    }
+    launch.status = launch.stopping ? "stopped" : code === 0 ? "done" : launch.batch === null ? "refused" : "done";
+    await this.save(launch);
+    if (launch.status === "done" && launch.batch !== null && launch.spec.conversation != null) {
+      this.dispatcher.dispatch("finished", {
+        launch: viewOf(launch),
+        results: launch.commands.results,
+        conversation: launch.spec.conversation
+      });
+    } else {
+      this.running(launch);
+    }
+  }
+  running(launch) {
+    const conversation = launch.spec.conversation;
+    if (conversation == null) {
+      return;
+    }
+    this.dispatcher.dispatch("running", {
+      conversation,
+      running: [...this.launches.values()].some((each) => each.status === "running" && each.spec.conversation === conversation)
+    });
+  }
+  async read(launch) {
+    if (launch.batch === null) {
+      const made = (await batchesIn(launch.commands.results)).filter((each) => !launch.known.has(each));
+      launch.batch = made.sort().at(-1) ?? null;
+      if (launch.batch !== null) {
+        await this.save(launch);
+      }
+    }
+    return viewOf(launch);
+  }
+  save(launch, replace = false) {
+    const saved = {
+      launch: viewOf(launch),
+      commands: launch.commands,
+      spec: launch.spec
+    };
+    const writing = this.writing.then(async () => {
+      await this.record.update((current) => {
+        const all = current ?? {};
+        if (!replace && all[launch.recordId]?.launch.id !== launch.id) {
+          return all;
+        }
+        return { ...all, [launch.recordId]: saved };
+      });
+    });
+    this.writing = writing.catch(() => {});
+    return writing;
+  }
+}
+function envOf(commands, spec) {
+  return {
+    SPLATES_EVALS_CASES: spec.scenarios.join(`
+`),
+    SPLATES_EVALS_TARGETS: spec.agents.map(targetOf).join(`
+`),
+    SPLATES_EVALS_RESULTS: commands.results,
+    ...spec.reps === undefined ? {} : { SPLATES_EVALS_REPS: String(spec.reps) },
+    ...spec.baseline === null ? {} : { SPLATES_EVALS_BASELINE: spec.baseline }
+  };
+}
+function idOf(commands, key) {
+  return `${commands.root}
+${key}`;
+}
+async function batchesIn(results) {
+  return (await readdir(results, { withFileTypes: true }).catch(() => [])).filter((each) => each.isDirectory()).map((each) => each.name);
+}
+function parsed2(line) {
+  try {
+    const value = JSON.parse(line);
+    return typeof value === "object" && value !== null && !Array.isArray(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+function viewOf(launch) {
+  return {
+    id: launch.id,
+    key: launch.key,
+    status: launch.status,
+    batch: launch.batch,
+    output: launch.output
+  };
+}
+async function* decoded(stream) {
+  const decoder = new TextDecoder;
+  for await (const chunk of stream) {
+    yield decoder.decode(chunk, { stream: true });
+  }
+}
+function tailOf(text) {
+  return text.trim().split(`
+`).slice(-TAIL).join(`
+`);
+}
+// evals/launch-routes.ts
+import { existsSync } from "node:fs";
+class EvalLaunchRoutes2 {
+  launches;
+  projects;
+  dirs;
+  constructor(launches, projects, dirs = { cwdOf: async () => null }) {
+    this.launches = launches;
+    this.projects = projects;
+    this.dirs = dirs;
+  }
+  scenarios = async (request) => {
+    const commands = await this.commands(request);
+    if (commands.list === null) {
+      fail2("not_found", "this project names no evals.list in splates.config.ts", {});
+    }
+    const conversation = new URL(request.url).searchParams.get("conversation");
+    return Response.json(await this.launches.scenarios(commands, await this.cwdOf(conversation)));
+  };
+  plan = async (request) => {
+    const commands = await this.runnable(request);
+    const sent = await body(request);
+    return Response.json(await this.launches.plan(commands, {
+      ...this.spec(sent),
+      cwd: await this.cwdOf(this.conversation(sent))
+    }));
+  };
+  start = async (request) => {
+    const commands = await this.runnable(request);
+    const sent = await body(request);
+    const spec = this.spec(sent);
+    const { key } = sent;
+    if (typeof key !== "string" || key === "") {
+      fail2("usage", "a run names its card by `key`", {});
+    }
+    const conversation = this.conversation(sent);
+    return Response.json(await this.launches.start(commands, {
+      ...spec,
+      key,
+      conversation,
+      cwd: await this.cwdOf(conversation)
+    }));
+  };
+  read = async (request) => Response.json(await this.launches.get(await this.commands(request), this.key(request)));
+  stop = async (request) => Response.json(await this.launches.stop(await this.commands(request), this.key(request)));
+  async commands(request) {
+    const commands = await this.projects.commandsOf(request);
+    if (commands === null) {
+      fail2("not_found", "this project names no evals in splates.config.ts", {});
+    }
+    return commands;
+  }
+  async runnable(request) {
+    const commands = await this.commands(request);
+    if (commands.run === null) {
+      fail2("not_found", "this project names no evals.run in splates.config.ts", {});
+    }
+    return commands;
+  }
+  async cwdOf(id) {
+    const cwd = id === null || id === "" ? null : await this.dirs.cwdOf(id);
+    return cwd !== null && existsSync(cwd) ? cwd : null;
+  }
+  conversation(sent) {
+    const { conversation } = sent;
+    if (conversation != null && typeof conversation !== "string") {
+      fail2("usage", "a run's conversation is its id", {});
+    }
+    return conversation ?? null;
+  }
+  key(request) {
+    return new URL(request.url).searchParams.get("key") ?? "";
+  }
+  spec(sent) {
+    const { skill, scenarios, agents, baseline, reps } = sent;
+    if (typeof skill !== "string" || skill === "") {
+      fail2("usage", "a run names the skill or suite of its card", {});
+    }
+    if (!Array.isArray(scenarios) || scenarios.length === 0 || !scenarios.every((each) => typeof each === "string" && each !== "")) {
+      fail2("usage", "pick at least one scenario", {});
+    }
+    if (!Array.isArray(agents) || agents.length === 0) {
+      fail2("usage", "pick at least one agent to run on", {});
+    }
+    for (const pick of agents) {
+      if (typeof pick?.agent !== "string" || !NATIVE_AGENTS.includes(pick.agent)) {
+        fail2("usage", `an agent is one of ${NATIVE_AGENTS.join(", ")}`, {});
+      }
+    }
+    if (baseline != null && typeof baseline !== "string") {
+      fail2("usage", "the baseline is a revision", {});
+    }
+    if (reps != null && (typeof reps !== "number" || !Number.isInteger(reps) || reps < 1)) {
+      fail2("usage", "repetitions are a whole number of at least 1", {});
+    }
+    return {
+      skill,
+      scenarios,
+      agents,
+      baseline: baseline ?? null,
+      ...reps == null ? {} : { reps }
+    };
+  }
+}
+// evals/routes.ts
+import { homedir } from "node:os";
+import { resolve as resolve2, sep as sep2 } from "node:path";
+
+// evals/store.ts
+import { readdir as readdir2, readFile as readFile2 } from "node:fs/promises";
+import { join as join3 } from "node:path";
+class EvalStore2 {
+  directory;
+  constructor(directory) {
+    this.directory = directory;
+  }
+  async batches() {
+    const grids = await Promise.all((await this.ids()).map((id) => this.grid(id)));
+    return grids.filter((grid) => grid !== null).map(({ grid: _, ...batch }) => batch).sort((first, second) => second.startedAt.localeCompare(first.startedAt));
+  }
+  async grid(id) {
+    const dir = join3(this.directory, id);
+    const saved = await readJson(join3(dir, "batch.json")).catch(() => null);
+    if (!isContractBatch(saved)) {
+      return null;
+    }
+    const { targets, grid } = await contractCells(dir, id, saved);
+    return gridOf(id, saved.title ?? null, saved.producer ?? null, saved.startedAt, targets, grid);
+  }
+  async run(id) {
+    const hash = id.lastIndexOf("#");
+    const batch = id.slice(0, Math.max(0, hash));
+    const dir = join3(this.directory, batch);
+    const saved = hash === -1 ? null : await readJson(join3(dir, "batch.json")).catch(() => null);
+    const detail = isContractBatch(saved) ? await contractDetail(dir, batch, saved, id.slice(hash + 1)) : null;
+    if (detail === null) {
+      throw new Error(`no eval run ${id}`);
+    }
+    return detail;
+  }
+  async ids() {
+    const entries = await readdir2(this.directory, {
+      withFileTypes: true
+    }).catch(() => []);
+    return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+  }
+}
+function gridOf(id, skill, producer, startedAt, targets, grid) {
+  const tally = {};
+  for (const cell of grid) {
+    const status = cell.run?.status ?? "pending";
+    tally[status] = (tally[status] ?? 0) + 1;
+  }
+  const runs = grid.flatMap((cell) => cell.run === null ? [] : [cell.run]);
+  return {
+    id,
+    skill,
+    producer,
+    startedAt,
+    targets,
+    tally,
+    cells: grid.length,
+    tokens: total(runs.map((each) => each.tokens)),
+    costUsd: total(runs.map((each) => each.costUsd)),
+    grid
+  };
+}
+async function readJson(path) {
+  const text = await readFile2(path, "utf8").catch((error) => {
+    if (error.code === "ENOENT") {
+      return null;
+    }
+    throw error;
+  });
+  return text === null ? null : JSON.parse(text);
+}
+function total(values) {
+  const known = values.filter((each) => each !== null);
+  return known.length === 0 ? null : known.reduce((sum, each) => sum + each, 0);
+}
+
+// evals/routes.ts
+class EvalRoutes2 {
+  opts;
+  stores = new Map;
+  constructor(opts) {
+    this.opts = opts;
+  }
+  list = async (request) => {
+    const directory = await this.opts.directory(request);
+    return Response.json(directory === null ? null : await this.storeOf(directory).batches());
+  };
+  batch = async (request) => {
+    const id = new URL(request.url).searchParams.get("id") ?? "";
+    const grid = await (await this.opened(request)).store.grid(id);
+    if (grid === null) {
+      fail2("not_found", `no eval batch ${id}`, { id });
+    }
+    return Response.json(grid);
+  };
+  run = async (request) => {
+    const id = new URL(request.url).searchParams.get("id") ?? "";
+    const { directory, store } = await this.opened(request);
+    const hash = id.lastIndexOf("#");
+    if (hash === -1 || !resolve2(directory, id.slice(0, hash)).startsWith(directory + sep2)) {
+      fail2("usage", `no eval run ${id}`, { id });
+    }
+    return Response.json(await store.run(id).catch(() => fail2("not_found", `no eval run ${id}`, { id })));
+  };
+  async donePrompt(directory, id) {
+    const store = this.storeOf(directory);
+    const grid = await store.grid(id);
+    if (grid === null) {
+      return null;
+    }
+    const missed = grid.grid.flatMap((cell) => cell.run === null || cell.run.status === "pass" ? [] : [cell.run.id]);
+    const details = await Promise.all(missed.map((run) => store.run(run).catch(() => null)));
+    return evalsDonePrompt2(grid, details.filter((each) => each !== null));
+  }
+  async opened(request) {
+    const directory = await this.opts.directory(request);
+    if (directory === null) {
+      fail2("not_found", "this project names no evals.results in splates.config.ts", {});
+    }
+    return { directory, store: this.storeOf(directory) };
+  }
+  storeOf(directory) {
+    let store = this.stores.get(directory);
+    if (store === undefined) {
+      store = new EvalStore2(directory);
+      this.stores.set(directory, store);
+    }
+    return store;
+  }
+}
+function projectEvalCommands2(ps, root, evals) {
+  if (evals === null) {
+    return null;
+  }
+  const { results } = evals;
+  return {
+    root,
+    results: results === "~" || results.startsWith("~/") ? resolve2(ps.env("HOME") ?? homedir(), results.slice(2)) : resolve2(root, results),
+    list: evals.list ?? null,
+    run: evals.run ?? null,
+    plan: evals.plan ?? null
+  };
+}
+export { EvalAgentsStore2, EvalAgentsRoutes2, CONTRACT_FORMAT2, RESULTS_FILE2, contractRunId2, targetName2, evalsDonePrompt2, writeImported2, inspectBatch2, EvalLaunches2, EvalLaunchRoutes2, EvalStore2, EvalRoutes2, projectEvalCommands2 };