offgrid-ai 0.16.0 → 0.17.0

package/src/benchmark/finalize.mjs

@@ -1,169 +0,0 @@
-// ── Benchmark finalization (metadata + summary rendering) ───────────────────
-// unloadModelFromServer has been moved to src/process.mjs (it's the managed-server
-// counterpart to stopProfile, used by both the benchmark flow and the Pi chat flow).
-
-import { existsSync } from "node:fs";
-import { readFile, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { pc, renderRows, renderSection } from "../ui.mjs";
-
-export async function finalizeBenchmarkRun(runDirectory, runResult, speedMetrics, speedMetricsError = null) {
-  const metadataPath = join(runDirectory, "metadata.json");
-  const metadata = JSON.parse(await readFile(metadataPath, "utf8"));
-  const now = new Date();
-  const timestamp = now.toISOString();
-
-  const kind = metadata.kind ?? "visual";
-  const isDs = kind === "data-science";
-  const requiredFile = isDs ? "analysis.ipynb" : "index.html";
-  const requiredPath = join(runDirectory, requiredFile);
-
-  const outputFiles = [];
-  for (const candidate of [requiredFile, isDs ? "summary.json" : "preview.png", isDs ? "chart-distribution.png" : "preview.webm", "preview.mp4"]) {
-    if (existsSync(join(runDirectory, candidate))) {
-      outputFiles.push(candidate);
-    }
-  }
-
-  const success = existsSync(requiredPath) && (await readFile(requiredPath, "utf8")).trim().length > 0;
-  const hasTurns = runResult.agentTurns > 0;
-
-  let failureReason = null;
-  if (runResult.error) {
-    failureReason = typeof runResult.error === "string" ? runResult.error : (runResult.error.message ?? "Unknown error");
-  } else if (!hasTurns) {
-    failureReason = "The model did not produce any response turns.";
-  } else if (!success) {
-    if (runResult.toolCalls === 0) {
-      failureReason = `The model finished without writing the required output file (${requiredFile}). It may have returned the response as chat text instead of using the write tool.`;
-    } else {
-      failureReason = `The required output file (${requiredFile}) was missing or empty after the run.`;
-    }
-  }
-
-  const failed = failureReason !== null;
-
-  metadata.status = failed ? "failed" : "completed";
-  metadata.updatedAt = timestamp;
-  if (failed) {
-    metadata.failedAt = timestamp;
-  } else {
-    metadata.completedAt = timestamp;
-  }
-
-  const totalTokens = runResult.promptTokens + runResult.completionTokens;
-
-  metadata.runner.tokenMetrics = {
-    reported: hasTurns,
-    promptTokens: runResult.promptTokens,
-    completionTokens: runResult.completionTokens,
-    totalTokens,
-  };
-
-  metadata.runner.speedMetrics = speedMetrics;
-  metadata.runner.metricSource = speedMetrics?.metricSource ?? null;
-  metadata.runner.speedMetricsError = speedMetricsError ?? null;
-
-  metadata.results = {
-    wallClockMs: runResult.wallClockMs,
-    agentTurns: runResult.agentTurns,
-    toolCalls: runResult.toolCalls,
-    toolResults: runResult.toolResults,
-    success,
-    outputFiles,
-    perTurn: runResult.perTurn,
-  };
-
-  if (failureReason) {
-    metadata.error = { message: failureReason, ...(typeof runResult.error === "object" && runResult.error?.stack ? { stack: runResult.error.stack } : {}) };
-  } else if (runResult.error) {
-    metadata.error = typeof runResult.error === "string"
-      ? { message: runResult.error }
-      : { message: runResult.error.message ?? "Unknown error", ...(runResult.error.stack ? { stack: runResult.error.stack } : {}) };
-  }
-
-  await writeFile(metadataPath, JSON.stringify(metadata, null, 2) + "\n", "utf8");
-  return metadata;
-}
-
-function formatMetric(value, formatter) {
-  if (value === null || value === undefined || !Number.isFinite(value)) return pc.dim("—");
-  return formatter(value);
-}
-
-function formatMs(ms) {
-  return formatMetric(ms, (n) => (n < 1000 ? `${Math.round(n)} ms` : `${(n / 1000).toFixed(1)} s`));
-}
-
-function formatNumber(n) {
-  return formatMetric(n, (v) => v.toLocaleString());
-}
-
-function formatTokPerSec(n) {
-  return formatMetric(n, (v) => `${v.toFixed(1)} tok/s`);
-}
-
-function formatPercent(n) {
-  return formatMetric(n, (v) => `${(v * 100).toFixed(0)} %`);
-}
-
-export function renderBenchmarkSummary(metadata) {
-  const { status, results, runner, error } = metadata;
-
-  const agentRows = [
-    ["Status", status === "completed" ? pc.green("completed") : pc.red(status ?? "failed")],
-    ["Duration", formatMs(results?.wallClockMs)],
-    ["Agent turns", formatNumber(results?.agentTurns)],
-    ["Input tokens", formatNumber(runner?.tokenMetrics?.promptTokens)],
-    ["Output tokens", formatNumber(runner?.tokenMetrics?.completionTokens)],
-    ["Total tokens", formatNumber(runner?.tokenMetrics?.totalTokens)],
-    ["Tool calls", formatNumber(results?.toolCalls)],
-    ["Tool results", formatNumber(results?.toolResults)],
-    ["Output files", (results?.outputFiles?.length ?? 0) > 0 ? results.outputFiles.join(", ") : pc.dim("—")],
-  ];
-
-  console.log("");
-  console.log(renderSection("Benchmark Result", renderRows(agentRows)));
-
-  if (status === "completed" && runner?.speedMetrics) {
-    const speed = runner.speedMetrics;
-    const speedRows = [
-      ["Prefill tok/s", formatTokPerSec(speed.prefillTokensPerSecond)],
-      ["Generation tok/s", formatTokPerSec(speed.generationTokensPerSecond)],
-      ["TTFT", formatMs(speed.ttftMs)],
-      ["Speculative decode", formatPercent(speed.speculativeDecodeAcceptance)],
-      ["KV cache tokens", formatNumber(speed.kvCacheTokens)],
-      ["Model load time", formatMs(speed.modelLoadMs)],
-      ["Metric source", speed.metricSource ?? pc.dim("—")],
-    ];
-    console.log(renderSection("Speed Metrics", renderRows(speedRows)));
-  } else if (error) {
-    const wrappedError = wrapText(error.message ?? "Unknown error");
-    console.log(renderSection("Error", pc.red(wrappedError)));
-    if (error.message?.includes("write tool") || error.message?.includes("required output file")) {
-      const tip = wrapText("Tip: This usually means the model returned the answer as chat text instead of writing the file. Try a model with stronger tool-use support, or run the prompt manually.", 64);
-      console.log(pc.dim("\n" + tip));
-    }
-  }
-
-  if (status === "completed" && !runner?.speedMetrics && runner?.speedMetricsError) {
-    console.log(pc.dim(`\nSpeed metrics unavailable: ${runner.speedMetricsError}`));
-  }
-}
-
-function wrapText(text, width = 64) {
-  if (!text) return "";
-  const words = text.split(/\s+/);
-  const lines = [];
-  let current = "";
-  for (const word of words) {
-    if ((current + " " + word).trim().length > width) {
-      if (current) lines.push(current.trim());
-      current = word;
-    } else {
-      current = current ? `${current} ${word}` : word;
-    }
-  }
-  if (current) lines.push(current.trim());
-  return lines.join("\n");
-}

package/src/benchmark/flow.mjs

@@ -1,239 +0,0 @@
-// ── Benchmark command flows ───────────────────────────────────────────────────
-
-import { join } from "node:path";
-import { ensureDirs } from "../config.mjs";
-import { backendFor } from "../backends.mjs";
-import { serverReady, startServer, waitForReady, stopProfile, modelAvailableOnServer, unloadModelFromServer } from "../process.mjs";
-import { loadProfiles } from "../profiles.mjs";
-import { pc, createPrompt } from "../ui.mjs";
-import { linkBenchmarkRepo } from "./repo.mjs";
-import { loadBenchmarks } from "./shared.mjs";
-import { prepareBenchmarkRun } from "./prepare.mjs";
-import { runBenchmarkInPi } from "./sdk-runner.mjs";
-import { queryServerMetrics } from "./metrics.mjs";
-import { finalizeBenchmarkRun, renderBenchmarkSummary } from "./finalize.mjs";
-
-function benchmarkModelSource(profile) {
-  if (!profile) return "cloud";
-  return profile.providerId === "llama-cpp-mtp" ? "llama-cpp-mtp" : profile.backend === "omlx" ? "omlx" : "llama-cpp";
-}
-
-async function chooseBenchmarkAction(prompt, canRun) {
-  const choices = [
-    { value: "run", label: "Run Benchmark", hint: "Automated with Pi" },
-    { value: "prepare", label: "Prepare Benchmark (manual)", hint: "Copy prompt and run yourself" },
-  ];
-  return await prompt.choice("Action", canRun ? choices : choices.filter((c) => c.value === "prepare"), canRun ? "run" : "prepare");
-}
-
-function managedModelId(profile) {
-  return profile.omlxModel ?? profile.modelAlias ?? profile.label;
-}
-
-async function ensureManagedModelAvailableForBenchmark(profile, backend) {
-  if (backend.type !== "managed-server") return;
-  if (await modelAvailableOnServer(profile)) return;
-  throw new Error(`${managedModelId(profile)} is not available on ${backend.label} at ${profile.baseUrl}.`);
-}
-
-async function ensureServerForBenchmark(profile) {
-  const backend = backendFor(profile.backend);
-  if (await serverReady(profile.baseUrl)) {
-    await ensureManagedModelAvailableForBenchmark(profile, backend);
-    console.log(pc.green(`[ready] ${backend.label} at ${profile.baseUrl}`));
-    return { started: false };
-  }
-
-  if (backend.type === "managed-server") {
-    throw new Error(`${backend.label} is not running at ${profile.baseUrl}. Start it and try again.`);
-  }
-
-  console.log(pc.dim(`Starting ${backend.label} for ${profile.label}...`));
-  const state = await startServer(profile);
-  await waitForReady(profile, state?.pid, state?.rawLogPath);
-  console.log(pc.green(`[ready] ${profile.baseUrl}/models`));
-  return { started: true, state };
-}
-
-export async function runPreparedBenchmark(profile, runDirectory, options = {}) {
-  const controller = new AbortController();
-  if (options.signal) {
-    options.signal.addEventListener("abort", () => controller.abort(), { once: true });
-  }
-  let serverStarted = false;
-  let benchmarkStarted = false;
-  let metadata;
-
-  const onSigint = () => {
-    controller.abort();
-  };
-  process.on("SIGINT", onSigint);
-
-  try {
-    const serverState = await ensureServerForBenchmark(profile);
-    serverStarted = serverState.started;
-
-    benchmarkStarted = true;
-    const runResult = await runBenchmarkInPi(profile, runDirectory, { signal: controller.signal });
-
-    let speedMetrics = null;
-    let speedMetricsError = null;
-    if (!runResult.error) {
-      try {
-        speedMetrics = await queryServerMetrics(profile);
-      } catch (err) {
-        // Non-fatal: speed metrics are a supplementary measurement, not the
-        // benchmark itself. Don't poison the run result; surface it as a note.
-        speedMetricsError = err.message;
-      }
-    }
-
-    metadata = await finalizeBenchmarkRun(runDirectory, runResult, speedMetrics, speedMetricsError);
-    renderBenchmarkSummary(metadata);
-  } catch (err) {
-    const failedResult = {
-      error: { message: err.message },
-      wallClockMs: null,
-      agentTurns: 0,
-      promptTokens: 0,
-      completionTokens: 0,
-      totalTokens: 0,
-      cacheRead: 0,
-      cacheWrite: 0,
-      toolCalls: 0,
-      toolResults: 0,
-      perTurn: [],
-    };
-    metadata = await finalizeBenchmarkRun(runDirectory, failedResult, null);
-    renderBenchmarkSummary(metadata);
-  } finally {
-    process.removeListener("SIGINT", onSigint);
-    if (serverStarted && !options.keepServer) {
-      const backend = backendFor(profile.backend);
-      if (backend.type !== "managed-server") {
-        const result = await stopProfile(profile);
-        console.log(result.stopped ? pc.green(`[stop] ${result.message}`) : pc.dim(`[stop] ${result.message}`));
-      }
-    }
-    if (benchmarkStarted) {
-      const unloadResult = await unloadModelFromServer(profile);
-      if (!unloadResult.unloaded && unloadResult.error) {
-        console.log(pc.yellow(`[unload] ${unloadResult.backend}: ${unloadResult.error}`));
-      } else if (!unloadResult.unloaded && unloadResult.reason) {
-        console.log(pc.dim(`[unload] ${unloadResult.backend}: ${unloadResult.reason}`));
-      }
-    }
-  }
-
-  return metadata;
-}
-
-// ── Shared benchmark selection ────────────────────────────────────────────
-
-async function selectBenchmark(prompt, repoPath) {
-  const kind = await prompt.choice("Benchmark category", [
-    { value: "visual", label: "Visual Benchmark", hint: "HTML/CSS/JS animation benchmarks" },
-    { value: "data-science", label: "Data Science", hint: "Analysis and charting benchmarks" },
-  ], "visual");
-
-  const benchDir = join(repoPath, "benchmarks");
-  const benchmarks = (await loadBenchmarks(benchDir)).filter((b) => b.kind === kind);
-  if (benchmarks.length === 0) {
-    console.log(pc.yellow(`No ${kind} benchmarks found in ${benchDir}`));
-    return null;
-  }
-  const benchmarkId = await prompt.choice("Prompt", benchmarks.map((b) => ({
-    value: b.id, label: b.title, hint: b.description || b.id,
-  })), benchmarks[0].id);
-  const benchmark = benchmarks.find((b) => b.id === benchmarkId);
-  if (!benchmark) return null;
-  return { kind, benchmark };
-}
-
-// ── Benchmark from a selected profile (from model picker) ────────────────
-
-export async function benchmarkForProfile(profile) {
-  await ensureDirs();
-  const prompt = createPrompt();
-  try {
-    const repoPath = await linkBenchmarkRepo(prompt);
-    if (!repoPath) return;
-
-    const selected = await selectBenchmark(prompt, repoPath);
-    if (!selected) return;
-    const { kind, benchmark: selectedBenchmark } = selected;
-
-    const modelId = profile.modelAlias;
-    const modelSource = benchmarkModelSource(profile);
-    const backendLabel = backendFor(profile.backend).label;
-
-    const canRun = modelSource !== "cloud";
-    const action = await chooseBenchmarkAction(prompt, canRun);
-
-    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
-
-    if (action === "run") {
-      return await runPreparedBenchmark(profile, runDirectory);
-    }
-
-    return runDirectory;
-  } finally {
-    prompt.close();
-  }
-}
-
-// ── Standalone benchmark flow (offgrid-ai benchmark) ──────────────────────
-
-export async function benchmarkFlow() {
-  await ensureDirs();
-  const prompt = createPrompt();
-  try {
-    const repoPath = await linkBenchmarkRepo(prompt);
-    if (!repoPath) return;
-
-    const selected = await selectBenchmark(prompt, repoPath);
-    if (!selected) return;
-    const { kind, benchmark: selectedBenchmark } = selected;
-
-    const profiles = await loadProfiles();
-    const source = await prompt.choice("Model source", [
-      { value: "profile", label: "Use existing profile", hint: "Pick a saved offgrid-ai profile" },
-      { value: "cloud", label: "Custom / cloud", hint: "Free-form model label for cloud runs" },
-    ], "profile");
-
-    let modelId, modelSource, backendLabel, profile;
-
-    if (source === "profile") {
-      if (profiles.length === 0) {
-        console.log(pc.yellow("No profiles yet. Run: offgrid-ai models"));
-        return;
-      }
-      const profileId = await prompt.choice("Profile", profiles.map((p) => ({
-        value: p.id, label: p.label, hint: `${backendFor(p.backend).label} · ${p.modelAlias}`,
-      })), profiles[0].id);
-      profile = profiles.find((p) => p.id === profileId);
-      if (!profile) return;
-      modelId = profile.modelAlias;
-      modelSource = benchmarkModelSource(profile);
-      backendLabel = backendFor(profile.backend).label;
-    } else {
-      backendLabel = await prompt.text("Backend label", "cloud");
-      modelId = await prompt.text("Model name", "");
-      if (!modelId) { console.log(pc.yellow("Model name is required.")); return; }
-      modelSource = "cloud";
-    }
-
-    const canRun = modelSource !== "cloud" && profile != null;
-    const action = await chooseBenchmarkAction(prompt, canRun);
-
-    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
-
-    if (action === "run" && profile) {
-      return await runPreparedBenchmark(profile, runDirectory);
-    }
-
-    return runDirectory;
-  } finally {
-    prompt.close();
-  }
-}

package/src/benchmark/metrics.mjs

@@ -1,113 +0,0 @@
-// ── Backend-aware server speed metrics ───────────────────────────────────────
-
-import { backendFor } from "../backends.mjs";
-
-const BENCH_SPEED_PROMPT = "Write a one-sentence summary of machine learning.";
-const SPEED_QUERY_TIMEOUT_MS = 120_000;
-const SPEED_QUERY_MAX_TOKENS = 64;
-
-export async function queryServerMetrics(profile) {
-  const backend = backendFor(profile.backend);
-
-  if (backend.id === "llama-cpp" || backend.id === "llama-cpp-mtp") {
-    return await queryLlamaCppMetrics(profile);
-  }
-  if (backend.id === "omlx") {
-    return await queryOmlxMetrics(profile);
-  }
-
-  throw new Error(`Unsupported backend for benchmark speed metrics: ${backend.id}`);
-}
-
-async function queryLlamaCppMetrics(profile) {
-  const body = {
-    model: profile.modelAlias,
-    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
-    stream: false,
-    max_tokens: SPEED_QUERY_MAX_TOKENS,
-  };
-
-  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(body),
-    signal: AbortSignal.timeout(SPEED_QUERY_TIMEOUT_MS),
-  });
-
-  if (!response.ok) {
-    throw new Error(`llama.cpp speed query failed: ${response.status} ${response.statusText}`);
-  }
-
-  const data = await response.json();
-  const timings = data.timings;
-  if (!timings || typeof timings.prompt_per_second !== "number" || typeof timings.predicted_per_second !== "number") {
-    throw new Error("llama.cpp response did not include usable timings object");
-  }
-  const draftN = timings.draft_n;
-  const draftAccepted = timings.draft_n_accepted;
-
-  return {
-    prefillTokensPerSecond: timings.prompt_per_second ?? null,
-    generationTokensPerSecond: timings.predicted_per_second ?? null,
-    ttftMs: timings.prompt_ms ?? null,
-    modelLoadMs: null,
-    speculativeDecodeAcceptance: (draftN && Number.isFinite(draftAccepted) && Number.isFinite(draftN) && draftN > 0)
-      ? draftAccepted / draftN
-      : null,
-    kvCacheTokens: timings.cache_n ?? null,
-    metricSource: "llama.cpp /v1/chat/completions timings",
-  };
-}
-
-async function queryOmlxMetrics(profile) {
-  const body = {
-    model: profile.modelAlias,
-    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
-    stream: true,
-    stream_options: { include_usage: true },
-    max_tokens: SPEED_QUERY_MAX_TOKENS,
-  };
-
-  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(body),
-    signal: AbortSignal.timeout(SPEED_QUERY_TIMEOUT_MS),
-  });
-
-  if (!response.ok) {
-    throw new Error(`oMLX speed query failed: ${response.status} ${response.statusText}`);
-  }
-
-  const text = await response.text();
-  let usage = null;
-  for (const line of text.split("\n").reverse()) {
-    const trimmed = line.trim();
-    if (!trimmed || !trimmed.startsWith("data:")) continue;
-    const payload = trimmed.slice(5).trim();
-    if (payload === "[DONE]") continue;
-    try {
-      const chunk = JSON.parse(payload);
-      if (chunk.usage) {
-        usage = chunk.usage;
-        break;
-      }
-    } catch {
-      // Ignore malformed SSE chunks.
-    }
-  }
-
-  if (!usage) {
-    throw new Error("oMLX speed query did not return usage in streaming response");
-  }
-
-  return {
-    prefillTokensPerSecond: usage.prompt_tokens_per_second ?? null,
-    generationTokensPerSecond: usage.generation_tokens_per_second ?? null,
-    ttftMs: usage.time_to_first_token != null ? usage.time_to_first_token * 1000 : null,
-    modelLoadMs: null,
-    speculativeDecodeAcceptance: null,
-    kvCacheTokens: usage.prompt_tokens_details?.cached_tokens ?? null,
-    metricSource: "oMLX /v1/chat/completions streaming include_usage",
-  };
-}

package/src/benchmark/prepare.mjs

@@ -1,118 +0,0 @@
-// ── Create a benchmark run directory ────────────────────────────────────────
-
-import { mkdir, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { pc, renderRows, renderSection } from "../ui.mjs";
-import { slugModelId, createRunId } from "./shared.mjs";
-import { parseModelName } from "../model-name.mjs";
-
-function harnessDisplayName(id) {
-  if (id === "pi") return "Pi";
-  return String(id).replace(/[-_]+/gu, " ").replace(/\b\w/gu, (char) => char.toUpperCase());
-}
-
-function intendedRunnerForProfile(profile) {
-  if (!profile) return "your tool";
-  const harnessEntries = Object.entries(profile.harnesses ?? {}).filter(([, config]) => config?.enabled !== false);
-  const [id] = harnessEntries.find(([key]) => key === "pi") ?? harnessEntries[0] ?? ["pi"];
-  return harnessDisplayName(id);
-}
-
-function printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel }) {
-  const runCommand = profile ? `offgrid-ai run ${profile.id}` : null;
-  const runnerCommand = runCommand ?? `Open ${runnerLabel} for ${modelId}`;
-
-  console.log("");
-  console.log(pc.bold("Next steps"));
-  console.log(`  1. Open the gallery. If it is not running: ${pc.cyan(`cd ${repoPath} && npm run dev`)}`);
-  console.log(`  2. ${pc.cyan(`cd ${runDirectory}`)}`);
-  console.log(`  3. ${pc.cyan(runnerCommand)}, then copy this run's prompt from the gallery and paste it into ${runnerLabel}`);
-}
-
-export async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps = true }) {
-  const toolPrompt = benchmark.prompt;
-  const now = new Date();
-  const runId = createRunId(now);
-  const modelSlug = slugModelId(modelId);
-  const runnerLabel = intendedRunnerForProfile(profile);
-  const runsDir = join(repoPath, "runs");
-  const benchmarkDirectory = join(runsDir, benchmark.id);
-  const modelDirectory = join(benchmarkDirectory, modelSlug);
-  const runDirectory = join(modelDirectory, runId);
-
-  await mkdir(runDirectory, { recursive: true });
-
-  const isDs = kind === "data-science";
-  const baseAssets = {
-    metadata: "metadata.json",
-    prompt: "prompt.md",
-  };
-  const metadata = {
-    schemaVersion: 1,
-    kind,
-    runId,
-    benchmark: { id: benchmark.id, title: benchmark.title, description: benchmark.description, prompt: benchmark.prompt },
-    model: { id: modelId, slug: modelSlug, displayName: parseModelName(modelId, modelSource === "omlx" ? "omlx" : "local-gguf").display },
-    status: "prepared",
-    createdAt: now.toISOString(),
-    updatedAt: now.toISOString(),
-    preparedAt: now.toISOString(),
-    runDirectory,
-    assets: isDs
-      ? { ...baseAssets, ds: { notebook: "analysis.ipynb", summary: "summary.json", chartDistribution: "chart-distribution.png", chartTreatmentEffect: "chart-treatment-effect.png", chartCompletionRates: "chart-completion-rates.png" } }
-      : { ...baseAssets, html: "index.html", preview: "preview.png", video: "preview.webm" },
-    runner: {
-      mode: modelSource === "cloud" ? "manual" : "external",
-      intendedRunner: profile ? runnerLabel : undefined,
-      ...(profile?.harnesses?.pi || runnerLabel === "Pi" ? { tool: "pi" } : {}),
-      ...(modelSource ? { modelSource } : {}),
-      ...(backendLabel ? { backendLabel } : {}),
-      ...(profile?.baseUrl ? { baseUrl: profile.baseUrl } : {}),
-      model: modelId,
-      retries: 0,
-      tokenMetrics: {
-        reported: false,
-        promptTokens: 0,
-        completionTokens: 0,
-        totalTokens: 0,
-      },
-      speedMetrics: {
-        prefillTokensPerSecond: null,
-        generationTokensPerSecond: null,
-        ttftMs: null,
-        modelLoadMs: null,
-        speculativeDecodeAcceptance: null,
-        kvCacheTokens: null,
-      },
-      metricSource: null,
-    },
-    results: {
-      wallClockMs: null,
-      agentTurns: 0,
-      toolCalls: 0,
-      toolResults: 0,
-      success: false,
-      outputFiles: [],
-      perTurn: [],
-    },
-  };
-
-  await writeFile(join(runDirectory, "metadata.json"), JSON.stringify(metadata, null, 2) + "\n", "utf8");
-  await writeFile(join(runDirectory, "prompt.md"), toolPrompt + "\n", "utf8");
-
-  console.log("");
-  console.log(pc.green("✓ Run slot prepared"));
-  console.log(renderSection("Run", renderRows([
-    ["Directory", pc.cyan(runDirectory)],
-    ["Benchmark", benchmark.title],
-    ["Kind", kind],
-    ["Model", pc.bold(modelId)],
-    ["Source", backendLabel || modelSource],
-  ])));
-
-  if (showNextSteps) {
-    printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel });
-  }
-
-  return runDirectory;
-}