offgrid-ai 0.7.4 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "offgrid-ai",
-  "version": "0.7.4",
+  "version": "0.8.0",
   "description": "Privacy-first CLI for running local LLMs — discover, configure, run, benchmark",
   "author": "Eeshan Srivastava (https://eeshans.com)",
   "type": "module",

package/src/benchmark.mjs

@@ -3,10 +3,12 @@ import { existsSync } from "node:fs";
 import { createHash } from "node:crypto";
 import { join, resolve } from "node:path";
 import { homedir } from "node:os";
-import { execFile } from "node:child_process";
+import { spawn, execFile } from "node:child_process";
 import { promisify } from "node:util";
 import { ensureDirs, loadConfig, saveConfig } from "./config.mjs";
 import { backendFor } from "./backends.mjs";
+import { hasPi, hasPiModel, syncPiConfig } from "./harness-pi.mjs";
+import { serverReady, startServer, waitForReady, stopProfile } from "./process.mjs";
 import { pc, createPrompt, renderRows, renderSection } from "./ui.mjs";
 
 const execFileAsync = promisify(execFile);
@@ -27,7 +29,7 @@ export function createRunId(date = new Date()) {
   return date.toISOString().replace(/:/gu, "-").replace(/\./gu, "-");
 }
 
-export function buildToolPrompt(benchmark, kind) {
+export function buildToolPrompt(benchmark) {
   return benchmark.prompt;
 }
 
@@ -151,8 +153,8 @@ function printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, run
   console.log(`  3. ${pc.cyan(runnerCommand)}, then copy this run's prompt from the gallery and paste it into ${runnerLabel}`);
 }
 
-async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSource, backendLabel, profile }) {
-  const toolPrompt = buildToolPrompt(benchmark, kind);
+export async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps = true }) {
+  const toolPrompt = buildToolPrompt(benchmark);
   const now = new Date();
   const runId = createRunId(now);
   const modelSlug = slugModelId(modelId);
@@ -165,6 +167,13 @@ async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSo
   await mkdir(runDirectory, { recursive: true });
 
   const isDs = kind === "data-science";
+  const baseAssets = {
+    metadata: "metadata.json",
+    prompt: "prompt.md",
+    rawResponse: "response.raw.txt",
+    stream: "stream.ndjson",
+    stderr: "stderr.log",
+  };
   const metadata = {
     schemaVersion: 1,
     kind,
@@ -177,8 +186,8 @@ async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSo
     preparedAt: now.toISOString(),
     runDirectory,
     assets: isDs
-      ? { metadata: "metadata.json", prompt: "prompt.md", rawResponse: "response.raw.txt", ds: { notebook: "analysis.ipynb", summary: "summary.json", chartDistribution: "chart-distribution.png", chartTreatmentEffect: "chart-treatment-effect.png", chartCompletionRates: "chart-completion-rates.png" } }
-      : { metadata: "metadata.json", prompt: "prompt.md", html: "index.html", preview: "preview.png", video: "preview.webm", rawResponse: "response.raw.txt" },
+      ? { ...baseAssets, ds: { notebook: "analysis.ipynb", summary: "summary.json", chartDistribution: "chart-distribution.png", chartTreatmentEffect: "chart-treatment-effect.png", chartCompletionRates: "chart-completion-rates.png" } }
+      : { ...baseAssets, html: "index.html", preview: "preview.png", video: "preview.webm" },
     runner: {
       mode: modelSource === "cloud" ? "manual" : "external",
       intendedRunner: profile ? runnerLabel : undefined,
@@ -188,7 +197,30 @@ async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSo
       ...(profile?.baseUrl ? { baseUrl: profile.baseUrl } : {}),
       model: modelId,
       retries: 0,
-      tokenMetrics: { reported: false },
+      tokenMetrics: {
+        reported: false,
+        promptTokens: 0,
+        completionTokens: 0,
+        totalTokens: 0,
+      },
+      speedMetrics: {
+        prefillTokensPerSecond: null,
+        generationTokensPerSecond: null,
+        ttftMs: null,
+        modelLoadMs: null,
+        speculativeDecodeAcceptance: null,
+        kvCacheTokens: null,
+      },
+      metricSource: null,
+    },
+    results: {
+      wallClockMs: null,
+      agentTurns: 0,
+      toolCalls: 0,
+      toolResults: 0,
+      success: false,
+      outputFiles: [],
+      perTurn: [],
     },
   };
 
@@ -205,11 +237,712 @@ async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSo
     ["Source", backendLabel || modelSource],
   ])));
 
-  printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel });
+  if (showNextSteps) {
+    printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel });
+  }
 
   return runDirectory;
 }
 
+// ── Run benchmark in Pi (non-interactive JSON mode) ───────────────────────
+
+const BENCH_COLORS = {
+  thinking: pc.magenta,
+  text: pc.green,
+  tool: pc.yellow,
+  toolOutput: pc.dim,
+  error: pc.red,
+  info: pc.cyan,
+  dim: pc.dim,
+};
+
+function formatToolCall(toolCall) {
+  const path = toolCall.arguments?.path || toolCall.arguments?.file_path || toolCall.arguments?.filename || "";
+  const summary = path ? ` → ${path}` : "";
+  return `[toolCall] ${toolCall.name}${summary}`;
+}
+
+function renderStreamEvent(parsed, state) {
+  const type = parsed.type;
+
+  switch (type) {
+    case "session":
+      console.log(BENCH_COLORS.dim(`[session] ${parsed.id}`));
+      break;
+    case "agent_start":
+      console.log(BENCH_COLORS.dim("[agent_start]"));
+      break;
+    case "turn_start": {
+      state.turn += 1;
+      console.log(BENCH_COLORS.info(`\n[turn ${state.turn}]`));
+      break;
+    }
+    case "message_start": {
+      const msg = parsed.message;
+      if (msg?.role === "assistant" && msg.provider && msg.model) {
+        console.log(BENCH_COLORS.info(`[assistant] ${msg.provider}/${msg.model}`));
+      }
+      break;
+    }
+    case "message_update": {
+      const evt = parsed.assistantMessageEvent;
+      if (!evt) return;
+      const subtype = String(evt.type ?? "").replace(/_/gu, "");
+      if (subtype === "thinkingstart" || subtype === "thinkingdelta") {
+        process.stdout.write(BENCH_COLORS.thinking(evt.delta || ""));
+      } else if (subtype === "textstart" || subtype === "textdelta") {
+        process.stdout.write(BENCH_COLORS.text(evt.delta || ""));
+      } else if (subtype === "toolcallstart") {
+        console.log(BENCH_COLORS.tool("\n[tool_call_start]"));
+      } else if (subtype === "toolcalldelta") {
+        process.stdout.write(BENCH_COLORS.tool(evt.delta || ""));
+      } else if (subtype === "toolcallend") {
+        console.log(BENCH_COLORS.tool("[tool_call_end]"));
+      }
+      break;
+    }
+    case "message_end": {
+      const msg = parsed.message;
+      if (msg?.role === "assistant" && Array.isArray(msg.content)) {
+        for (const item of msg.content) {
+          if (item.type === "toolCall") {
+            console.log(BENCH_COLORS.tool(`\n${formatToolCall(item)}`));
+          }
+        }
+      }
+      break;
+    }
+    case "tool_execution_start":
+      console.log(BENCH_COLORS.tool(`\n[exec] ${parsed.toolName}`));
+      break;
+    case "tool_execution_update":
+      if (parsed.content) {
+        process.stdout.write(BENCH_COLORS.toolOutput(parsed.content));
+      }
+      break;
+    case "tool_execution_end":
+      console.log(BENCH_COLORS.tool(`[exec done] ${parsed.toolName}`));
+      break;
+    case "toolResult": {
+      const errorFlag = parsed.isError ? BENCH_COLORS.error(" error") : "";
+      console.log(BENCH_COLORS.tool(`\n[result] ${parsed.toolName}${errorFlag}`));
+      break;
+    }
+    case "agent_end":
+      console.log(BENCH_COLORS.dim("\n[agent_end]"));
+      break;
+    default:
+      break;
+  }
+}
+
+export function piModelString(profile) {
+  return profile.harnesses?.pi?.model ?? `${profile.providerId}/${profile.modelAlias}`;
+}
+
+export async function runBenchmarkInPi(profile, runDirectory, { signal } = {}) {
+  const model = piModelString(profile);
+  const args = ["--model", model, "--mode", "json", "-p", "@prompt.md"];
+
+  const child = spawn("pi", args, {
+    cwd: runDirectory,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const runResult = {
+    model,
+    exitCode: null,
+    wallClockMs: null,
+    agentTurns: 0,
+    promptTokens: 0,
+    completionTokens: 0,
+    totalTokens: 0,
+    cacheRead: 0,
+    cacheWrite: 0,
+    toolCalls: 0,
+    toolResults: 0,
+    perTurn: [],
+    rawResponseLines: [],
+    error: null,
+  };
+
+  let streamBuffer = "";
+  let responseBuffer = "";
+  let currentTurnStartMs = null;
+  let lastTurnEndMs = null;
+  let runStartMs = null;
+  let firstEventMs = null;
+  let lastEventMs = null;
+  let cancelled = false;
+
+  const streamPath = join(runDirectory, "stream.ndjson");
+  const stderrPath = join(runDirectory, "stderr.log");
+  const responsePath = join(runDirectory, "response.raw.txt");
+
+  const streamHandle = await openFileHandle(streamPath, "w");
+  const stderrHandle = await openFileHandle(stderrPath, "w");
+
+  const renderState = { turn: 0 };
+
+  function appendResponse(text) {
+    responseBuffer += text;
+  }
+
+  function flushResponse() {
+    if (responseBuffer) {
+      runResult.rawResponseLines.push(responseBuffer);
+      responseBuffer = "";
+    }
+  }
+
+  function updateTimeBounds(timestamp) {
+    if (!timestamp) return;
+    if (firstEventMs === null) firstEventMs = timestamp;
+    lastEventMs = timestamp;
+  }
+
+  function beginTurn() {
+    runResult.agentTurns += 1;
+    currentTurnStartMs = lastTurnEndMs ?? runStartMs ?? null;
+  }
+
+  function endTurn(usage, timestamp) {
+    const turnEndMs = timestamp ?? null;
+    const wallClockMs = currentTurnStartMs && turnEndMs ? turnEndMs - currentTurnStartMs : null;
+    runResult.perTurn.push({
+      turn: runResult.agentTurns,
+      inputTokens: usage?.input ?? 0,
+      outputTokens: usage?.output ?? 0,
+      cacheRead: usage?.cacheRead ?? 0,
+      cacheWrite: usage?.cacheWrite ?? 0,
+      wallClockMs,
+      toolCalls: 0,
+    });
+    if (turnEndMs) lastTurnEndMs = turnEndMs;
+    currentTurnStartMs = null;
+  }
+
+  function processLine(line) {
+    if (!line.trim()) return;
+    streamHandle.write(line + "\n");
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (err) {
+      console.log(BENCH_COLORS.error(`[parse error] ${err.message}`));
+      return;
+    }
+
+    const timestamp = extractTimestamp(parsed);
+    updateTimeBounds(timestamp);
+
+    renderStreamEvent(parsed, renderState);
+
+    if (parsed.type === "session" || parsed.type === "agent_start") {
+      if (timestamp && runStartMs === null) runStartMs = timestamp;
+    }
+
+    if (parsed.type === "turn_start") {
+      beginTurn();
+    }
+
+    if (parsed.type === "turn_end" && parsed.message?.usage) {
+      const usage = parsed.message.usage;
+      runResult.promptTokens += usage.input ?? 0;
+      runResult.completionTokens += usage.output ?? 0;
+      runResult.totalTokens += usage.totalTokens ?? 0;
+      runResult.cacheRead += usage.cacheRead ?? 0;
+      runResult.cacheWrite += usage.cacheWrite ?? 0;
+      endTurn(usage, timestamp);
+    }
+
+    if (parsed.type === "message_update" && parsed.assistantMessageEvent) {
+      const evt = parsed.assistantMessageEvent;
+      const subtype = String(evt.type ?? "").replace(/_/gu, "");
+      if (subtype === "thinkingdelta" || subtype === "textdelta") {
+        appendResponse(evt.delta || "");
+      }
+    }
+
+    if (parsed.type === "message_end" && parsed.message?.role === "assistant") {
+      flushResponse();
+      const content = parsed.message.content ?? [];
+      for (const item of content) {
+        if (item.type === "toolCall") {
+          runResult.toolCalls += 1;
+          appendResponse(`\n${formatToolCall(item)}\n`);
+          const currentTurn = runResult.perTurn[runResult.perTurn.length - 1];
+          if (currentTurn) currentTurn.toolCalls += 1;
+        }
+      }
+    }
+
+    if (parsed.type === "toolResult") {
+      runResult.toolResults += 1;
+      const status = parsed.isError ? "error" : "ok";
+      appendResponse(`\n[toolResult] ${parsed.toolName} (${status})\n`);
+    }
+
+    if (parsed.type === "agent_end") {
+      flushResponse();
+    }
+  }
+
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => {
+    streamBuffer += chunk;
+    const lines = streamBuffer.split("\n");
+    streamBuffer = lines.pop();
+    for (const line of lines) {
+      processLine(line);
+    }
+  });
+
+  child.stderr.setEncoding("utf8");
+  child.stderr.on("data", (chunk) => {
+    stderrHandle.write(chunk);
+  });
+
+  const abortListener = () => {
+    if (cancelled) return;
+    cancelled = true;
+    console.log(BENCH_COLORS.error("\n\n[Cancelled by user]"));
+    child.kill("SIGTERM");
+  };
+
+  if (signal) {
+    signal.addEventListener("abort", abortListener);
+  }
+
+  return new Promise((resolve) => {
+    child.on("exit", async (code) => {
+      if (signal) signal.removeEventListener("abort", abortListener);
+      if (streamBuffer.trim()) {
+        processLine(streamBuffer);
+      }
+      flushResponse();
+      await streamHandle.close();
+      await stderrHandle.close();
+      await writeFile(responsePath, runResult.rawResponseLines.join(""), "utf8");
+
+      runResult.exitCode = code ?? 0;
+      if (firstEventMs !== null && lastEventMs !== null) {
+        runResult.wallClockMs = lastEventMs - firstEventMs;
+      }
+
+      if (cancelled) {
+        runResult.error = { message: "Cancelled by user" };
+        resolve(runResult);
+        return;
+      }
+
+      if (runResult.exitCode !== 0) {
+        runResult.error = { message: `Pi exited with code ${runResult.exitCode}` };
+        resolve(runResult);
+        return;
+      }
+
+      resolve(runResult);
+    });
+
+    child.on("error", async (err) => {
+      if (signal) signal.removeEventListener("abort", abortListener);
+      await streamHandle.close();
+      await stderrHandle.close();
+      runResult.error = { message: err.message };
+      resolve(runResult);
+    });
+  });
+}
+
+function extractTimestamp(event) {
+  const raw = event?.message?.timestamp ?? event?.timestamp ?? event?.assistantMessageEvent?.partial?.timestamp;
+  if (typeof raw === "number") return raw;
+  if (typeof raw === "string") {
+    const parsed = Date.parse(raw);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  const iso = event?.message?.createdAt ?? event?.createdAt ?? event?.created_at;
+  if (typeof iso === "string") {
+    const parsed = Date.parse(iso);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return null;
+}
+
+async function openFileHandle(path, flags) {
+  const { open } = await import("node:fs/promises");
+  return open(path, flags);
+}
+
+// ── Backend-aware server speed metrics ───────────────────────────────────
+
+const BENCH_SPEED_PROMPT = "Write a one-sentence summary of machine learning.";
+
+export async function queryServerMetrics(profile) {
+  const backend = backendFor(profile.backend);
+
+  if (backend.id === "llama-cpp" || backend.id === "llama-cpp-mtp") {
+    return await queryLlamaCppMetrics(profile);
+  }
+  if (backend.id === "omlx") {
+    return await queryOmlxMetrics(profile);
+  }
+  if (backend.id === "ollama") {
+    return await queryOllamaMetrics(profile);
+  }
+
+  throw new Error(`Unsupported backend for benchmark speed metrics: ${backend.id}`);
+}
+
+async function queryLlamaCppMetrics(profile) {
+  const body = {
+    model: profile.modelAlias,
+    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
+    stream: false,
+  };
+
+  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`llama.cpp speed query failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = await response.json();
+  const timings = data.timings;
+  if (!timings || typeof timings.prompt_per_second !== "number" || typeof timings.predicted_per_second !== "number") {
+    throw new Error("llama.cpp response did not include usable timings object");
+  }
+  const draftN = timings.draft_n;
+  const draftAccepted = timings.draft_n_accepted;
+
+  return {
+    prefillTokensPerSecond: timings.prompt_per_second ?? null,
+    generationTokensPerSecond: timings.predicted_per_second ?? null,
+    ttftMs: timings.prompt_ms ?? null,
+    modelLoadMs: null,
+    speculativeDecodeAcceptance: (draftN && Number.isFinite(draftAccepted) && Number.isFinite(draftN) && draftN > 0)
+      ? draftAccepted / draftN
+      : null,
+    kvCacheTokens: timings.cache_n ?? null,
+    metricSource: "llama.cpp /v1/chat/completions timings",
+  };
+}
+
+async function queryOmlxMetrics(profile) {
+  const body = {
+    model: profile.modelAlias,
+    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
+    stream: true,
+    stream_options: { include_usage: true },
+  };
+
+  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`oMLX speed query failed: ${response.status} ${response.statusText}`);
+  }
+
+  const text = await response.text();
+  let usage = null;
+  for (const line of text.split("\n").reverse()) {
+    const trimmed = line.trim();
+    if (!trimmed || !trimmed.startsWith("data:")) continue;
+    const payload = trimmed.slice(5).trim();
+    if (payload === "[DONE]") continue;
+    try {
+      const chunk = JSON.parse(payload);
+      if (chunk.usage) {
+        usage = chunk.usage;
+        break;
+      }
+    } catch {
+      // Ignore malformed SSE chunks.
+    }
+  }
+
+  if (!usage) {
+    throw new Error("oMLX speed query did not return usage in streaming response");
+  }
+
+  return {
+    prefillTokensPerSecond: usage.prompt_tokens_per_second ?? null,
+    generationTokensPerSecond: usage.generation_tokens_per_second ?? null,
+    ttftMs: usage.time_to_first_token != null ? usage.time_to_first_token * 1000 : null,
+    modelLoadMs: null,
+    speculativeDecodeAcceptance: null,
+    kvCacheTokens: usage.prompt_tokens_details?.cached_tokens ?? null,
+    metricSource: "oMLX /v1/chat/completions streaming include_usage",
+  };
+}
+
+async function queryOllamaMetrics(profile) {
+  const body = {
+    model: profile.modelAlias,
+    prompt: BENCH_SPEED_PROMPT,
+    stream: false,
+  };
+
+  const apiBaseUrl = (profile.baseUrl
+    ? profile.baseUrl.replace(/\/v1\/?$/u, "")
+    : backendFor(profile.backend).apiBaseUrl).replace(/\/$/u, "");
+
+  const response = await fetch(`${apiBaseUrl}/api/generate`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Ollama speed query failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = await response.json();
+  const promptEvalNs = data.prompt_eval_duration ?? 0;
+  const evalNs = data.eval_duration ?? 0;
+  const loadNs = data.load_duration ?? 0;
+
+  const promptEvalCount = data.prompt_eval_count ?? 0;
+  const evalCount = data.eval_count ?? 0;
+
+  return {
+    prefillTokensPerSecond: promptEvalNs > 0 ? (promptEvalCount / (promptEvalNs / 1e9)) : null,
+    generationTokensPerSecond: evalNs > 0 ? (evalCount / (evalNs / 1e9)) : null,
+    ttftMs: promptEvalNs / 1e6,
+    modelLoadMs: loadNs / 1e6,
+    speculativeDecodeAcceptance: null,
+    kvCacheTokens: null,
+    metricSource: "Ollama /api/generate",
+  };
+}
+
+// ── Finalize benchmark run metadata ──────────────────────────────────────
+
+export async function finalizeBenchmarkRun(runDirectory, runResult, speedMetrics) {
+  const metadataPath = join(runDirectory, "metadata.json");
+  const metadata = JSON.parse(await readFile(metadataPath, "utf8"));
+  const now = new Date();
+  const timestamp = now.toISOString();
+
+  const kind = metadata.kind ?? "visual";
+  const isDs = kind === "data-science";
+  const requiredFile = isDs ? "analysis.ipynb" : "index.html";
+  const requiredPath = join(runDirectory, requiredFile);
+
+  const outputFiles = [];
+  for (const candidate of [requiredFile, isDs ? "summary.json" : "preview.png", isDs ? "chart-distribution.png" : "preview.webm", "preview.mp4"]) {
+    if (existsSync(join(runDirectory, candidate))) {
+      outputFiles.push(candidate);
+    }
+  }
+
+  const success = existsSync(requiredPath) && (await readFile(requiredPath, "utf8")).trim().length > 0;
+
+  metadata.status = runResult.error ? "failed" : "completed";
+  metadata.updatedAt = timestamp;
+  if (runResult.error) {
+    metadata.failedAt = timestamp;
+  } else {
+    metadata.completedAt = timestamp;
+  }
+
+  metadata.runner.tokenMetrics = {
+    reported: true,
+    promptTokens: runResult.promptTokens,
+    completionTokens: runResult.completionTokens,
+    totalTokens: runResult.totalTokens,
+  };
+
+  metadata.runner.speedMetrics = speedMetrics;
+  metadata.runner.metricSource = speedMetrics?.metricSource ?? null;
+
+  metadata.results = {
+    wallClockMs: runResult.wallClockMs,
+    agentTurns: runResult.agentTurns,
+    toolCalls: runResult.toolCalls,
+    toolResults: runResult.toolResults,
+    success,
+    outputFiles,
+    perTurn: runResult.perTurn,
+  };
+
+  if (runResult.error) {
+    metadata.error = typeof runResult.error === "string"
+      ? { message: runResult.error }
+      : { message: runResult.error.message ?? "Unknown error", ...(runResult.error.stack ? { stack: runResult.error.stack } : {}) };
+  }
+
+  await writeFile(metadataPath, JSON.stringify(metadata, null, 2) + "\n", "utf8");
+  return metadata;
+}
+
+async function ensureServerForBenchmark(profile) {
+  const backend = backendFor(profile.backend);
+  if (await serverReady(profile.baseUrl)) {
+    console.log(pc.green(`[ready] ${backend.label} at ${profile.baseUrl}`));
+    return { started: false };
+  }
+
+  if (backend.type === "managed-server") {
+    throw new Error(`${backend.label} is not running at ${profile.baseUrl}. Start it and try again.`);
+  }
+
+  console.log(pc.dim(`Starting ${backend.label} for ${profile.label}...`));
+  const state = await startServer(profile);
+  await waitForReady(profile, state?.pid, state?.rawLogPath);
+  console.log(pc.green(`[ready] ${profile.baseUrl}/models`));
+  return { started: true, state };
+}
+
+export async function runPreparedBenchmark(profile, runDirectory, options = {}) {
+  const controller = new AbortController();
+  if (options.signal) {
+    options.signal.addEventListener("abort", () => controller.abort(), { once: true });
+  }
+  let serverStarted = false;
+  let metadata = null;
+
+  const onSigint = () => {
+    controller.abort();
+  };
+  process.on("SIGINT", onSigint);
+
+  try {
+    if (!(await hasPi())) {
+      console.log(pc.yellow("\nPi is not installed. Run prepared for manual execution."));
+      return metadata;
+    }
+
+    const serverState = await ensureServerForBenchmark(profile);
+    serverStarted = serverState.started;
+
+    if (!(await hasPiModel(profile))) {
+      await syncPiConfig(profile);
+    }
+
+    const runResult = await runBenchmarkInPi(profile, runDirectory, { signal: controller.signal });
+
+    let speedMetrics = null;
+    if (!runResult.error) {
+      try {
+        speedMetrics = await queryServerMetrics(profile);
+      } catch (err) {
+        runResult.error = { message: `Speed metrics query failed: ${err.message}` };
+      }
+    }
+
+    metadata = await finalizeBenchmarkRun(runDirectory, runResult, speedMetrics);
+    renderBenchmarkSummary(metadata);
+  } catch (err) {
+    const failedResult = {
+      error: { message: err.message },
+      wallClockMs: null,
+      agentTurns: 0,
+      promptTokens: 0,
+      completionTokens: 0,
+      totalTokens: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      toolCalls: 0,
+      toolResults: 0,
+      perTurn: [],
+    };
+    metadata = await finalizeBenchmarkRun(runDirectory, failedResult, null);
+    renderBenchmarkSummary(metadata);
+  } finally {
+    process.removeListener("SIGINT", onSigint);
+    if (serverStarted && !options.keepServer) {
+      const backend = backendFor(profile.backend);
+      if (backend.type !== "managed-server") {
+        const result = await stopProfile(profile);
+        console.log(result.stopped ? pc.green(`[stop] ${result.message}`) : pc.dim(`[stop] ${result.message}`));
+      }
+    }
+  }
+
+  return metadata;
+}
+
+function formatMetric(value, formatter) {
+  if (value === null || value === undefined || !Number.isFinite(value)) return pc.dim("—");
+  return formatter(value);
+}
+
+function formatMs(ms) {
+  return formatMetric(ms, (n) => (n < 1000 ? `${Math.round(n)} ms` : `${(n / 1000).toFixed(1)} s`));
+}
+
+function formatNumber(n) {
+  return formatMetric(n, (v) => v.toLocaleString());
+}
+
+function formatTokPerSec(n) {
+  return formatMetric(n, (v) => `${v.toFixed(1)} tok/s`);
+}
+
+function formatPercent(n) {
+  return formatMetric(n, (v) => `${(v * 100).toFixed(0)} %`);
+}
+
+export function renderBenchmarkSummary(metadata) {
+  const { status, results, runner, error } = metadata;
+
+  const agentRows = [
+    ["Status", status === "completed" ? pc.green("completed") : pc.red(status ?? "failed")],
+    ["Duration", formatMs(results?.wallClockMs)],
+    ["Agent turns", formatNumber(results?.agentTurns)],
+    ["Input tokens", formatNumber(runner?.tokenMetrics?.promptTokens)],
+    ["Output tokens", formatNumber(runner?.tokenMetrics?.completionTokens)],
+    ["Total tokens", formatNumber(runner?.tokenMetrics?.totalTokens)],
+    ["Tool calls", formatNumber(results?.toolCalls)],
+    ["Tool results", formatNumber(results?.toolResults)],
+    ["Output files", (results?.outputFiles?.length ?? 0) > 0 ? results.outputFiles.join(", ") : pc.dim("—")],
+  ];
+
+  console.log("");
+  console.log(renderSection("Benchmark Result", renderRows(agentRows)));
+
+  if (status === "completed" && runner?.speedMetrics) {
+    const speed = runner.speedMetrics;
+    const speedRows = [
+      ["Prefill tok/s", formatTokPerSec(speed.prefillTokensPerSecond)],
+      ["Generation tok/s", formatTokPerSec(speed.generationTokensPerSecond)],
+      ["TTFT", formatMs(speed.ttftMs)],
+      ["Speculative decode", formatPercent(speed.speculativeDecodeAcceptance)],
+      ["KV cache tokens", formatNumber(speed.kvCacheTokens)],
+      ["Model load time", formatMs(speed.modelLoadMs)],
+      ["Metric source", speed.metricSource ?? pc.dim("—")],
+    ];
+    console.log(renderSection("Speed Metrics", renderRows(speedRows)));
+  } else if (error) {
+    console.log(renderSection("Error", pc.red(error.message ?? "Unknown error")));
+  }
+}
+
+function benchmarkModelSource(profile) {
+  if (!profile) return "cloud";
+  return profile.providerId === "llama-cpp-mtp" ? "llama-cpp-mtp" : profile.backend === "ollama" ? "ollama" : profile.backend === "omlx" ? "omlx" : "llama-cpp";
+}
+
+async function chooseBenchmarkAction(prompt, canRun) {
+  const choices = [
+    { value: "run", label: "Run Benchmark", hint: "Automated with Pi" },
+    { value: "prepare", label: "Prepare Benchmark (manual)", hint: "Copy prompt and run yourself" },
+  ];
+  return await prompt.choice("Action", canRun ? choices : choices.filter((c) => c.value === "prepare"), canRun ? "run" : "prepare");
+}
+
 // ── Benchmark from a selected profile (from model picker) ────────────────
 
 export async function benchmarkForProfile(profile) {
@@ -237,10 +970,19 @@ export async function benchmarkForProfile(profile) {
     if (!selectedBenchmark) return;
 
     const modelId = profile.modelAlias;
-    const modelSource = profile.providerId === "llama-cpp-mtp" ? "llama-cpp-mtp" : profile.backend === "ollama" ? "ollama" : profile.backend === "omlx" ? "omlx" : "llama-cpp";
+    const modelSource = benchmarkModelSource(profile);
     const backendLabel = backendFor(profile.backend).label;
 
-    return await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile });
+    const canRun = (await hasPi()) && modelSource !== "cloud";
+    const action = await chooseBenchmarkAction(prompt, canRun);
+
+    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
+
+    if (action === "run") {
+      return await runPreparedBenchmark(profile, runDirectory);
+    }
+
+    return runDirectory;
   } finally {
     prompt.close();
   }
@@ -294,7 +1036,7 @@ export async function benchmarkFlow() {
       profile = profiles.find((p) => p.id === profileId);
       if (!profile) return;
       modelId = profile.modelAlias;
-      modelSource = profile.providerId === "llama-cpp-mtp" ? "llama-cpp-mtp" : profile.backend === "ollama" ? "ollama" : profile.backend === "omlx" ? "omlx" : "llama-cpp";
+      modelSource = benchmarkModelSource(profile);
       backendLabel = backendFor(profile.backend).label;
     } else {
       backendLabel = await prompt.text("Backend label", "cloud");
@@ -303,7 +1045,16 @@ export async function benchmarkFlow() {
       modelSource = "cloud";
     }
 
-    return await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile });
+    const canRun = (await hasPi()) && modelSource !== "cloud" && profile != null;
+    const action = await chooseBenchmarkAction(prompt, canRun);
+
+    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
+
+    if (action === "run" && profile) {
+      return await runPreparedBenchmark(profile, runDirectory);
+    }
+
+    return runDirectory;
   } finally {
     prompt.close();
   }