offgrid-ai 0.8.15 → 0.9.2

package/src/benchmark/flow.mjs

@@ -0,0 +1,237 @@
+// ── Benchmark command flows ───────────────────────────────────────────────────
+
+import { join } from "node:path";
+import { ensureDirs } from "../config.mjs";
+import { backendFor } from "../backends.mjs";
+import { hasPi, hasPiModel, syncPiConfig } from "../harness-pi.mjs";
+import { serverReady, startServer, waitForReady, stopProfile } from "../process.mjs";
+import { loadProfiles } from "../profiles.mjs";
+import { pc, createPrompt } from "../ui.mjs";
+import { linkBenchmarkRepo } from "./repo.mjs";
+import { loadBenchmarks } from "./shared.mjs";
+import { prepareBenchmarkRun } from "./prepare.mjs";
+import { runBenchmarkInPi } from "./pi-runner.mjs";
+import { queryServerMetrics } from "./metrics.mjs";
+import { unloadModelFromServer } from "./finalize.mjs";
+import { finalizeBenchmarkRun, renderBenchmarkSummary } from "./finalize.mjs";
+
+function benchmarkModelSource(profile) {
+  if (!profile) return "cloud";
+  return profile.providerId === "llama-cpp-mtp" ? "llama-cpp-mtp" : profile.backend === "ollama" ? "ollama" : profile.backend === "omlx" ? "omlx" : "llama-cpp";
+}
+
+async function chooseBenchmarkAction(prompt, canRun) {
+  const choices = [
+    { value: "run", label: "Run Benchmark", hint: "Automated with Pi" },
+    { value: "prepare", label: "Prepare Benchmark (manual)", hint: "Copy prompt and run yourself" },
+  ];
+  return await prompt.choice("Action", canRun ? choices : choices.filter((c) => c.value === "prepare"), canRun ? "run" : "prepare");
+}
+
+async function ensureServerForBenchmark(profile) {
+  const backend = backendFor(profile.backend);
+  if (await serverReady(profile.baseUrl)) {
+    console.log(pc.green(`[ready] ${backend.label} at ${profile.baseUrl}`));
+    return { started: false };
+  }
+
+  if (backend.type === "managed-server") {
+    throw new Error(`${backend.label} is not running at ${profile.baseUrl}. Start it and try again.`);
+  }
+
+  console.log(pc.dim(`Starting ${backend.label} for ${profile.label}...`));
+  const state = await startServer(profile);
+  await waitForReady(profile, state?.pid, state?.rawLogPath);
+  console.log(pc.green(`[ready] ${profile.baseUrl}/models`));
+  return { started: true, state };
+}
+
+export async function runPreparedBenchmark(profile, runDirectory, options = {}) {
+  const controller = new AbortController();
+  if (options.signal) {
+    options.signal.addEventListener("abort", () => controller.abort(), { once: true });
+  }
+  let serverStarted = false;
+  let metadata = null;
+
+  const onSigint = () => {
+    controller.abort();
+  };
+  process.on("SIGINT", onSigint);
+
+  try {
+    if (!(await hasPi())) {
+      console.log(pc.yellow("\nPi is not installed. Run prepared for manual execution."));
+      return metadata;
+    }
+
+    const serverState = await ensureServerForBenchmark(profile);
+    serverStarted = serverState.started;
+
+    if (!(await hasPiModel(profile))) {
+      await syncPiConfig(profile);
+    }
+
+    const runResult = await runBenchmarkInPi(profile, runDirectory, { signal: controller.signal });
+
+    let speedMetrics = null;
+    if (!runResult.error) {
+      try {
+        speedMetrics = await queryServerMetrics(profile);
+      } catch (err) {
+        runResult.error = { message: `Speed metrics query failed: ${err.message}` };
+      }
+    }
+
+    metadata = await finalizeBenchmarkRun(runDirectory, runResult, speedMetrics);
+    renderBenchmarkSummary(metadata);
+  } catch (err) {
+    const failedResult = {
+      error: { message: err.message },
+      wallClockMs: null,
+      agentTurns: 0,
+      promptTokens: 0,
+      completionTokens: 0,
+      totalTokens: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      toolCalls: 0,
+      toolResults: 0,
+      perTurn: [],
+    };
+    metadata = await finalizeBenchmarkRun(runDirectory, failedResult, null);
+    renderBenchmarkSummary(metadata);
+  } finally {
+    process.removeListener("SIGINT", onSigint);
+    if (serverStarted && !options.keepServer) {
+      const backend = backendFor(profile.backend);
+      if (backend.type !== "managed-server") {
+        const result = await stopProfile(profile);
+        console.log(result.stopped ? pc.green(`[stop] ${result.message}`) : pc.dim(`[stop] ${result.message}`));
+      }
+    }
+    const unloadResult = await unloadModelFromServer(profile);
+    if (!unloadResult.unloaded && unloadResult.error) {
+      console.log(pc.yellow(`[unload] ${unloadResult.backend}: ${unloadResult.error}`));
+    } else if (!unloadResult.unloaded && unloadResult.reason) {
+      console.log(pc.dim(`[unload] ${unloadResult.backend}: ${unloadResult.reason}`));
+    }
+  }
+
+  return metadata;
+}
+
+// ── Benchmark from a selected profile (from model picker) ────────────────
+
+export async function benchmarkForProfile(profile) {
+  await ensureDirs();
+  const prompt = createPrompt();
+  try {
+    const repoPath = await linkBenchmarkRepo(prompt);
+    if (!repoPath) return;
+
+    const kind = await prompt.choice("Benchmark category", [
+      { value: "visual", label: "Visual Benchmark", hint: "HTML/CSS/JS animation benchmarks" },
+      { value: "data-science", label: "Data Science", hint: "Analysis and charting benchmarks" },
+    ], "visual");
+
+    const benchDir = join(repoPath, "benchmarks");
+    const benchmarks = (await loadBenchmarks(benchDir)).filter((b) => b.kind === kind);
+    if (benchmarks.length === 0) {
+      console.log(pc.yellow(`No ${kind} benchmarks found in ${benchDir}`));
+      return;
+    }
+    const benchmarkId = await prompt.choice("Prompt", benchmarks.map((b) => ({
+      value: b.id, label: b.title, hint: b.description || b.id,
+    })), benchmarks[0].id);
+    const selectedBenchmark = benchmarks.find((b) => b.id === benchmarkId);
+    if (!selectedBenchmark) return;
+
+    const modelId = profile.modelAlias;
+    const modelSource = benchmarkModelSource(profile);
+    const backendLabel = backendFor(profile.backend).label;
+
+    const canRun = (await hasPi()) && modelSource !== "cloud";
+    const action = await chooseBenchmarkAction(prompt, canRun);
+
+    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
+
+    if (action === "run") {
+      return await runPreparedBenchmark(profile, runDirectory);
+    }
+
+    return runDirectory;
+  } finally {
+    prompt.close();
+  }
+}
+
+// ── Standalone benchmark flow (offgrid-ai benchmark) ──────────────────────
+
+export async function benchmarkFlow() {
+  await ensureDirs();
+
+  const prompt = createPrompt();
+  try {
+    const repoPath = await linkBenchmarkRepo(prompt);
+    if (!repoPath) return;
+
+    const kind = await prompt.choice("Benchmark category", [
+      { value: "visual", label: "Visual Benchmark", hint: "HTML/CSS/JS animation benchmarks" },
+      { value: "data-science", label: "Data Science", hint: "Analysis and charting benchmarks" },
+    ], "visual");
+
+    const benchDir = join(repoPath, "benchmarks");
+    const benchmarks = (await loadBenchmarks(benchDir)).filter((b) => b.kind === kind);
+    if (benchmarks.length === 0) {
+      console.log(pc.yellow(`No ${kind} benchmarks found in ${benchDir}`));
+      return;
+    }
+    const benchmarkId = await prompt.choice("Prompt", benchmarks.map((b) => ({
+      value: b.id, label: b.title, hint: b.description || b.id,
+    })), benchmarks[0].id);
+    const selectedBenchmark = benchmarks.find((b) => b.id === benchmarkId);
+    if (!selectedBenchmark) return;
+
+    const profiles = await loadProfiles();
+    const source = await prompt.choice("Model source", [
+      { value: "profile", label: "Use existing profile", hint: "Pick a saved offgrid-ai profile" },
+      { value: "cloud", label: "Custom / cloud", hint: "Free-form model label for cloud runs" },
+    ], "profile");
+
+    let modelId, modelSource, backendLabel, profile;
+
+    if (source === "profile") {
+      if (profiles.length === 0) {
+        console.log(pc.yellow("No profiles yet. Run: offgrid-ai models"));
+        return;
+      }
+      const profileId = await prompt.choice("Profile", profiles.map((p) => ({
+        value: p.id, label: p.label, hint: `${backendFor(p.backend).label} · ${p.modelAlias}`,
+      })), profiles[0].id);
+      profile = profiles.find((p) => p.id === profileId);
+      if (!profile) return;
+      modelId = profile.modelAlias;
+      modelSource = benchmarkModelSource(profile);
+      backendLabel = backendFor(profile.backend).label;
+    } else {
+      backendLabel = await prompt.text("Backend label", "cloud");
+      modelId = await prompt.text("Model name", "");
+      if (!modelId) { console.log(pc.yellow("Model name is required.")); return; }
+      modelSource = "cloud";
+    }
+
+    const canRun = (await hasPi()) && modelSource !== "cloud" && profile != null;
+    const action = await chooseBenchmarkAction(prompt, canRun);
+
+    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
+
+    if (action === "run" && profile) {
+      return await runPreparedBenchmark(profile, runDirectory);
+    }
+
+    return runDirectory;
+  } finally {
+    prompt.close();
+  }
+}

package/src/benchmark/metrics.mjs

@@ -0,0 +1,152 @@
+// ── Backend-aware server speed metrics ───────────────────────────────────────
+
+import { backendFor } from "../backends.mjs";
+import { apiRootUrl } from "../process.mjs";
+
+const BENCH_SPEED_PROMPT = "Write a one-sentence summary of machine learning.";
+
+export async function queryServerMetrics(profile) {
+  const backend = backendFor(profile.backend);
+
+  if (backend.id === "llama-cpp" || backend.id === "llama-cpp-mtp") {
+    return await queryLlamaCppMetrics(profile);
+  }
+  if (backend.id === "omlx") {
+    return await queryOmlxMetrics(profile);
+  }
+  if (backend.id === "ollama") {
+    return await queryOllamaMetrics(profile);
+  }
+
+  throw new Error(`Unsupported backend for benchmark speed metrics: ${backend.id}`);
+}
+
+async function queryLlamaCppMetrics(profile) {
+  const body = {
+    model: profile.modelAlias,
+    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
+    stream: false,
+  };
+
+  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`llama.cpp speed query failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = await response.json();
+  const timings = data.timings;
+  if (!timings || typeof timings.prompt_per_second !== "number" || typeof timings.predicted_per_second !== "number") {
+    throw new Error("llama.cpp response did not include usable timings object");
+  }
+  const draftN = timings.draft_n;
+  const draftAccepted = timings.draft_n_accepted;
+
+  return {
+    prefillTokensPerSecond: timings.prompt_per_second ?? null,
+    generationTokensPerSecond: timings.predicted_per_second ?? null,
+    ttftMs: timings.prompt_ms ?? null,
+    modelLoadMs: null,
+    speculativeDecodeAcceptance: (draftN && Number.isFinite(draftAccepted) && Number.isFinite(draftN) && draftN > 0)
+      ? draftAccepted / draftN
+      : null,
+    kvCacheTokens: timings.cache_n ?? null,
+    metricSource: "llama.cpp /v1/chat/completions timings",
+  };
+}
+
+async function queryOmlxMetrics(profile) {
+  const body = {
+    model: profile.modelAlias,
+    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
+    stream: true,
+    stream_options: { include_usage: true },
+  };
+
+  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`oMLX speed query failed: ${response.status} ${response.statusText}`);
+  }
+
+  const text = await response.text();
+  let usage = null;
+  for (const line of text.split("\n").reverse()) {
+    const trimmed = line.trim();
+    if (!trimmed || !trimmed.startsWith("data:")) continue;
+    const payload = trimmed.slice(5).trim();
+    if (payload === "[DONE]") continue;
+    try {
+      const chunk = JSON.parse(payload);
+      if (chunk.usage) {
+        usage = chunk.usage;
+        break;
+      }
+    } catch {
+      // Ignore malformed SSE chunks.
+    }
+  }
+
+  if (!usage) {
+    throw new Error("oMLX speed query did not return usage in streaming response");
+  }
+
+  return {
+    prefillTokensPerSecond: usage.prompt_tokens_per_second ?? null,
+    generationTokensPerSecond: usage.generation_tokens_per_second ?? null,
+    ttftMs: usage.time_to_first_token != null ? usage.time_to_first_token * 1000 : null,
+    modelLoadMs: null,
+    speculativeDecodeAcceptance: null,
+    kvCacheTokens: usage.prompt_tokens_details?.cached_tokens ?? null,
+    metricSource: "oMLX /v1/chat/completions streaming include_usage",
+  };
+}
+
+async function queryOllamaMetrics(profile) {
+  const body = {
+    model: profile.modelAlias,
+    prompt: BENCH_SPEED_PROMPT,
+    stream: false,
+  };
+
+  const apiBaseUrl = apiRootUrl(profile.baseUrl || backendFor(profile.backend).apiBaseUrl || "");
+
+  const response = await fetch(`${apiBaseUrl}/api/generate`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(body),
+    signal: AbortSignal.timeout(60000),
+  });
+
+  if (!response.ok) {
+    throw new Error(`Ollama speed query failed: ${response.status} ${response.statusText}`);
+  }
+
+  const data = await response.json();
+  const promptEvalNs = data.prompt_eval_duration ?? 0;
+  const evalNs = data.eval_duration ?? 0;
+  const loadNs = data.load_duration ?? 0;
+
+  const promptEvalCount = data.prompt_eval_count ?? 0;
+  const evalCount = data.eval_count ?? 0;
+
+  return {
+    prefillTokensPerSecond: promptEvalNs > 0 ? (promptEvalCount / (promptEvalNs / 1e9)) : null,
+    generationTokensPerSecond: evalNs > 0 ? (evalCount / (evalNs / 1e9)) : null,
+    ttftMs: promptEvalNs / 1e6,
+    modelLoadMs: loadNs / 1e6,
+    speculativeDecodeAcceptance: null,
+    kvCacheTokens: null,
+    metricSource: "Ollama /api/generate",
+  };
+}

package/src/benchmark/pi-runner.mjs

@@ -0,0 +1,252 @@
+// ── Run benchmark in Pi (non-interactive JSON mode) ───────────────────────────
+
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { spawn } from "node:child_process";
+import {
+  BENCH_COLORS, renderStreamEvent,
+  formatToolCall,
+} from "./stream-renderer.mjs";
+import { piModelString } from "./shared.mjs";
+
+export async function runBenchmarkInPi(profile, runDirectory, { signal } = {}) {
+  const model = piModelString(profile);
+  const args = ["--model", model, "--mode", "json", "-p", "@prompt.md"];
+
+  const child = spawn("pi", args, {
+    cwd: runDirectory,
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const runResult = {
+    model,
+    exitCode: null,
+    wallClockMs: null,
+    agentTurns: 0,
+    promptTokens: 0,
+    completionTokens: 0,
+    totalTokens: 0,
+    cacheRead: 0,
+    cacheWrite: 0,
+    toolCalls: 0,
+    toolResults: 0,
+    perTurn: [],
+    rawResponseLines: [],
+    error: null,
+  };
+
+  let streamBuffer = "";
+  let responseBuffer = "";
+  let currentTurnStartMs = null;
+  let lastTurnEndMs = null;
+  let runStartMs = null;
+  let firstEventMs = null;
+  let lastEventMs = null;
+  let cancelled = false;
+
+  const streamPath = join(runDirectory, "stream.ndjson");
+  const stderrPath = join(runDirectory, "stderr.log");
+  const responsePath = join(runDirectory, "response.raw.txt");
+
+  const streamHandle = await openFileHandle(streamPath, "w");
+  const stderrHandle = await openFileHandle(stderrPath, "w");
+
+  const verbose = Boolean(process.env.OFFGRID_BENCHMARK_VERBOSE);
+  const renderState = {
+    cwd: runDirectory,
+    turn: 0,
+    turnHadToolError: false,
+    modelPrinted: false,
+    activeTool: null,
+    status: { mode: "idle", toolName: null, bytes: 0, tokens: 0 },
+  };
+
+  function appendResponse(text) {
+    responseBuffer += text;
+  }
+
+  function flushResponse() {
+    if (responseBuffer) {
+      runResult.rawResponseLines.push(responseBuffer);
+      responseBuffer = "";
+    }
+  }
+
+  function updateTimeBounds(timestamp) {
+    if (!timestamp) return;
+    if (firstEventMs === null) firstEventMs = timestamp;
+    lastEventMs = timestamp;
+  }
+
+  function beginTurn() {
+    runResult.agentTurns += 1;
+    currentTurnStartMs = lastTurnEndMs ?? runStartMs ?? null;
+  }
+
+  function endTurn(usage, timestamp) {
+    const turnEndMs = timestamp ?? null;
+    const wallClockMs = currentTurnStartMs && turnEndMs ? turnEndMs - currentTurnStartMs : null;
+    runResult.perTurn.push({
+      turn: runResult.agentTurns,
+      inputTokens: usage?.input ?? 0,
+      outputTokens: usage?.output ?? 0,
+      cacheRead: usage?.cacheRead ?? 0,
+      cacheWrite: usage?.cacheWrite ?? 0,
+      wallClockMs,
+      toolCalls: 0,
+    });
+    if (turnEndMs) lastTurnEndMs = turnEndMs;
+    currentTurnStartMs = null;
+  }
+
+  function processLine(line) {
+    if (!line.trim()) return;
+    streamHandle.write(line + "\n");
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch (err) {
+      console.log(BENCH_COLORS.error(`[parse error] ${err.message}`));
+      return;
+    }
+
+    const timestamp = extractTimestamp(parsed);
+    updateTimeBounds(timestamp);
+
+    renderStreamEvent(parsed, renderState, { verbose });
+
+    if (parsed.type === "session" || parsed.type === "agent_start") {
+      if (timestamp && runStartMs === null) runStartMs = timestamp;
+    }
+
+    if (parsed.type === "turn_start") {
+      beginTurn();
+    }
+
+    if (parsed.type === "turn_end" && parsed.message?.usage) {
+      const usage = parsed.message.usage;
+      runResult.promptTokens += usage.input ?? 0;
+      runResult.completionTokens += usage.output ?? 0;
+      runResult.cacheRead += usage.cacheRead ?? 0;
+      runResult.cacheWrite += usage.cacheWrite ?? 0;
+      endTurn(usage, timestamp);
+    }
+
+    if (parsed.type === "message_update" && parsed.assistantMessageEvent) {
+      const evt = parsed.assistantMessageEvent;
+      const subtype = String(evt.type ?? "").replace(/_/gu, "");
+      if (subtype === "thinkingdelta" || subtype === "textdelta") {
+        appendResponse(evt.delta || "");
+      }
+    }
+
+    if (parsed.type === "message_end" && parsed.message?.role === "assistant") {
+      flushResponse();
+      const content = parsed.message.content ?? [];
+      for (const item of content) {
+        if (item.type === "toolCall") {
+          runResult.toolCalls += 1;
+          appendResponse(`\n${formatToolCall(item)}\n`);
+          const currentTurn = runResult.perTurn[runResult.perTurn.length - 1];
+          if (currentTurn) currentTurn.toolCalls += 1;
+        }
+      }
+    }
+
+    if (parsed.type === "toolResult") {
+      runResult.toolResults += 1;
+      const status = parsed.isError ? "error" : "ok";
+      appendResponse(`\n[toolResult] ${parsed.toolName} (${status})\n`);
+    }
+
+    if (parsed.type === "agent_end") {
+      flushResponse();
+    }
+  }
+
+  child.stdout.setEncoding("utf8");
+  child.stdout.on("data", (chunk) => {
+    streamBuffer += chunk;
+    const lines = streamBuffer.split("\n");
+    streamBuffer = lines.pop();
+    for (const line of lines) {
+      processLine(line);
+    }
+  });
+
+  child.stderr.setEncoding("utf8");
+  child.stderr.on("data", (chunk) => {
+    stderrHandle.write(chunk);
+  });
+
+  const abortListener = () => {
+    if (cancelled) return;
+    cancelled = true;
+    console.log(BENCH_COLORS.error("\n\n[Cancelled by user]"));
+    child.kill("SIGTERM");
+  };
+
+  if (signal) {
+    signal.addEventListener("abort", abortListener);
+  }
+
+  return new Promise((resolve) => {
+    child.on("exit", async (code) => {
+      if (signal) signal.removeEventListener("abort", abortListener);
+      if (streamBuffer.trim()) {
+        processLine(streamBuffer);
+      }
+      flushResponse();
+      await streamHandle.close();
+      await stderrHandle.close();
+      await writeFile(responsePath, runResult.rawResponseLines.join(""), "utf8");
+
+      runResult.exitCode = code ?? 0;
+      if (firstEventMs !== null && lastEventMs !== null) {
+        runResult.wallClockMs = lastEventMs - firstEventMs;
+      }
+
+      if (cancelled) {
+        runResult.error = { message: "Cancelled by user" };
+        resolve(runResult);
+        return;
+      }
+
+      if (runResult.exitCode !== 0) {
+        runResult.error = { message: `Pi exited with code ${runResult.exitCode}` };
+        resolve(runResult);
+        return;
+      }
+
+      resolve(runResult);
+    });
+
+    child.on("error", async (err) => {
+      if (signal) signal.removeEventListener("abort", abortListener);
+      await streamHandle.close();
+      await stderrHandle.close();
+      runResult.error = { message: err.message };
+      resolve(runResult);
+    });
+  });
+}
+
+function extractTimestamp(event) {
+  const raw = event?.message?.timestamp ?? event?.timestamp ?? event?.assistantMessageEvent?.partial?.timestamp;
+  if (typeof raw === "number") return raw;
+  if (typeof raw === "string") {
+    const parsed = Date.parse(raw);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  const iso = event?.message?.createdAt ?? event?.createdAt ?? event?.created_at;
+  if (typeof iso === "string") {
+    const parsed = Date.parse(iso);
+    if (Number.isFinite(parsed)) return parsed;
+  }
+  return null;
+}
+
+async function openFileHandle(path, flags) {
+  const { open } = await import("node:fs/promises");
+  return open(path, flags);
+}