offgrid-ai 0.16.3 → 0.17.0

package/src/benchmark/flow.mjs

@@ -1,240 +0,0 @@
-// ── Benchmark command flows ───────────────────────────────────────────────────
-
-import { join } from "node:path";
-import { ensureDirs } from "../config.mjs";
-import { backendFor } from "../backends.mjs";
-import { serverReady, startServer, waitForReady, stopProfile, modelAvailableOnServer, unloadModelFromServer } from "../process.mjs";
-import { loadProfiles } from "../profiles.mjs";
-import { pc, createPrompt } from "../ui.mjs";
-import { linkBenchmarkRepo } from "./repo.mjs";
-import { loadBenchmarks } from "./shared.mjs";
-import { prepareBenchmarkRun } from "./prepare.mjs";
-import { runBenchmarkInPi } from "./sdk-runner.mjs";
-import { queryServerMetrics } from "./metrics.mjs";
-import { finalizeBenchmarkRun, renderBenchmarkSummary } from "./finalize.mjs";
-
-function benchmarkModelSource(profile) {
-  if (!profile) return "cloud";
-  return profile.backend === "omlx" ? "omlx" : "llama-cpp";
-}
-
-async function chooseBenchmarkAction(prompt, canRun) {
-  const choices = [
-    { value: "run", label: "Run Benchmark", hint: "Automated with Pi" },
-    { value: "prepare", label: "Prepare Benchmark (manual)", hint: "Copy prompt and run yourself" },
-  ];
-  return await prompt.choice("Action", canRun ? choices : choices.filter((c) => c.value === "prepare"), canRun ? "run" : "prepare");
-}
-
-function managedModelId(profile) {
-  return profile.omlxModel ?? profile.modelAlias ?? profile.label;
-}
-
-async function ensureManagedModelAvailableForBenchmark(profile, backend) {
-  if (backend.type !== "managed-server") return;
-  if (await modelAvailableOnServer(profile)) return;
-  throw new Error(`${managedModelId(profile)} is not available on ${backend.label} at ${profile.baseUrl}.`);
-}
-
-async function ensureServerForBenchmark(profile) {
-  const backend = backendFor(profile.backend);
-  if (await serverReady(profile.baseUrl)) {
-    await ensureManagedModelAvailableForBenchmark(profile, backend);
-    console.log(pc.green(`[ready] ${backend.label} at ${profile.baseUrl}`));
-    return { started: false };
-  }
-
-  if (backend.type === "managed-server") {
-    throw new Error(`${backend.label} is not running at ${profile.baseUrl}. Start it and try again.`);
-  }
-
-  console.log(pc.dim(`Starting ${backend.label} for ${profile.label}...`));
-  const state = await startServer(profile);
-  await waitForReady(profile, state?.pid, state?.rawLogPath);
-  console.log(pc.green(`[ready] ${profile.baseUrl}/models`));
-  return { started: true, state };
-}
-
-export async function runPreparedBenchmark(profile, runDirectory, options = {}) {
-  const controller = new AbortController();
-  if (options.signal) {
-    options.signal.addEventListener("abort", () => controller.abort(), { once: true });
-  }
-  let serverStarted = false;
-  let benchmarkStarted = false;
-  let metadata;
-
-  const onSigint = () => {
-    controller.abort();
-  };
-  process.on("SIGINT", onSigint);
-
-  try {
-    const serverState = await ensureServerForBenchmark(profile);
-    serverStarted = serverState.started;
-
-    benchmarkStarted = true;
-    const runResult = await runBenchmarkInPi(profile, runDirectory, { signal: controller.signal });
-
-    let speedMetrics = null;
-    let speedMetricsError = null;
-    if (!runResult.error) {
-      try {
-        speedMetrics = await queryServerMetrics(profile);
-      } catch (err) {
-        // Non-fatal: speed metrics are a supplementary measurement, not the
-        // benchmark itself. Don't poison the run result; surface it as a note.
-        speedMetricsError = err.message;
-      }
-    }
-
-    metadata = await finalizeBenchmarkRun(runDirectory, runResult, speedMetrics, speedMetricsError);
-    renderBenchmarkSummary(metadata);
-  } catch (err) {
-    const failedResult = {
-      error: { message: err.message },
-      wallClockMs: null,
-      agentTurns: 0,
-      promptTokens: 0,
-      completionTokens: 0,
-      totalTokens: 0,
-      cacheRead: 0,
-      cacheWrite: 0,
-      toolCalls: 0,
-      toolResults: 0,
-      perTurn: [],
-    };
-    metadata = await finalizeBenchmarkRun(runDirectory, failedResult, null);
-    renderBenchmarkSummary(metadata);
-  } finally {
-    process.removeListener("SIGINT", onSigint);
-    if (serverStarted && !options.keepServer) {
-      const backend = backendFor(profile.backend);
-      if (backend.type !== "managed-server") {
-        const result = await stopProfile(profile);
-        console.log(result.stopped ? pc.green(`[stop] ${result.message}`) : pc.dim(`[stop] ${result.message}`));
-      }
-    }
-    if (benchmarkStarted) {
-      const unloadResult = await unloadModelFromServer(profile);
-      if (!unloadResult.unloaded && unloadResult.error) {
-        console.log(pc.yellow(`[unload] ${unloadResult.backend}: ${unloadResult.error}`));
-      } else if (!unloadResult.unloaded && unloadResult.reason) {
-        console.log(pc.dim(`[unload] ${unloadResult.backend}: ${unloadResult.reason}`));
-      }
-    }
-  }
-
-  return metadata;
-}
-
-// ── Shared benchmark selection ────────────────────────────────────────────
-
-async function selectBenchmark(prompt, repoPath) {
-  const kind = await prompt.choice("Benchmark category", [
-    { value: "visual", label: "Visual Benchmark", hint: "HTML/CSS/JS animation benchmarks" },
-    { value: "data-science", label: "Data Science", hint: "Analysis and charting benchmarks" },
-  ], "visual");
-
-  const benchDir = join(repoPath, "benchmarks");
-  const benchmarks = (await loadBenchmarks(benchDir)).filter((b) => b.kind === kind);
-  if (benchmarks.length === 0) {
-    console.log(pc.yellow(`No ${kind} benchmarks found in ${benchDir}`));
-    return null;
-  }
-  const benchmarkId = await prompt.choice("Prompt", benchmarks.map((b) => ({
-    value: b.id, label: b.title, hint: b.description || b.id,
-  })), benchmarks[0].id);
-  const benchmark = benchmarks.find((b) => b.id === benchmarkId);
-  if (!benchmark) return null;
-  return { kind, benchmark };
-}
-
-// ── Shared benchmark setup ───────────────────────────────────────────────
-
-async function benchmarkSetup() {
-  await ensureDirs();
-  const prompt = createPrompt();
-  const repoPath = await linkBenchmarkRepo(prompt);
-  if (!repoPath) return { prompt, repoPath: null, selected: null };
-  const selected = await selectBenchmark(prompt, repoPath);
-  return { prompt, repoPath, selected };
-}
-
-// ── Benchmark from a selected profile (from model picker) ────────────────
-
-export async function benchmarkForProfile(profile) {
-  const { prompt, repoPath, selected } = await benchmarkSetup();
-  try {
-    if (!selected) return;
-    const { kind, benchmark: selectedBenchmark } = selected;
-
-    const modelId = profile.modelAlias;
-    const modelSource = benchmarkModelSource(profile);
-    const backendLabel = backendFor(profile.backend).label;
-
-    const canRun = modelSource !== "cloud";
-    const action = await chooseBenchmarkAction(prompt, canRun);
-
-    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
-
-    if (action === "run") {
-      return await runPreparedBenchmark(profile, runDirectory);
-    }
-
-    return runDirectory;
-  } finally {
-    prompt.close();
-  }
-}
-
-// ── Standalone benchmark flow (offgrid-ai benchmark) ──────────────────────
-
-export async function benchmarkFlow() {
-  const { prompt, repoPath, selected } = await benchmarkSetup();
-  try {
-    if (!selected) return;
-    const { kind, benchmark: selectedBenchmark } = selected;
-
-    const profiles = await loadProfiles();
-    const source = await prompt.choice("Model source", [
-      { value: "profile", label: "Use existing profile", hint: "Pick a saved offgrid-ai profile" },
-      { value: "cloud", label: "Custom / cloud", hint: "Free-form model label for cloud runs" },
-    ], "profile");
-
-    let modelId, modelSource, backendLabel, profile;
-
-    if (source === "profile") {
-      if (profiles.length === 0) {
-        console.log(pc.yellow("No profiles yet. Run: offgrid-ai models"));
-        return;
-      }
-      const profileId = await prompt.choice("Profile", profiles.map((p) => ({
-        value: p.id, label: p.label, hint: `${backendFor(p.backend).label} · ${p.modelAlias}`,
-      })), profiles[0].id);
-      profile = profiles.find((p) => p.id === profileId);
-      if (!profile) return;
-      modelId = profile.modelAlias;
-      modelSource = benchmarkModelSource(profile);
-      backendLabel = backendFor(profile.backend).label;
-    } else {
-      backendLabel = await prompt.text("Backend label", "cloud");
-      modelId = await prompt.text("Model name", "");
-      if (!modelId) { console.log(pc.yellow("Model name is required.")); return; }
-      modelSource = "cloud";
-    }
-
-    const canRun = modelSource !== "cloud" && profile != null;
-    const action = await chooseBenchmarkAction(prompt, canRun);
-
-    const runDirectory = await prepareBenchmarkRun({ repoPath, benchmark: selectedBenchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps: action === "prepare" });
-
-    if (action === "run" && profile) {
-      return await runPreparedBenchmark(profile, runDirectory);
-    }
-
-    return runDirectory;
-  } finally {
-    prompt.close();
-  }
-}

package/src/benchmark/metrics.mjs

@@ -1,107 +0,0 @@
-// ── Backend-aware server speed metrics ───────────────────────────────────────
-
-import { backendFor } from "../backends.mjs";
-
-const BENCH_SPEED_PROMPT = "Write a one-sentence summary of machine learning.";
-const SPEED_QUERY_TIMEOUT_MS = 120_000;
-const SPEED_QUERY_MAX_TOKENS = 64;
-
-export async function queryServerMetrics(profile) {
-  const backend = backendFor(profile.backend);
-
-  if (backend.id === "llama-cpp") {
-    return await queryLlamaCppMetrics(profile);
-  }
-  if (backend.id === "omlx") {
-    return await queryOmlxMetrics(profile);
-  }
-
-  throw new Error(`Unsupported backend for benchmark speed metrics: ${backend.id}`);
-}
-
-async function speedQueryFetch(profile, { stream = false, streamOptions = null, errorLabel = "speed query" } = {}) {
-  const body = {
-    model: profile.modelAlias,
-    messages: [{ role: "user", content: BENCH_SPEED_PROMPT }],
-    stream,
-    max_tokens: SPEED_QUERY_MAX_TOKENS,
-    ...(streamOptions ? { stream_options: streamOptions } : {}),
-  };
-
-  const response = await fetch(profile.baseUrl.replace(/\/$/u, "") + "/chat/completions", {
-    method: "POST",
-    headers: { "Content-Type": "application/json" },
-    body: JSON.stringify(body),
-    signal: AbortSignal.timeout(SPEED_QUERY_TIMEOUT_MS),
-  });
-
-  if (!response.ok) {
-    throw new Error(`${errorLabel} failed: ${response.status} ${response.statusText}`);
-  }
-
-  return response;
-}
-
-async function queryLlamaCppMetrics(profile) {
-  const response = await speedQueryFetch(profile, { errorLabel: "llama.cpp speed query" });
-
-  const data = await response.json();
-  const timings = data.timings;
-  if (!timings || typeof timings.prompt_per_second !== "number" || typeof timings.predicted_per_second !== "number") {
-    throw new Error("llama.cpp response did not include usable timings object");
-  }
-  const draftN = timings.draft_n;
-  const draftAccepted = timings.draft_n_accepted;
-
-  return {
-    prefillTokensPerSecond: timings.prompt_per_second ?? null,
-    generationTokensPerSecond: timings.predicted_per_second ?? null,
-    ttftMs: timings.prompt_ms ?? null,
-    modelLoadMs: null,
-    speculativeDecodeAcceptance: (draftN && Number.isFinite(draftAccepted) && Number.isFinite(draftN) && draftN > 0)
-      ? draftAccepted / draftN
-      : null,
-    kvCacheTokens: timings.cache_n ?? null,
-    metricSource: "llama.cpp /v1/chat/completions timings",
-  };
-}
-
-async function queryOmlxMetrics(profile) {
-  const response = await speedQueryFetch(profile, {
-    stream: true,
-    streamOptions: { include_usage: true },
-    errorLabel: "oMLX speed query",
-  });
-
-  const text = await response.text();
-  let usage = null;
-  for (const line of text.split("\n").reverse()) {
-    const trimmed = line.trim();
-    if (!trimmed || !trimmed.startsWith("data:")) continue;
-    const payload = trimmed.slice(5).trim();
-    if (payload === "[DONE]") continue;
-    try {
-      const chunk = JSON.parse(payload);
-      if (chunk.usage) {
-        usage = chunk.usage;
-        break;
-      }
-    } catch {
-      // Ignore malformed SSE chunks.
-    }
-  }
-
-  if (!usage) {
-    throw new Error("oMLX speed query did not return usage in streaming response");
-  }
-
-  return {
-    prefillTokensPerSecond: usage.prompt_tokens_per_second ?? null,
-    generationTokensPerSecond: usage.generation_tokens_per_second ?? null,
-    ttftMs: usage.time_to_first_token != null ? usage.time_to_first_token * 1000 : null,
-    modelLoadMs: null,
-    speculativeDecodeAcceptance: null,
-    kvCacheTokens: usage.prompt_tokens_details?.cached_tokens ?? null,
-    metricSource: "oMLX /v1/chat/completions streaming include_usage",
-  };
-}

package/src/benchmark/prepare.mjs

@@ -1,118 +0,0 @@
-// ── Create a benchmark run directory ────────────────────────────────────────
-
-import { mkdir, writeFile } from "node:fs/promises";
-import { join } from "node:path";
-import { pc, renderRows, renderSection } from "../ui.mjs";
-import { slugModelId, createRunId } from "./shared.mjs";
-import { parseModelName } from "../model-name.mjs";
-
-function harnessDisplayName(id) {
-  if (id === "pi") return "Pi";
-  return String(id).replace(/[-_]+/gu, " ").replace(/\b\w/gu, (char) => char.toUpperCase());
-}
-
-function intendedRunnerForProfile(profile) {
-  if (!profile) return "your tool";
-  const harnessEntries = Object.entries(profile.harnesses ?? {}).filter(([, config]) => config?.enabled !== false);
-  const [id] = harnessEntries.find(([key]) => key === "pi") ?? harnessEntries[0] ?? ["pi"];
-  return harnessDisplayName(id);
-}
-
-function printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel }) {
-  const runCommand = profile ? `offgrid-ai run ${profile.id}` : null;
-  const runnerCommand = runCommand ?? `Open ${runnerLabel} for ${modelId}`;
-
-  console.log("");
-  console.log(pc.bold("Next steps"));
-  console.log(`  1. Open the gallery. If it is not running: ${pc.cyan(`cd ${repoPath} && npm run dev`)}`);
-  console.log(`  2. ${pc.cyan(`cd ${runDirectory}`)}`);
-  console.log(`  3. ${pc.cyan(runnerCommand)}, then copy this run's prompt from the gallery and paste it into ${runnerLabel}`);
-}
-
-export async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps = true }) {
-  const toolPrompt = benchmark.prompt;
-  const now = new Date();
-  const runId = createRunId(now);
-  const modelSlug = slugModelId(modelId);
-  const runnerLabel = intendedRunnerForProfile(profile);
-  const runsDir = join(repoPath, "runs");
-  const benchmarkDirectory = join(runsDir, benchmark.id);
-  const modelDirectory = join(benchmarkDirectory, modelSlug);
-  const runDirectory = join(modelDirectory, runId);
-
-  await mkdir(runDirectory, { recursive: true });
-
-  const isDs = kind === "data-science";
-  const baseAssets = {
-    metadata: "metadata.json",
-    prompt: "prompt.md",
-  };
-  const metadata = {
-    schemaVersion: 1,
-    kind,
-    runId,
-    benchmark: { id: benchmark.id, title: benchmark.title, description: benchmark.description, prompt: benchmark.prompt },
-    model: { id: modelId, slug: modelSlug, displayName: parseModelName(modelId, modelSource === "omlx" ? "omlx" : "local-gguf").display },
-    status: "prepared",
-    createdAt: now.toISOString(),
-    updatedAt: now.toISOString(),
-    preparedAt: now.toISOString(),
-    runDirectory,
-    assets: isDs
-      ? { ...baseAssets, ds: { notebook: "analysis.ipynb", summary: "summary.json", chartDistribution: "chart-distribution.png", chartTreatmentEffect: "chart-treatment-effect.png", chartCompletionRates: "chart-completion-rates.png" } }
-      : { ...baseAssets, html: "index.html", preview: "preview.png", video: "preview.webm" },
-    runner: {
-      mode: modelSource === "cloud" ? "manual" : "external",
-      intendedRunner: profile ? runnerLabel : undefined,
-      ...(profile?.harnesses?.pi || runnerLabel === "Pi" ? { tool: "pi" } : {}),
-      ...(modelSource ? { modelSource } : {}),
-      ...(backendLabel ? { backendLabel } : {}),
-      ...(profile?.baseUrl ? { baseUrl: profile.baseUrl } : {}),
-      model: modelId,
-      retries: 0,
-      tokenMetrics: {
-        reported: false,
-        promptTokens: 0,
-        completionTokens: 0,
-        totalTokens: 0,
-      },
-      speedMetrics: {
-        prefillTokensPerSecond: null,
-        generationTokensPerSecond: null,
-        ttftMs: null,
-        modelLoadMs: null,
-        speculativeDecodeAcceptance: null,
-        kvCacheTokens: null,
-      },
-      metricSource: null,
-    },
-    results: {
-      wallClockMs: null,
-      agentTurns: 0,
-      toolCalls: 0,
-      toolResults: 0,
-      success: false,
-      outputFiles: [],
-      perTurn: [],
-    },
-  };
-
-  await writeFile(join(runDirectory, "metadata.json"), JSON.stringify(metadata, null, 2) + "\n", "utf8");
-  await writeFile(join(runDirectory, "prompt.md"), toolPrompt + "\n", "utf8");
-
-  console.log("");
-  console.log(pc.green("✓ Run slot prepared"));
-  console.log(renderSection("Run", renderRows([
-    ["Directory", pc.cyan(runDirectory)],
-    ["Benchmark", benchmark.title],
-    ["Kind", kind],
-    ["Model", pc.bold(modelId)],
-    ["Source", backendLabel || modelSource],
-  ])));
-
-  if (showNextSteps) {
-    printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel });
-  }
-
-  return runDirectory;
-}

package/src/benchmark/repo.mjs

@@ -1,77 +0,0 @@
-// ── Benchmark repo linking ────────────────────────────────────────────────────
-
-import { existsSync } from "node:fs";
-import { join, resolve } from "node:path";
-import { homedir } from "node:os";
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
-import { loadConfig, saveConfig } from "../config.mjs";
-import { pc } from "../ui.mjs";
-
-const execFileAsync = promisify(execFile);
-
-const BENCHMARK_REPO = "https://github.com/eeshansrivastava89/local-llm-visual-benchmark.git";
-
-export async function findBenchmarkRepo() {
-  const config = await loadConfig();
-  if (config.benchmarkRepoPath && existsSync(join(config.benchmarkRepoPath, "benchmarks"))) {
-    return config.benchmarkRepoPath;
-  }
-  return null;
-}
-
-export async function linkBenchmarkRepo(prompt) {
-  const existing = await findBenchmarkRepo();
-  if (existing) return existing;
-
-  const candidates = [
-    join(homedir(), "dev", "local-llm-visual-benchmark"),
-    join(homedir(), "projects", "local-llm-visual-benchmark"),
-    join(homedir(), "local-llm-visual-benchmark"),
-  ];
-  for (const candidate of candidates) {
-    if (existsSync(join(candidate, "benchmarks"))) {
-      const config = await loadConfig();
-      config.benchmarkRepoPath = candidate;
-      await saveConfig(config);
-      return candidate;
-    }
-  }
-
-  console.log(pc.dim("\nThe benchmark gallery needs to be linked to offgrid-ai."));
-  console.log(pc.dim("This is the local-llm-visual-benchmark repo that stores prompts and run results.\n"));
-
-  const choice = await prompt.choice("Link benchmark gallery", [
-    { value: "clone", label: "Clone from GitHub", hint: "git clone into ~/dev" },
-    { value: "manual", label: "Enter path manually", hint: "If you already have it cloned" },
-  ], "clone");
-
-  if (choice === "clone") {
-    const targetDir = join(homedir(), "dev", "local-llm-visual-benchmark");
-    console.log(pc.dim(`\nCloning ${BENCHMARK_REPO}...`));
-    try {
-      await execFileAsync("git", ["clone", BENCHMARK_REPO, targetDir], { stdio: "pipe" });
-      const config = await loadConfig();
-      config.benchmarkRepoPath = targetDir;
-      await saveConfig(config);
-      console.log(pc.green(`✓ Cloned to ${targetDir}`));
-      return targetDir;
-    } catch (err) {
-      console.log(pc.red(`Clone failed: ${err.message}`));
-      return null;
-    }
-  }
-
-  const path = await prompt.text("Path to local-llm-visual-benchmark", "");
-  if (!path) return null;
-  const resolved = resolve(path.replace(/^~/, homedir()));
-  if (!existsSync(join(resolved, "benchmarks"))) {
-    console.log(pc.red(`No benchmarks/ directory found at ${resolved}`));
-    return null;
-  }
-  const config = await loadConfig();
-  config.benchmarkRepoPath = resolved;
-  await saveConfig(config);
-  console.log(pc.green(`✓ Linked to ${resolved}`));
-  return resolved;
-}