offgrid-ai 0.8.15 → 0.9.2

package/src/benchmark/prepare.mjs

@@ -0,0 +1,121 @@
+// ── Create a benchmark run directory ────────────────────────────────────────
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { pc, renderRows, renderSection } from "../ui.mjs";
+import { slugModelId, createRunId, buildToolPrompt } from "./shared.mjs";
+import { parseModelName } from "../model-name.mjs";
+
+function harnessDisplayName(id) {
+  if (id === "pi") return "Pi";
+  return String(id).replace(/[-_]+/gu, " ").replace(/\b\w/gu, (char) => char.toUpperCase());
+}
+
+function intendedRunnerForProfile(profile) {
+  if (!profile) return "your tool";
+  const harnessEntries = Object.entries(profile.harnesses ?? {}).filter(([, config]) => config?.enabled !== false);
+  const [id] = harnessEntries.find(([key]) => key === "pi") ?? harnessEntries[0] ?? ["pi"];
+  return harnessDisplayName(id);
+}
+
+function printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel }) {
+  const runCommand = profile ? `offgrid-ai run ${profile.id}` : null;
+  const runnerCommand = runCommand ?? `Open ${runnerLabel} for ${modelId}`;
+
+  console.log("");
+  console.log(pc.bold("Next steps"));
+  console.log(`  1. Open the gallery. If it is not running: ${pc.cyan(`cd ${repoPath} && npm run dev`)}`);
+  console.log(`  2. ${pc.cyan(`cd ${runDirectory}`)}`);
+  console.log(`  3. ${pc.cyan(runnerCommand)}, then copy this run's prompt from the gallery and paste it into ${runnerLabel}`);
+}
+
+export async function prepareBenchmarkRun({ repoPath, benchmark, kind, modelId, modelSource, backendLabel, profile, showNextSteps = true }) {
+  const toolPrompt = buildToolPrompt(benchmark);
+  const now = new Date();
+  const runId = createRunId(now);
+  const modelSlug = slugModelId(modelId);
+  const runnerLabel = intendedRunnerForProfile(profile);
+  const runsDir = join(repoPath, "runs");
+  const benchmarkDirectory = join(runsDir, benchmark.id);
+  const modelDirectory = join(benchmarkDirectory, modelSlug);
+  const runDirectory = join(modelDirectory, runId);
+
+  await mkdir(runDirectory, { recursive: true });
+
+  const isDs = kind === "data-science";
+  const baseAssets = {
+    metadata: "metadata.json",
+    prompt: "prompt.md",
+    rawResponse: "response.raw.txt",
+    stream: "stream.ndjson",
+    stderr: "stderr.log",
+  };
+  const metadata = {
+    schemaVersion: 1,
+    kind,
+    runId,
+    benchmark: { id: benchmark.id, title: benchmark.title, description: benchmark.description, prompt: benchmark.prompt },
+    model: { id: modelId, slug: modelSlug, displayName: parseModelName(modelId, modelSource === "ollama" ? "ollama" : modelSource === "omlx" ? "omlx" : "local-gguf").display },
+    status: "prepared",
+    createdAt: now.toISOString(),
+    updatedAt: now.toISOString(),
+    preparedAt: now.toISOString(),
+    runDirectory,
+    assets: isDs
+      ? { ...baseAssets, ds: { notebook: "analysis.ipynb", summary: "summary.json", chartDistribution: "chart-distribution.png", chartTreatmentEffect: "chart-treatment-effect.png", chartCompletionRates: "chart-completion-rates.png" } }
+      : { ...baseAssets, html: "index.html", preview: "preview.png", video: "preview.webm" },
+    runner: {
+      mode: modelSource === "cloud" ? "manual" : "external",
+      intendedRunner: profile ? runnerLabel : undefined,
+      ...(profile?.harnesses?.pi || runnerLabel === "Pi" ? { tool: "pi" } : {}),
+      ...(modelSource ? { modelSource } : {}),
+      ...(backendLabel ? { backendLabel } : {}),
+      ...(profile?.baseUrl ? { baseUrl: profile.baseUrl } : {}),
+      model: modelId,
+      retries: 0,
+      tokenMetrics: {
+        reported: false,
+        promptTokens: 0,
+        completionTokens: 0,
+        totalTokens: 0,
+      },
+      speedMetrics: {
+        prefillTokensPerSecond: null,
+        generationTokensPerSecond: null,
+        ttftMs: null,
+        modelLoadMs: null,
+        speculativeDecodeAcceptance: null,
+        kvCacheTokens: null,
+      },
+      metricSource: null,
+    },
+    results: {
+      wallClockMs: null,
+      agentTurns: 0,
+      toolCalls: 0,
+      toolResults: 0,
+      success: false,
+      outputFiles: [],
+      perTurn: [],
+    },
+  };
+
+  await writeFile(join(runDirectory, "metadata.json"), JSON.stringify(metadata, null, 2) + "\n", "utf8");
+  await writeFile(join(runDirectory, "prompt.md"), toolPrompt + "\n", "utf8");
+
+  console.log("");
+  console.log(pc.green("✓ Run slot prepared"));
+  console.log(renderSection("Run", renderRows([
+    ["Directory", pc.cyan(runDirectory)],
+    ["Benchmark", benchmark.title],
+    ["Kind", kind],
+    ["Model", pc.bold(modelId)],
+    ["Source", backendLabel || modelSource],
+  ])));
+
+  if (showNextSteps) {
+    printBenchmarkNextSteps({ repoPath, runDirectory, profile, modelId, runnerLabel });
+  }
+
+  return runDirectory;
+}

package/src/benchmark/repo.mjs

@@ -0,0 +1,77 @@
+// ── Benchmark repo linking ────────────────────────────────────────────────────
+
+import { existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { homedir } from "node:os";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { loadConfig, saveConfig } from "../config.mjs";
+import { pc } from "../ui.mjs";
+
+const execFileAsync = promisify(execFile);
+
+const BENCHMARK_REPO = "https://github.com/eeshansrivastava89/local-llm-visual-benchmark.git";
+
+export async function findBenchmarkRepo() {
+  const config = await loadConfig();
+  if (config.benchmarkRepoPath && existsSync(join(config.benchmarkRepoPath, "benchmarks"))) {
+    return config.benchmarkRepoPath;
+  }
+  return null;
+}
+
+export async function linkBenchmarkRepo(prompt) {
+  const existing = await findBenchmarkRepo();
+  if (existing) return existing;
+
+  const candidates = [
+    join(homedir(), "dev", "local-llm-visual-benchmark"),
+    join(homedir(), "projects", "local-llm-visual-benchmark"),
+    join(homedir(), "local-llm-visual-benchmark"),
+  ];
+  for (const candidate of candidates) {
+    if (existsSync(join(candidate, "benchmarks"))) {
+      const config = await loadConfig();
+      config.benchmarkRepoPath = candidate;
+      await saveConfig(config);
+      return candidate;
+    }
+  }
+
+  console.log(pc.dim("\nThe benchmark gallery needs to be linked to offgrid-ai."));
+  console.log(pc.dim("This is the local-llm-visual-benchmark repo that stores prompts and run results.\n"));
+
+  const choice = await prompt.choice("Link benchmark gallery", [
+    { value: "clone", label: "Clone from GitHub", hint: "git clone into ~/dev" },
+    { value: "manual", label: "Enter path manually", hint: "If you already have it cloned" },
+  ], "clone");
+
+  if (choice === "clone") {
+    const targetDir = join(homedir(), "dev", "local-llm-visual-benchmark");
+    console.log(pc.dim(`\nCloning ${BENCHMARK_REPO}...`));
+    try {
+      await execFileAsync("git", ["clone", BENCHMARK_REPO, targetDir], { stdio: "pipe" });
+      const config = await loadConfig();
+      config.benchmarkRepoPath = targetDir;
+      await saveConfig(config);
+      console.log(pc.green(`✓ Cloned to ${targetDir}`));
+      return targetDir;
+    } catch (err) {
+      console.log(pc.red(`Clone failed: ${err.message}`));
+      return null;
+    }
+  }
+
+  const path = await prompt.text("Path to local-llm-visual-benchmark", "");
+  if (!path) return null;
+  const resolved = resolve(path.replace(/^~/, homedir()));
+  if (!existsSync(join(resolved, "benchmarks"))) {
+    console.log(pc.red(`No benchmarks/ directory found at ${resolved}`));
+    return null;
+  }
+  const config = await loadConfig();
+  config.benchmarkRepoPath = resolved;
+  await saveConfig(config);
+  console.log(pc.green(`✓ Linked to ${resolved}`));
+  return resolved;
+}

package/src/benchmark/shared.mjs

@@ -0,0 +1,54 @@
+// ── Shared utilities (matches local-llm-visual-benchmark) ──────────────────
+
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export function slugModelId(modelId, maxLength = 80) {
+  const hash = createHash("sha256").update(modelId).digest("hex").slice(0, 10);
+  const normalized = modelId.normalize("NFKD").replace(/[\u0300-\u036f]/gu, "").toLowerCase();
+  const slug = normalized.replace(/[^a-z0-9]+/gu, "-").replace(/^-+|-+$/gu, "").replace(/-{2,}/gu, "-");
+  if (slug.length > 0 && slug.length <= maxLength && slug === normalized) return slug;
+  const baseMaxLength = Math.max(1, maxLength - 11);
+  const base = slug.slice(0, baseMaxLength).replace(/^-+|-+$/gu, "") || "model";
+  return `${base}-${hash}`;
+}
+
+export function createRunId(date = new Date()) {
+  return date.toISOString().replace(/:/gu, "-").replace(/\./gu, "-");
+}
+
+export function buildToolPrompt(benchmark) {
+  return benchmark.prompt;
+}
+
+export async function loadBenchmarks(benchDir) {
+  const entries = await readdir(benchDir);
+  const markdownFiles = entries.filter((f) => f.endsWith(".md")).sort();
+  const benchmarks = [];
+  for (const filename of markdownFiles) {
+    const raw = await readFile(join(benchDir, filename), "utf8");
+    const match = raw.match(/^---\n([\s\S]*?)\n---\n([\s\S]*)$/);
+    const frontmatter = match ? match[1] : "";
+    const content = match ? match[2].trim() : raw.trim();
+    let id = filename.replace(/\.md$/u, "");
+    let title = id;
+    let description = "";
+    for (const line of frontmatter.split("\n")) {
+      const kv = line.match(/^(\w+):\s*(.+)$/);
+      if (kv) {
+        const [, key, val] = kv;
+        if (key === "id") id = val.trim();
+        if (key === "title") title = val.trim();
+        if (key === "description") description = val.trim();
+      }
+    }
+    const kind = id === "ab-test-analysis" ? "data-science" : "visual";
+    benchmarks.push({ id, title, description, prompt: content, kind });
+  }
+  return benchmarks;
+}
+
+export function piModelString(profile) {
+  return profile.harnesses?.pi?.model ?? `${profile.providerId}/${profile.modelAlias}`;
+}

package/src/benchmark/stream-renderer.mjs

@@ -0,0 +1,274 @@
+// ── Semantic stream renderer for Pi benchmark output ─────────────────────────
+
+import { relative, basename } from "node:path";
+import { pc } from "../ui.mjs";
+
+export const BENCH_COLORS = {
+  thinking: pc.magenta,
+  text: pc.green,
+  tool: pc.yellow,
+  success: pc.green,
+  warning: pc.yellow,
+  toolOutput: pc.dim,
+  error: pc.red,
+  info: pc.cyan,
+  dim: pc.dim,
+};
+
+export function formatToolCall(toolCall) {
+  const path = toolCall.arguments?.path || toolCall.arguments?.file_path || toolCall.arguments?.filename || "";
+  const summary = path ? ` → ${path}` : "";
+  return `[toolCall] ${toolCall.name}${summary}`;
+}
+
+export function formatTokens(n) {
+  if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+  if (n >= 1_000) return `${Math.round(n / 1_000)}k`;
+  return String(Math.round(n));
+}
+
+export function estimatedTokensFromBytes(bytes) {
+  return Math.max(1, Math.ceil(bytes / 4));
+}
+
+export function clearStatusLine() {
+  if (process.stdout.isTTY) {
+    process.stdout.write("\r\x1b[K");
+  }
+}
+
+export function printStatusLine(text) {
+  if (process.stdout.isTTY) {
+    process.stdout.write(`\r\x1b[K${text}`);
+  }
+}
+
+export function printFinalLine(text) {
+  clearStatusLine();
+  console.log(text);
+}
+
+export function renderStreamEvent(parsed, state, opts = {}) {
+  const verbose = Boolean(opts.verbose);
+  const type = parsed.type;
+
+  switch (type) {
+    case "session":
+      printFinalLine(BENCH_COLORS.info("Pi benchmark started"));
+      if (parsed.id) printFinalLine(BENCH_COLORS.dim(`  Session  ${parsed.id}`));
+      break;
+    case "agent_start":
+      break;
+    case "turn_start": {
+      state.turn += 1;
+      state.turnHadToolError = false;
+      resetStatus(state, "thinking");
+      printFinalLine("");
+      printFinalLine(BENCH_COLORS.info(`Turn ${state.turn}`));
+      break;
+    }
+    case "message_start": {
+      const msg = parsed.message;
+      if (!state.modelPrinted && msg?.role === "assistant" && msg.provider && msg.model) {
+        state.modelPrinted = true;
+        printFinalLine(BENCH_COLORS.dim(`  Model    ${msg.provider}/${msg.model}`));
+      }
+      break;
+    }
+    case "message_update": {
+      const evt = parsed.assistantMessageEvent;
+      if (!evt) return;
+      const subtype = String(evt.type ?? "").replace(/_/gu, "");
+      if (subtype === "thinkingstart") {
+        resetStatus(state, "thinking");
+      } else if (subtype === "thinkingdelta") {
+        if (verbose) process.stdout.write(BENCH_COLORS.thinking(evt.delta || ""));
+        updateStatusFromDelta(state, evt.delta, "thinking");
+      } else if (subtype === "textstart") {
+        resetStatus(state, "text");
+      } else if (subtype === "textdelta") {
+        if (verbose) process.stdout.write(BENCH_COLORS.text(evt.delta || ""));
+        updateStatusFromDelta(state, evt.delta, "text");
+      } else if (subtype === "toolcallstart") {
+        resetStatus(state, "tool");
+      } else if (subtype === "toolcalldelta") {
+        if (verbose) process.stdout.write(BENCH_COLORS.tool(evt.delta || ""));
+        updateStatusFromDelta(state, evt.delta, "tool");
+      }
+      break;
+    }
+    case "message_end":
+      break;
+    case "tool_execution_start": {
+      state.activeTool = {
+        name: parsed.toolName,
+        args: parsed.args ?? {},
+        outputText: "",
+      };
+      resetStatus(state, "exec", parsed.toolName);
+      printFinalLine(BENCH_COLORS.tool(formatToolStart(parsed.toolName, parsed.args ?? {}, state)));
+      break;
+    }
+    case "tool_execution_update": {
+      const text = toolResultText(parsed.partialResult ?? parsed.result ?? parsed);
+      if (text) {
+        if (verbose) process.stdout.write(BENCH_COLORS.toolOutput(text));
+        if (state.activeTool) state.activeTool.outputText = text;
+        updateStatusFromDelta(state, text, "exec");
+      }
+      break;
+    }
+    case "tool_execution_end": {
+      const lines = formatToolEnd(parsed, state);
+      if (parsed.isError) state.turnHadToolError = true;
+      for (const line of lines) printFinalLine(line);
+      state.activeTool = null;
+      resetStatus(state, "idle");
+      break;
+    }
+    case "toolResult": {
+      if (parsed.isError) state.turnHadToolError = true;
+      const status = parsed.isError ? BENCH_COLORS.error("✗") : BENCH_COLORS.success("✓");
+      printFinalLine(`${status} ${parsed.toolName ?? "tool"}`);
+      break;
+    }
+    case "turn_end": {
+      const usage = parsed.message?.usage;
+      const tokenPart = usage ? ` · ${formatTokens(usage.output ?? usage.totalTokens ?? 0)} tokens` : "";
+      const marker = state.turnHadToolError ? BENCH_COLORS.warning("⚠") : BENCH_COLORS.success("✓");
+      const suffix = state.turnHadToolError ? " · tool issue" : "";
+      printFinalLine(`${marker} turn ${state.turn}${tokenPart}${suffix}`);
+      break;
+    }
+    case "agent_end":
+      clearStatusLine();
+      printFinalLine(BENCH_COLORS.info("Pi benchmark finished"));
+      break;
+    default:
+      break;
+  }
+}
+
+export function resetStatus(state, mode, toolName = null) {
+  state.status.mode = mode;
+  state.status.toolName = toolName;
+  state.status.bytes = 0;
+  state.status.tokens = 0;
+}
+
+export function updateStatusFromDelta(state, delta, mode = state.status.mode) {
+  if (!delta) return;
+  state.status.mode = mode;
+  state.status.bytes += Buffer.byteLength(delta, "utf8");
+  state.status.tokens = estimatedTokensFromBytes(state.status.bytes);
+  const label = state.status.toolName ? ` · ${state.status.toolName}` : "";
+  const modeLabel = {
+    thinking: "thinking…",
+    text: "drafting response…",
+    tool: "preparing tool…",
+    exec: "running tool…",
+  }[state.status.mode] ?? "working…";
+  const bytes = formatBytes(state.status.bytes);
+  const tokens = formatTokens(state.status.tokens);
+  printStatusLine(BENCH_COLORS.dim(`Turn ${state.turn} ${modeLabel}${label} · ${bytes} (~${tokens} tokens)`));
+}
+
+export function formatToolStart(toolName, args, state) {
+  if (toolName === "read") return `→ read ${displayPath(args.path, state)}`;
+  if (toolName === "write") {
+    const size = args.content ? ` · ${formatBytes(Buffer.byteLength(String(args.content), "utf8"))}` : "";
+    return `→ write ${displayPath(args.path, state)}${size}`;
+  }
+  if (toolName === "edit") {
+    const count = Array.isArray(args.edits) ? args.edits.length : 0;
+    const suffix = count > 0 ? ` · ${count} replacement${count === 1 ? "" : "s"}` : "";
+    return `→ edit ${displayPath(args.path, state)}${suffix}`;
+  }
+  if (toolName === "bash") return `→ run ${truncateOneLine(args.command ?? "")}`;
+  return `→ ${toolName}${compactArgs(args)}`;
+}
+
+export function formatToolEnd(parsed, state) {
+  const toolName = parsed.toolName ?? state.activeTool?.name ?? "tool";
+  const args = parsed.args ?? state.activeTool?.args ?? {};
+  const text = toolResultText(parsed.result) || state.activeTool?.outputText || "";
+  const marker = parsed.isError ? BENCH_COLORS.error("✗") : BENCH_COLORS.success("✓");
+
+  if (parsed.isError) {
+    return [`${marker} ${toolName} failed · ${firstUsefulLine(text)}`];
+  }
+
+  if (toolName === "write") return [`${marker} wrote ${displayPath(args.path, state)}${parsedWriteSize(text)}`];
+  if (toolName === "read") return [`${marker} read ${displayPath(args.path, state)}${text ? ` · ${formatBytes(Buffer.byteLength(text, "utf8"))}` : ""}`];
+  if (toolName === "edit") return [`${marker} edited ${displayPath(args.path, state)}`];
+  if (toolName === "bash") return formatBashResult(marker, text);
+
+  const summary = firstUsefulLine(text);
+  return [`${marker} ${toolName}${summary ? ` · ${summary}` : ""}`];
+}
+
+export function formatBashResult(marker, text) {
+  const lines = meaningfulLines(text).slice(0, 2);
+  if (lines.length === 0) return [`${marker} command completed`];
+  return [`${marker} ${lines[0]}`, ...lines.slice(1).map((line) => BENCH_COLORS.dim(`  ${line}`))];
+}
+
+export function parsedWriteSize(text) {
+  const match = String(text).match(/Successfully wrote\s+([0-9,]+)\s+bytes/iu);
+  if (!match) return "";
+  const bytes = Number(match[1].replace(/,/gu, ""));
+  return Number.isFinite(bytes) ? ` · ${formatBytes(bytes)}` : "";
+}
+
+export function toolResultText(result) {
+  const content = result?.content;
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .map((item) => typeof item?.text === "string" ? item.text : "")
+    .filter(Boolean)
+    .join("\n");
+}
+
+export function firstUsefulLine(text) {
+  return meaningfulLines(text)[0] ?? "no details";
+}
+
+export function meaningfulLines(text) {
+  const lines = String(text ?? "")
+    .split(/\r?\n/u)
+    .map((line) => line.trim())
+    .filter(Boolean)
+    .filter((line) => !/^\^+$/u.test(line));
+  const errorLine = lines.find((line) => /(?:error|exception|failed|not found|command exited with code|validation failed)/iu.test(line));
+  if (errorLine) return [errorLine, ...lines.filter((line) => line !== errorLine)];
+  return lines;
+}
+
+export function displayPath(value, state) {
+  if (!value) return "unknown";
+  const path = String(value);
+  const rel = state.cwd ? relative(state.cwd, path) : path;
+  if (rel && !rel.startsWith("..") && rel !== ".") return rel;
+  return basename(path) || path;
+}
+
+export function compactArgs(args) {
+  const entries = Object.entries(args ?? {}).filter(([, value]) => value !== undefined && value !== null && value !== "");
+  if (entries.length === 0) return "";
+  return ` · ${truncateOneLine(entries.map(([key, value]) => `${key}=${String(value)}`).join(" "))}`;
+}
+
+export function truncateOneLine(value, max = Math.max(60, Math.min(process.stdout.columns ?? 100, 140) - 12)) {
+  const text = String(value ?? "").replace(/\s+/gu, " ").trim();
+  return text.length > max ? `${text.slice(0, Math.max(1, max - 1))}…` : text;
+}
+
+export function formatBytes(bytes) {
+  if (!Number.isFinite(bytes)) return "unknown";
+  const units = ["B", "KB", "MB", "GB", "TB"];
+  let size = bytes;
+  let unit = 0;
+  while (size >= 1024 && unit < units.length - 1) { size /= 1024; unit += 1; }
+  return `${size.toFixed(unit === 0 ? 0 : 2)} ${units[unit]}`;
+}