@takara-ai/miru-code 1.1.3 → 1.2.2

package/src/benchmark/history.ts

@@ -0,0 +1,206 @@
+/**
+ * Benchmark mode history: JSONL of `{ r, m, g, s, p }` (repo + token deltas, no
+ * query text). Appends use `O_APPEND` (no lock). `read_benchmark` sums the file.
+ *
+ * Path: `<miru-state>/benchmark-history.json`, or `MIRU_BENCHMARK_HISTORY_PATH`.
+ * Cleared by `miru benchmark clear` / `miru uninstall`.
+ * Tests: `runWithBenchmarkHistoryPath` (AsyncLocalStorage).
+ */
+
+import { AsyncLocalStorage } from "node:async_hooks";
+import { appendFile, mkdir, rename } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import { resolveMiruStateDir } from "../credentials.ts";
+import { isGitUrl } from "../utils.ts";
+import { agentBenchmarkFromTokens } from "./summary.ts";
+import type { AgentBenchmarkRollup, AgentBenchmarkSummary, SearchBenchmarkBlock } from "./types.ts";
+
+const HISTORY_FILENAME = "benchmark-history.json";
+const historyPathStore = new AsyncLocalStorage<string>();
+
+/** One JSONL line: r=repo, m=miru, g=grep, s=saved, p=save_pct. */
+export interface BenchmarkContribution {
+  r: string;
+  m: number;
+  g: number;
+  s: number;
+  p: number;
+}
+
+/** Totals for one repo (or the global fold). pct_sum → mean = round(pct_sum / n). */
+export interface TokenTotals {
+  n: number;
+  miru: number;
+  grep: number;
+  saved: number;
+  pct_sum: number;
+}
+
+/** In-memory fold of the JSONL file. */
+export interface BenchmarkHistoryFile extends TokenTotals {
+  repos: Record<string, TokenTotals>;
+}
+
+export function runWithBenchmarkHistoryPath<T>(path: string, fn: () => Promise<T>): Promise<T> {
+  return historyPathStore.run(path, fn);
+}
+
+export function resolveBenchmarkHistoryPath(): string {
+  return (
+    historyPathStore.getStore() ??
+    process.env.MIRU_BENCHMARK_HISTORY_PATH?.trim() ??
+    join(resolveMiruStateDir(), HISTORY_FILENAME)
+  );
+}
+
+/** Absolute local path (no trailing slash), or git URL with trailing slashes stripped. */
+export function normalizeBenchmarkRepoKey(repo: string): string {
+  const trimmed = repo.trim();
+  if (!trimmed) return trimmed;
+  if (isGitUrl(trimmed)) return trimmed.replace(/\/+$/, "") || trimmed;
+  return resolve(trimmed).replace(/[/\\]+$/, "") || resolve(trimmed);
+}
+
+export function recordFromBenchmark(
+  repo: string,
+  benchmark: SearchBenchmarkBlock,
+): BenchmarkContribution {
+  const summary = agentBenchmarkFromTokens(
+    benchmark.miru.workflow_tokens,
+    benchmark.grep_read.workflow_full_tokens,
+    benchmark.accuracy.rank1_match,
+  );
+  summary.save_pct = benchmark.efficiency.token_savings_pct;
+  return recordFromAgentSummary(repo, summary);
+}
+
+export function recordFromAgentSummary(
+  repo: string,
+  summary: Pick<AgentBenchmarkSummary, "miru_tok" | "grep_tok" | "save_pct" | "saved_tok">,
+): BenchmarkContribution {
+  return {
+    r: normalizeBenchmarkRepoKey(repo),
+    m: summary.miru_tok,
+    g: summary.grep_tok,
+    s: summary.saved_tok,
+    p: summary.save_pct,
+  };
+}
+
+function emptyTotals(): TokenTotals {
+  return { n: 0, miru: 0, grep: 0, saved: 0, pct_sum: 0 };
+}
+
+function isContribution(value: unknown): value is BenchmarkContribution {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+  const row = value as Record<string, unknown>;
+  return (
+    typeof row.r === "string" &&
+    [row.m, row.g, row.s, row.p].every((n) => typeof n === "number" && Number.isFinite(n))
+  );
+}
+
+function addContribution(totals: TokenTotals, c: BenchmarkContribution): void {
+  totals.n += 1;
+  totals.miru += c.m;
+  totals.grep += c.g;
+  totals.saved += c.s;
+  totals.pct_sum += c.p;
+}
+
+function foldContributions(rows: BenchmarkContribution[]): BenchmarkHistoryFile {
+  const history: BenchmarkHistoryFile = { ...emptyTotals(), repos: {} };
+  for (const row of rows) {
+    addContribution(history, row);
+    const repo = history.repos[row.r] ?? emptyTotals();
+    addContribution(repo, row);
+    history.repos[row.r] = repo;
+  }
+  return history;
+}
+
+function parseContributions(text: string): BenchmarkContribution[] {
+  const out: BenchmarkContribution[] = [];
+  for (const line of text.split("\n")) {
+    if (!line) continue;
+    const parsed: unknown = JSON.parse(line);
+    if (!isContribution(parsed)) throw new Error("invalid benchmark history line");
+    out.push(parsed);
+  }
+  return out;
+}
+
+async function rotateCorruptHistory(path: string): Promise<void> {
+  await rename(path, `${path}.bak.${Date.now()}`).catch(() => {});
+}
+
+/** Missing/empty → zeros. Bad line → rotate file aside, return zeros. */
+export async function loadBenchmarkHistory(
+  path: string = resolveBenchmarkHistoryPath(),
+): Promise<BenchmarkHistoryFile> {
+  if (!(await Bun.file(path).exists())) return { ...emptyTotals(), repos: {} };
+  const text = await Bun.file(path).text();
+  if (!text.trim()) return { ...emptyTotals(), repos: {} };
+  try {
+    return foldContributions(parseContributions(text));
+  } catch {
+    await rotateCorruptHistory(path);
+    return { ...emptyTotals(), repos: {} };
+  }
+}
+
+export async function clearBenchmarkHistory(
+  path: string = resolveBenchmarkHistoryPath(),
+): Promise<{ cleared: boolean; path: string }> {
+  if (!(await Bun.file(path).exists())) return { cleared: false, path };
+  await Bun.file(path).delete();
+  return { cleared: true, path };
+}
+
+/** Append one JSONL line (`O_APPEND`). */
+export async function appendBenchmarkQuery(
+  contribution: BenchmarkContribution,
+  options?: { path?: string },
+): Promise<void> {
+  const path = options?.path ?? resolveBenchmarkHistoryPath();
+  await mkdir(dirname(path), { recursive: true });
+  await appendFile(path, `${JSON.stringify(contribution)}\n`, "utf8");
+}
+
+function totalsToRollup(t: TokenTotals): AgentBenchmarkRollup {
+  return {
+    n: t.n,
+    saved: t.saved,
+    save_pct: t.n > 0 ? Math.round(t.pct_sum / t.n) : 0,
+    miru: t.miru,
+    grep: t.grep,
+  };
+}
+
+/** Optional `repo` filter. With 2+ repos unfiltered, attach `repos` by saved desc. */
+export function toAgentBenchmarkRollup(
+  history: BenchmarkHistoryFile,
+  options?: { repo?: string },
+): AgentBenchmarkRollup {
+  if (options?.repo) {
+    const key = normalizeBenchmarkRepoKey(options.repo);
+    return totalsToRollup(history.repos[key] ?? emptyTotals());
+  }
+  const out = totalsToRollup(history);
+  const entries = Object.entries(history.repos);
+  if (entries.length > 1) {
+    out.repos = entries
+      .map(([r, t]) => ({ r, n: t.n, saved: t.saved, save_pct: totalsToRollup(t).save_pct }))
+      .sort((a, b) => b.saved - a.saved);
+  }
+  return out;
+}
+
+export async function readBenchmarkRollup(options?: {
+  path?: string;
+  repo?: string;
+}): Promise<AgentBenchmarkRollup> {
+  return toAgentBenchmarkRollup(await loadBenchmarkHistory(options?.path), {
+    repo: options?.repo,
+  });
+}

package/src/benchmark/locate-compare.ts

@@ -0,0 +1,65 @@
+/**
+ * Compare Miru `locate` agent payload tokens vs the costlier native Grep path.
+ *
+ * Baseline mirrors Cursor/Claude Grep-style output: fixed-string matches with
+ * ±GREP_CONTEXT lines (not slim `rg -n`). That is the expensive path agents
+ * take without `locate`, so savings measure real workflow cost — not an
+ * apples-to-apples payload contest against optimally-minimal ripgrep.
+ */
+
+import {
+  formatLiteralLocate,
+  type LiteralLocateOptions,
+  type LiteralLocateResult,
+} from "../literal.ts";
+import type { MiruIndex } from "../miru-index.ts";
+import { countTokens } from "../token-count.ts";
+import { GREP_CONTEXT } from "./grep.ts";
+import { rgLiteralOutput } from "./rg-literal.ts";
+import { agentBenchmarkFromTokens } from "./summary.ts";
+import type { AgentBenchmarkSummary } from "./types.ts";
+
+export { attachAgentBenchmark as attachLocateBenchmark } from "./summary.ts";
+
+export interface LocateBenchmarkComparison {
+  result: LiteralLocateResult;
+  payload: Record<string, unknown>;
+  benchmark: AgentBenchmarkSummary;
+  latency_ms: {
+    miru: number;
+    grep: number;
+  };
+}
+
+export async function benchmarkLocateComparison(options: {
+  literal: string;
+  repoPath: string;
+  index: MiruIndex;
+  locate?: LiteralLocateOptions;
+}): Promise<LocateBenchmarkComparison> {
+  const locateOpts: LiteralLocateOptions = {
+    mode: options.locate?.mode ?? "lines",
+    ignore_case: options.locate?.ignore_case,
+    ...(options.locate?.limit != null ? { limit: options.locate.limit } : {}),
+  };
+
+  const miruStart = performance.now();
+  const result = options.index.locateLiteral(options.literal, locateOpts);
+  const miruMs = performance.now() - miruStart;
+  const payload = formatLiteralLocate(result);
+  const miruTok = countTokens(JSON.stringify(payload));
+
+  // Unbounded agent-style Grep dump (±context) — the costlier path without locate.
+  const grep = await rgLiteralOutput(options.repoPath, options.literal, {
+    context: GREP_CONTEXT,
+    maxCount: 0,
+    ignoreCase: locateOpts.ignore_case,
+  });
+
+  return {
+    result,
+    payload,
+    benchmark: agentBenchmarkFromTokens(miruTok, grep.tokens, result.n > 0),
+    latency_ms: { miru: miruMs, grep: grep.latency_ms },
+  };
+}

package/src/benchmark/rg-literal.ts

@@ -0,0 +1,143 @@
+/** Exact-literal ripgrep baseline for locate benchmarking. */
+
+import { relative } from "node:path";
+import { countTokens } from "../token-count.ts";
+import { RG_EXCLUDE_ARGS, selectBenchmarkSearchTool } from "./grep.ts";
+
+export interface RgLiteralOutput {
+  text: string;
+  tokens: number;
+  latency_ms: number;
+  n: number;
+  files: number;
+}
+
+/** Count match lines (path:line:…) and unique files from `rg -n` output. */
+export function parseRgLiteralStats(text: string, repoRoot: string): { n: number; files: number } {
+  const files = new Set<string>();
+  let n = 0;
+  for (const line of text.split("\n")) {
+    if (!line || line.startsWith("--")) {
+      continue;
+    }
+    // path:line:content or path-line-content for context, including Windows drive letters.
+    const parsed = parsePathLinePrefix(line);
+    if (!parsed) {
+      continue;
+    }
+    files.add(relative(repoRoot, parsed.path).replaceAll("\\", "/"));
+    if (line.includes(`${parsed.path}:${parsed.line}:`)) {
+      n += 1;
+    }
+  }
+  return { n, files: files.size };
+}
+
+function parsePathLinePrefix(line: string): { path: string; line: number } | null {
+  for (let i = 0; i < line.length; i++) {
+    const delim = line[i];
+    if (delim !== ":" && delim !== "-") {
+      continue;
+    }
+    let j = i + 1;
+    while (j < line.length && line[j] >= "0" && line[j] <= "9") {
+      j++;
+    }
+    if (j === i + 1 || j >= line.length) {
+      continue;
+    }
+    const trailing = line[j];
+    if (trailing !== ":" && trailing !== "-") {
+      continue;
+    }
+    const lineNo = Number(line.slice(i + 1, j));
+    if (!Number.isFinite(lineNo)) {
+      continue;
+    }
+    return { path: line.slice(0, i), line: lineNo };
+  }
+  return null;
+}
+
+export async function rgLiteralOutput(
+  repoRoot: string,
+  literal: string,
+  options: {
+    context?: number;
+    maxCount?: number;
+    ignoreCase?: boolean;
+  } = {},
+): Promise<RgLiteralOutput> {
+  const context = options.context ?? 0;
+  const maxCount = options.maxCount ?? 20;
+  const tool = selectBenchmarkSearchTool();
+  if (!tool) {
+    throw new Error("No search tool found in PATH (tried rg, grep, and findstr on Windows)");
+  }
+  const start = performance.now();
+  const proc = Bun.spawn(
+    buildLiteralArgs(tool, repoRoot, literal, context, maxCount, !!options.ignoreCase),
+    {
+      stdout: "pipe",
+      stderr: "pipe",
+      ...(tool === "findstr" ? { cwd: repoRoot } : {}),
+    },
+  );
+  const text = await new Response(proc.stdout).text();
+  await proc.exited;
+  const latency_ms = performance.now() - start;
+  const stats = parseRgLiteralStats(text, repoRoot);
+  return { text, tokens: countTokens(text), latency_ms, ...stats };
+}
+
+function buildLiteralArgs(
+  tool: "rg" | "grep" | "findstr",
+  repoRoot: string,
+  literal: string,
+  context: number,
+  maxCount: number,
+  ignoreCase: boolean,
+): string[] {
+  if (tool === "rg") {
+    const args = ["rg", "-F", "-n", "--no-heading", ...RG_EXCLUDE_ARGS];
+    if (ignoreCase) {
+      args.push("-i");
+    }
+    if (context > 0) {
+      args.push("-C", String(context));
+    }
+    if (maxCount > 0) {
+      args.push("-m", String(maxCount));
+    }
+    args.push(literal, repoRoot);
+    return args;
+  }
+  if (tool === "grep") {
+    const args = [
+      "grep",
+      "-R",
+      "-I",
+      "-F",
+      "-n",
+      "--exclude-dir=node_modules",
+      "--exclude-dir=.git",
+      "--exclude-dir=tokenizer",
+    ];
+    if (ignoreCase) {
+      args.push("-i");
+    }
+    if (context > 0) {
+      args.push("-C", String(context));
+    }
+    if (maxCount > 0) {
+      args.push("-m", String(maxCount));
+    }
+    args.push(literal, repoRoot);
+    return args;
+  }
+  const args = ["findstr", "/S", "/N", "/P", `/C:${literal}`, "*"];
+  if (ignoreCase) {
+    args.splice(4, 0, "/I");
+  }
+  return args;
+}

package/src/benchmark/summary.ts

@@ -0,0 +1,29 @@
+import type { AgentBenchmarkSummary } from "./types.ts";
+
+/** Percent of baseline tokens avoided by Miru (0–100). */
+export function tokenSavingsPct(miruTok: number, grepTok: number): number {
+  return grepTok > 0 ? Math.round((1 - miruTok / grepTok) * 100) : 0;
+}
+
+/** Compact agent-facing savings from Miru vs Grep token counts. */
+export function agentBenchmarkFromTokens(
+  miruTok: number,
+  grepTok: number,
+  rank1: boolean,
+): AgentBenchmarkSummary {
+  return {
+    save_pct: tokenSavingsPct(miruTok, grepTok),
+    miru_tok: miruTok,
+    grep_tok: grepTok,
+    saved_tok: Math.max(0, grepTok - miruTok),
+    rank1,
+  };
+}
+
+/** Attach compact `benchmark` to an MCP tool payload. */
+export function attachAgentBenchmark<T extends Record<string, unknown>>(
+  payload: T,
+  benchmark: AgentBenchmarkSummary,
+): T & { benchmark: AgentBenchmarkSummary } {
+  return { ...payload, benchmark };
+}

package/src/benchmark/types.ts

@@ -0,0 +1,65 @@
+export interface SearchBenchmarkBlock {
+  mode: true;
+  token_count_method: "wordpiece";
+  tokenizer_json: string | null;
+  miru: {
+    search_tokens: number;
+    workflow_tokens: number;
+    latency_ms: number;
+    top_file: string | null;
+    top_files: string[];
+  };
+  grep_read: {
+    search_tokens: number;
+    read_full_tokens: number;
+    read_window_tokens: number;
+    workflow_full_tokens: number;
+    workflow_window_tokens: number;
+    latency_ms: number;
+    top_file: string | null;
+    top_files: string[];
+    pattern: string | null;
+    keywords: string[];
+  };
+  efficiency: {
+    token_savings_pct: number;
+    baseline: "grep_search_plus_read_full";
+  };
+  accuracy: {
+    rank1_match: boolean;
+    top_k_overlap_pct: number;
+    miru_only: string[];
+    grep_only: string[];
+    labeled_recall?: {
+      miru: boolean;
+      grep: boolean;
+    };
+  };
+  overhead: {
+    parallel_total_ms: number;
+    miru_share_ms: number;
+    grep_share_ms: number;
+  };
+}
+
+/** Compact per-search stats attached to MCP `search` responses (agent-facing). */
+export interface AgentBenchmarkSummary {
+  save_pct: number;
+  miru_tok: number;
+  grep_tok: number;
+  saved_tok: number;
+  rank1: boolean;
+  /** Present only when non-empty; capped at 3 paths. */
+  miru_only?: string[];
+}
+
+/** Compact cumulative rollup for MCP `read_benchmark` (agent-facing). */
+export interface AgentBenchmarkRollup {
+  n: number;
+  saved: number;
+  save_pct: number;
+  miru: number;
+  grep: number;
+  /** Only when more than one repo is present. */
+  repos?: Array<{ r: string; n: number; saved: number; save_pct: number }>;
+}