@takara-ai/miru-code 1.1.3 → 1.2.1

package/src/benchmark/compare.ts

@@ -0,0 +1,278 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { relativePathFromRoot } from "../index/incremental.ts";
+import type { MiruIndex } from "../miru-index.ts";
+import { applySnippetsToResults, estimateResultTokens } from "../snippet.ts";
+import { countTokens, tokenCountMethod, tokenizerJsonPath } from "../token-count.ts";
+import type { SearchResult } from "../types.ts";
+import {
+  DEFAULT_EXPAND_AFTER,
+  DEFAULT_EXPAND_BEFORE,
+  dedupeResultsByFile,
+  expandChunksAtLine,
+} from "../utils.ts";
+import { type GrepFileHit, grepSearch } from "./grep.ts";
+import { agentBenchmarkFromTokens, attachAgentBenchmark, tokenSavingsPct } from "./summary.ts";
+import type { AgentBenchmarkSummary, SearchBenchmarkBlock } from "./types.ts";
+
+/** Repo-relative `/`-separated path for all benchmark file identity. */
+function canonicalRepoPath(repoPath: string, filePath: string): string {
+  // Normalize separators first — Node's resolve treats `\` as a literal on POSIX.
+  return relativePathFromRoot(repoPath, filePath.replace(/\\/g, "/"));
+}
+
+function canonicalRepoPaths(repoPath: string, files: string[]): string[] {
+  return files.map((file) => canonicalRepoPath(repoPath, file));
+}
+
+function firstGrepMatchLine(hit: GrepFileHit | undefined): number | null {
+  if (!hit?.output) {
+    return null;
+  }
+  for (const line of hit.output.split("\n")) {
+    if (!line.trim() || line === "--") {
+      continue;
+    }
+    const withPath = line.match(/:(\d+)[:-]/);
+    if (withPath?.[1]) {
+      return Number(withPath[1]);
+    }
+    const bare = line.match(/^(\d+)[:-]/);
+    if (bare?.[1]) {
+      return Number(bare[1]);
+    }
+  }
+  return null;
+}
+
+function expandLineSpan(chunks: { start_line: number; end_line: number }[]): number {
+  if (chunks.length === 0) {
+    return 0;
+  }
+  const start = Math.min(...chunks.map((c) => c.start_line));
+  const end = Math.max(...chunks.map((c) => c.end_line));
+  return end - start + 1;
+}
+
+async function readFileTokens(absPath: string): Promise<number> {
+  try {
+    const text = await readFile(absPath, "utf-8");
+    return countTokens(text);
+  } catch {
+    return 0;
+  }
+}
+
+async function readLineWindowTokens(
+  absPath: string,
+  centerLine: number,
+  lineSpan: number,
+): Promise<number> {
+  try {
+    const text = await readFile(absPath, "utf-8");
+    const lines = text.split("\n");
+    if (lines.length === 0) {
+      return 0;
+    }
+    const half = Math.max(1, Math.floor(lineSpan / 2));
+    const start = Math.max(1, centerLine - half);
+    const end = Math.min(lines.length, centerLine + half);
+    return countTokens(lines.slice(start - 1, end).join("\n"));
+  } catch {
+    return 0;
+  }
+}
+
+function miruExpandTokens(
+  index: MiruIndex,
+  repoPath: string,
+  top: SearchResult | undefined,
+  query: string,
+): { tokens: number; lineSpan: number } {
+  if (!top) {
+    return { tokens: 0, lineSpan: 0 };
+  }
+  const [{ meta }] = applySnippetsToResults([top], query);
+  const line = meta.truncated ? meta.anchor_line : top.chunk.start_line;
+  const { chunks } = expandChunksAtLine(
+    index.chunks,
+    top.chunk.file_path,
+    line,
+    repoPath,
+    DEFAULT_EXPAND_BEFORE,
+    DEFAULT_EXPAND_AFTER,
+  );
+  return {
+    tokens: chunks.reduce((sum, chunk) => sum + countTokens(chunk.content), 0),
+    lineSpan: expandLineSpan(chunks),
+  };
+}
+
+function topKOverlapPct(miruFiles: string[], grepFiles: string[]): number {
+  if (miruFiles.length === 0 && grepFiles.length === 0) {
+    return 100;
+  }
+  const union = new Set([...miruFiles, ...grepFiles]);
+  if (union.size === 0) {
+    return 0;
+  }
+  const overlap = miruFiles.filter((file) => grepFiles.includes(file)).length;
+  return Math.round((overlap / union.size) * 100);
+}
+
+function uniqueFiles(files: string[], other: string[]): string[] {
+  return files.filter((file) => !other.includes(file));
+}
+
+async function grepReadEstimates(
+  repoPath: string,
+  grep: Awaited<ReturnType<typeof grepSearch>>,
+  matchedLineSpan: number,
+): Promise<{ readFull: number; readWindow: number; latencyMs: number }> {
+  const start = performance.now();
+  const topGrepHit = grep.hits[0];
+  const grepTop = topGrepHit?.file ?? null;
+  const grepAbsPath = grepTop ? join(repoPath, grepTop) : null;
+  const grepMatchLine = firstGrepMatchLine(topGrepHit);
+  const readFull = grepAbsPath ? await readFileTokens(grepAbsPath) : 0;
+  const span = matchedLineSpan > 0 ? matchedLineSpan : 60;
+  const readWindow =
+    grepAbsPath && grepMatchLine != null
+      ? await readLineWindowTokens(grepAbsPath, grepMatchLine, span)
+      : 0;
+  return {
+    readFull,
+    readWindow,
+    latencyMs: performance.now() - start,
+  };
+}
+
+export async function benchmarkSearchComparison(options: {
+  query: string;
+  repoPath: string;
+  index: MiruIndex;
+  topK: number;
+  /** Keep only the best hit per file (default true). Matches MCP `dedupe_by_file`. */
+  dedupeByFile?: boolean;
+  relevant?: string[];
+}): Promise<{ benchmark: SearchBenchmarkBlock; results: SearchResult[] }> {
+  const { query, repoPath, index, topK, relevant } = options;
+  const dedupeByFile = options.dedupeByFile !== false;
+  const parallelStart = performance.now();
+
+  const miruPromise = (async () => {
+    const started = performance.now();
+    const raw = await index.search({ query, topK, rerank: true });
+    const results = (dedupeByFile ? dedupeResultsByFile(raw) : raw).slice(0, topK);
+    const snippetResults = applySnippetsToResults(results, query).map((entry) => entry.result);
+    const searchTokens = estimateResultTokens(snippetResults);
+    const expand = miruExpandTokens(index, repoPath, results[0], query);
+    return {
+      results,
+      searchTokens,
+      workflowTokens: searchTokens + expand.tokens,
+      lineSpan: expand.lineSpan,
+      latencyMs: performance.now() - started,
+    };
+  })();
+
+  const grepSearchPromise = (async () => {
+    const started = performance.now();
+    const grep = await grepSearch(repoPath, query, topK);
+    return { grep, latencyMs: performance.now() - started };
+  })();
+
+  const [miruOutcome, grepSearchOutcome] = await Promise.all([miruPromise, grepSearchPromise]);
+  const grepReads = await grepReadEstimates(repoPath, grepSearchOutcome.grep, miruOutcome.lineSpan);
+
+  const parallelTotalMs = performance.now() - parallelStart;
+  const grepOutcome = {
+    grep: grepSearchOutcome.grep,
+    readFull: grepReads.readFull,
+    readWindow: grepReads.readWindow,
+    latencyMs: grepSearchOutcome.latencyMs + grepReads.latencyMs,
+  };
+  const miruFiles = canonicalRepoPaths(
+    repoPath,
+    miruOutcome.results.map((result) => result.chunk.file_path),
+  );
+  const grepFiles = canonicalRepoPaths(repoPath, grepOutcome.grep.files);
+  const miruTop = miruFiles[0] ?? null;
+  const grepTop = grepFiles[0] ?? null;
+  const workflowFull = grepOutcome.grep.tokens + grepOutcome.readFull;
+  const workflowWindow = grepOutcome.grep.tokens + grepOutcome.readWindow;
+  const tokenSavings = tokenSavingsPct(miruOutcome.workflowTokens, workflowFull);
+
+  const block: SearchBenchmarkBlock = {
+    mode: true,
+    token_count_method: tokenCountMethod(),
+    tokenizer_json: tokenizerJsonPath(),
+    miru: {
+      search_tokens: miruOutcome.searchTokens,
+      workflow_tokens: miruOutcome.workflowTokens,
+      latency_ms: Math.round(miruOutcome.latencyMs),
+      top_file: miruTop,
+      top_files: miruFiles,
+    },
+    grep_read: {
+      search_tokens: grepOutcome.grep.tokens,
+      read_full_tokens: grepOutcome.readFull,
+      read_window_tokens: grepOutcome.readWindow,
+      workflow_full_tokens: workflowFull,
+      workflow_window_tokens: workflowWindow,
+      latency_ms: Math.round(grepOutcome.latencyMs),
+      top_file: grepTop,
+      top_files: grepFiles,
+      pattern: grepOutcome.grep.pattern,
+      keywords: grepOutcome.grep.keywords,
+    },
+    efficiency: {
+      token_savings_pct: tokenSavings,
+      baseline: "grep_search_plus_read_full",
+    },
+    accuracy: {
+      rank1_match: miruTop != null && miruTop === grepTop,
+      top_k_overlap_pct: topKOverlapPct(miruFiles, grepFiles),
+      miru_only: uniqueFiles(miruFiles, grepFiles),
+      grep_only: uniqueFiles(grepFiles, miruFiles),
+    },
+    overhead: {
+      parallel_total_ms: Math.round(parallelTotalMs),
+      miru_share_ms: Math.round(miruOutcome.latencyMs),
+      grep_share_ms: Math.round(grepOutcome.latencyMs),
+    },
+  };
+
+  if (relevant && relevant.length > 0) {
+    const relevantFiles = new Set(canonicalRepoPaths(repoPath, relevant));
+    block.accuracy.labeled_recall = {
+      miru: miruFiles.some((file) => relevantFiles.has(file)),
+      grep: grepFiles.some((file) => relevantFiles.has(file)),
+    };
+  }
+
+  return { benchmark: block, results: miruOutcome.results };
+}
+
+const MAX_AGENT_MIRU_ONLY = 3;
+
+/** Strip internal/debug fields down to the compact MCP agent payload. */
+export function toAgentBenchmarkSummary(block: SearchBenchmarkBlock): AgentBenchmarkSummary {
+  const summary = agentBenchmarkFromTokens(
+    block.miru.workflow_tokens,
+    block.grep_read.workflow_full_tokens,
+    block.accuracy.rank1_match,
+  );
+  summary.save_pct = block.efficiency.token_savings_pct;
+  if (block.accuracy.miru_only.length > 0) {
+    summary.miru_only = block.accuracy.miru_only.slice(0, MAX_AGENT_MIRU_ONLY);
+  }
+  return summary;
+}
+
+export function attachSearchBenchmark<T extends Record<string, unknown>>(
+  payload: T,
+  benchmark: SearchBenchmarkBlock,
+): T & { benchmark: AgentBenchmarkSummary } {
+  return attachAgentBenchmark(payload, toAgentBenchmarkSummary(benchmark));
+}

package/src/benchmark/grep.ts

@@ -0,0 +1,352 @@
+/** Native grep baseline for Miru benchmark comparisons. */
+
+import { relative } from "node:path";
+import { countTokens } from "../token-count.ts";
+
+const STOPWORDS = new Set(
+  "a an and are as at be by do does for from has have how if in is it not of on or the to was what when where which who why with".split(
+    " ",
+  ),
+);
+
+export const GREP_LINES_PER_FILE = 3;
+export const GREP_CONTEXT = 2;
+
+/** Shared `-g` excludes for ripgrep baselines (search + locate). */
+export const RG_EXCLUDE_ARGS = [
+  "-g",
+  "!node_modules",
+  "-g",
+  "!.git",
+  "-g",
+  "!tokenizer",
+  "-g",
+  "!tokenizer/**",
+] as const;
+
+export interface GrepFileHit {
+  file: string;
+  matchCount: number;
+  output: string;
+}
+
+export interface GrepSearchResult {
+  files: string[];
+  hits: GrepFileHit[];
+  tokens: number;
+  pattern: string | null;
+  keywords: string[];
+}
+
+export type BenchmarkSearchTool = "rg" | "grep" | "findstr";
+
+export function selectBenchmarkSearchTool(options?: {
+  platform?: NodeJS.Platform;
+  hasRg?: boolean;
+  hasGrep?: boolean;
+  hasFindstr?: boolean;
+}): BenchmarkSearchTool | null {
+  const platform = options?.platform ?? process.platform;
+  const hasRg = options?.hasRg ?? Bun.which("rg") != null;
+  const hasGrep = options?.hasGrep ?? Bun.which("grep") != null;
+  const hasFindstr = options?.hasFindstr ?? Bun.which("findstr") != null;
+  if (hasRg) {
+    return "rg";
+  }
+  if (hasGrep) {
+    return "grep";
+  }
+  if (platform === "win32" && hasFindstr) {
+    return "findstr";
+  }
+  return null;
+}
+export function queryKeywords(query: string): string[] {
+  const words = (query.match(/[a-zA-Z_][a-zA-Z0-9_-]*/g) ?? [])
+    .map((w) => w.toLowerCase())
+    .filter((w) => w.length >= 3 && !STOPWORDS.has(w));
+
+  const unique = [...new Set(words)];
+  unique.sort((a, b) => b.length - a.length);
+  return unique.slice(0, 6);
+}
+
+function escapeRegex(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function buildGrepPattern(keywords: string[]): string | null {
+  if (keywords.length === 0) {
+    return null;
+  }
+  return keywords.map(escapeRegex).join("|");
+}
+
+function normalizeRepoFile(repoRoot: string, filePath: string): string {
+  const rootNorm = repoRoot.replace(/\\/g, "/").replace(/\/+$/, "");
+  const fileNorm = filePath.replace(/\\/g, "/");
+  const rootPrefix = `${rootNorm}/`;
+  if (fileNorm === rootNorm) {
+    return "";
+  }
+  if (fileNorm.toLowerCase().startsWith(rootPrefix.toLowerCase())) {
+    return fileNorm.slice(rootPrefix.length);
+  }
+  const rel = relative(repoRoot, filePath).replace(/\\/g, "/");
+  if (rel.startsWith("../") || /^[A-Za-z]:\//.test(rel)) {
+    return fileNorm;
+  }
+  return rel;
+}
+
+function parsePathLinePrefix(line: string): { path: string; line: number } | null {
+  for (let i = 0; i < line.length; i++) {
+    const delim = line[i];
+    if (delim !== ":" && delim !== "-") {
+      continue;
+    }
+    let j = i + 1;
+    while (j < line.length && line[j] >= "0" && line[j] <= "9") {
+      j++;
+    }
+    if (j === i + 1 || j >= line.length) {
+      continue;
+    }
+    const trailing = line[j];
+    if (trailing !== ":" && trailing !== "-") {
+      continue;
+    }
+    const lineNo = Number(line.slice(i + 1, j));
+    if (!Number.isFinite(lineNo)) {
+      continue;
+    }
+    return { path: line.slice(0, i), line: lineNo };
+  }
+  return null;
+}
+
+export async function grepSearch(
+  repoRoot: string,
+  query: string,
+  topK: number,
+): Promise<GrepSearchResult> {
+  const keywords = queryKeywords(query);
+  const pattern = buildGrepPattern(keywords);
+  if (!pattern) {
+    return { files: [], hits: [], tokens: 0, pattern: null, keywords };
+  }
+
+  const tool = selectBenchmarkSearchTool();
+  if (!tool) {
+    throw new Error("No search tool found in PATH (tried rg, grep, and findstr on Windows)");
+  }
+
+  const ranked =
+    tool === "rg"
+      ? await rgRankedMatches(repoRoot, pattern, topK)
+      : tool === "grep"
+        ? await grepRankedMatches(repoRoot, pattern, topK)
+        : await findstrRankedMatches(repoRoot, keywords, topK);
+
+  const hits: GrepFileHit[] = [];
+  let tokens = 0;
+
+  for (const row of ranked) {
+    const output =
+      tool === "rg"
+        ? await rgFilePreview(row.absPath, pattern)
+        : tool === "grep"
+          ? await grepFilePreview(row.absPath, pattern)
+          : await findstrFilePreview(row.absPath, keywords);
+    hits.push({ file: row.file, matchCount: row.matchCount, output });
+    tokens += countTokens(output);
+  }
+
+  return {
+    files: ranked.map((r) => r.file),
+    hits,
+    tokens,
+    pattern,
+    keywords,
+  };
+}
+
+async function rgRankedMatches(repoRoot: string, pattern: string, topK: number) {
+  const countProc = Bun.spawn(
+    ["rg", "-i", "--count-matches", pattern, repoRoot, ...RG_EXCLUDE_ARGS],
+    { stdout: "pipe", stderr: "pipe" },
+  );
+  const countText = await new Response(countProc.stdout).text();
+  await countProc.exited;
+  return parseCountMatches(repoRoot, countText, topK);
+}
+
+async function grepRankedMatches(repoRoot: string, pattern: string, topK: number) {
+  const countProc = Bun.spawn(
+    [
+      "grep",
+      "-R",
+      "-I",
+      "-i",
+      "-E",
+      "-o",
+      "--exclude-dir=node_modules",
+      "--exclude-dir=.git",
+      "--exclude-dir=tokenizer",
+      pattern,
+      repoRoot,
+    ],
+    { stdout: "pipe", stderr: "pipe" },
+  );
+  const countText = await new Response(countProc.stdout).text();
+  await countProc.exited;
+  const counts = new Map<string, number>();
+  for (const line of countText.split("\n")) {
+    if (!line) {
+      continue;
+    }
+    const split = splitSearchToolLine(line);
+    if (!split) {
+      continue;
+    }
+    const absPath = split.path;
+    counts.set(absPath, (counts.get(absPath) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .map(([absPath, matchCount]) => ({
+      file: normalizeRepoFile(repoRoot, absPath),
+      absPath,
+      matchCount,
+    }))
+    .sort((a, b) => b.matchCount - a.matchCount)
+    .slice(0, topK);
+}
+
+function splitSearchToolLine(line: string): { path: string; rest: string } | null {
+  if (/^[A-Za-z]:/.test(line)) {
+    const colon = line.indexOf(":", 2);
+    if (colon < 0) {
+      return null;
+    }
+    return { path: line.slice(0, colon), rest: line.slice(colon + 1) };
+  }
+  const colon = line.indexOf(":");
+  if (colon < 0) {
+    return null;
+  }
+  return { path: line.slice(0, colon), rest: line.slice(colon + 1) };
+}
+
+async function findstrRankedMatches(repoRoot: string, keywords: string[], topK: number) {
+  const counts = new Map<string, number>();
+  for (const keyword of keywords) {
+    const proc = Bun.spawn(["findstr", "/S", "/N", "/I", "/P", `/C:${keyword}`, "*"], {
+      stdout: "pipe",
+      stderr: "pipe",
+      cwd: repoRoot,
+    });
+    const text = await new Response(proc.stdout).text();
+    await proc.exited;
+    for (const line of text.split("\n")) {
+      const parsed = parsePathLinePrefix(line);
+      if (!parsed?.path) {
+        continue;
+      }
+      const pathText = parsed.path.replace(/\\/g, "/");
+      const absPath = /^[A-Za-z]:\//.test(pathText)
+        ? pathText
+        : `${repoRoot}/${pathText}`.replace(/\\/g, "/");
+      if (
+        absPath.includes("/node_modules/") ||
+        absPath.includes("/.git/") ||
+        absPath.includes("/tokenizer/")
+      ) {
+        continue;
+      }
+      counts.set(absPath, (counts.get(absPath) ?? 0) + 1);
+    }
+  }
+  return [...counts.entries()]
+    .map(([absPath, matchCount]) => ({
+      file: normalizeRepoFile(repoRoot, absPath),
+      absPath,
+      matchCount,
+    }))
+    .sort((a, b) => b.matchCount - a.matchCount)
+    .slice(0, topK);
+}
+
+function parseCountMatches(repoRoot: string, countText: string, topK: number) {
+  return countText
+    .split("\n")
+    .filter(Boolean)
+    .map((line) => {
+      const colon = line.lastIndexOf(":");
+      if (colon < 0) {
+        return null;
+      }
+      const absPath = line.slice(0, colon);
+      const matchCount = Number(line.slice(colon + 1));
+      if (!Number.isFinite(matchCount)) {
+        return null;
+      }
+      return {
+        file: normalizeRepoFile(repoRoot, absPath),
+        absPath,
+        matchCount,
+      };
+    })
+    .filter((x): x is NonNullable<typeof x> => x != null)
+    .sort((a, b) => b.matchCount - a.matchCount)
+    .slice(0, topK);
+}
+
+async function rgFilePreview(absPath: string, pattern: string): Promise<string> {
+  const proc = Bun.spawn(
+    [
+      "rg",
+      "-i",
+      "-n",
+      "-C",
+      String(GREP_CONTEXT),
+      "-m",
+      String(GREP_LINES_PER_FILE),
+      pattern,
+      absPath,
+    ],
+    { stdout: "pipe", stderr: "pipe" },
+  );
+  const output = await new Response(proc.stdout).text();
+  await proc.exited;
+  return output;
+}
+
+async function grepFilePreview(absPath: string, pattern: string): Promise<string> {
+  const proc = Bun.spawn(
+    [
+      "grep",
+      "-I",
+      "-i",
+      "-n",
+      "-E",
+      "-C",
+      String(GREP_CONTEXT),
+      "-m",
+      String(GREP_LINES_PER_FILE),
+      pattern,
+      absPath,
+    ],
+    { stdout: "pipe", stderr: "pipe" },
+  );
+  const output = await new Response(proc.stdout).text();
+  await proc.exited;
+  return output;
+}
+
+async function findstrFilePreview(absPath: string, keywords: string[]): Promise<string> {
+  const args = ["findstr", "/N", "/I", "/P", ...keywords.map((k) => `/C:${k}`), absPath];
+  const proc = Bun.spawn(args, { stdout: "pipe", stderr: "pipe" });
+  const output = await new Response(proc.stdout).text();
+  await proc.exited;
+  return output;
+}