@takara-ai/miru-code 1.1.2 → 1.2.1

package/src/literal.ts

@@ -0,0 +1,139 @@
+/** Exact literal location over already-indexed Miru chunks (no disk re-walk). */
+
+import type { Chunk } from "./types.ts";
+
+export type LiteralMode = "count" | "locations" | "lines";
+
+export interface LiteralHit {
+  file_path: string;
+  line: number;
+  /** Present only in `lines` mode. */
+  text?: string;
+}
+
+export interface LiteralLocateResult {
+  literal: string;
+  mode: LiteralMode;
+  /** Total unique file:line matches. */
+  n: number;
+  /** Unique files with at least one match. */
+  files: number;
+  /** True only when an explicit `limit` cut the hit list short. */
+  truncated: boolean;
+  hits: LiteralHit[];
+}
+
+export interface LiteralLocateOptions {
+  mode?: LiteralMode;
+  /**
+   * Optional cap on returned hits (ignored for `count`).
+   * Omit for all matches — agents must not fall back to Grep when truncated.
+   */
+  limit?: number;
+  ignore_case?: boolean;
+}
+
+function resolveLimit(limit: number | undefined): number | null {
+  if (limit == null || !Number.isFinite(limit)) {
+    return null;
+  }
+  return Math.max(1, Math.floor(limit));
+}
+
+function includesLiteral(haystack: string, needle: string, ignoreCase: boolean): boolean {
+  if (!ignoreCase) {
+    return haystack.includes(needle);
+  }
+  return haystack.toLowerCase().includes(needle.toLowerCase());
+}
+
+/**
+ * Scan indexed chunks for an exact substring. Dedupes by file:line when chunks overlap.
+ * Only covers indexed content (same corpus as `search` / `expand`).
+ * Returns every match unless `limit` is set.
+ */
+export function locateLiteral(
+  chunks: readonly Chunk[],
+  literal: string,
+  options: LiteralLocateOptions = {},
+): LiteralLocateResult {
+  const needle = literal;
+  const mode = options.mode ?? "lines";
+  const ignoreCase = options.ignore_case === true;
+  const limit = resolveLimit(options.limit);
+
+  if (!needle) {
+    return { literal: needle, mode, n: 0, files: 0, truncated: false, hits: [] };
+  }
+
+  const seen = new Set<string>();
+  const fileSet = new Set<string>();
+  const hits: LiteralHit[] = [];
+  let n = 0;
+
+  for (const chunk of chunks) {
+    if (!includesLiteral(chunk.content, needle, ignoreCase)) {
+      continue;
+    }
+    const lines = chunk.content.split("\n");
+    for (let i = 0; i < lines.length; i++) {
+      const lineText = lines[i] ?? "";
+      if (!includesLiteral(lineText, needle, ignoreCase)) {
+        continue;
+      }
+      const line = chunk.start_line + i;
+      const key = `${chunk.file_path}:${line}`;
+      if (seen.has(key)) {
+        continue;
+      }
+      seen.add(key);
+      fileSet.add(chunk.file_path);
+      n += 1;
+      if (mode === "count") {
+        continue;
+      }
+      if (limit != null && hits.length >= limit) {
+        continue;
+      }
+      const hit: LiteralHit = { file_path: chunk.file_path, line };
+      if (mode === "lines") {
+        hit.text = lineText;
+      }
+      hits.push(hit);
+    }
+  }
+
+  return {
+    literal: needle,
+    mode,
+    n,
+    files: fileSet.size,
+    truncated: mode !== "count" && limit != null && n > hits.length,
+    hits: mode === "count" ? [] : hits,
+  };
+}
+
+/** Compact agent-facing JSON (minimal keys). */
+export function formatLiteralLocate(result: LiteralLocateResult): Record<string, unknown> {
+  const base: Record<string, unknown> = {
+    literal: result.literal,
+    mode: result.mode,
+    n: result.n,
+    files: result.files,
+  };
+
+  if (result.mode === "count") {
+    return base;
+  }
+
+  if (result.truncated) {
+    base.truncated = true;
+  }
+  base.hits = result.hits.map((hit) => {
+    if (hit.text !== undefined) {
+      return { f: hit.file_path, l: hit.line, t: hit.text };
+    }
+    return { f: hit.file_path, l: hit.line };
+  });
+  return base;
+}

package/src/mcp/index-cache.ts

@@ -4,7 +4,7 @@ import { walkFiles } from "../index/file-walker.ts";
 import { getExtensions } from "../index/files.ts";
 import { normalizeRelativePath, relativePathFromRoot } from "../index/incremental.ts";
 import { MiruIndex } from "../miru-index.ts";
-import type { ContentType } from "../types.ts";
+import { type ContentType, defaultContentTypes } from "../types.ts";
 import {
   computeSourceCacheKey,
   isAllowedRepoSource,
@@ -71,7 +71,7 @@ export class IndexCache {
   private readonly entries = new Map<string, CacheEntry>();
   readonly watchers = new Map<string, WatcherHandle>();
 
-  constructor(content: ContentType[] = ["code"], defaultRef: string | null = null) {
+  constructor(content: ContentType[] = defaultContentTypes(), defaultRef: string | null = null) {
     this.content = content;
     this.defaultRef = defaultRef;
   }

package/src/mcp/serve.ts

@@ -1,4 +1,4 @@
-import type { ContentType } from "../types.ts";
+import { type ContentType, defaultContentTypes } from "../types.ts";
 import { IndexCache } from "./index-cache.ts";
 import { createMcpServer } from "./server.ts";
 import { StdioTransport } from "./stdio.ts";
@@ -6,9 +6,10 @@ import { StdioTransport } from "./stdio.ts";
 export async function serveMcp(options: {
   ref?: string | null;
   content?: ContentType[];
+  benchmark?: boolean;
 }): Promise<void> {
-  const cache = new IndexCache(options.content ?? ["code"], options.ref ?? null);
-  const server = createMcpServer(cache);
+  const cache = new IndexCache(options.content ?? defaultContentTypes(), options.ref ?? null);
+  const server = createMcpServer(cache, { benchmark: options.benchmark ?? false });
   const transport = new StdioTransport();
   await server.connect(transport);
 }

package/src/mcp/server.ts

@@ -1,14 +1,28 @@
 import * as z from "zod";
 import packageJson from "../../package.json";
+import { attachSearchBenchmark, benchmarkSearchComparison } from "../benchmark/compare.ts";
 import {
+  appendBenchmarkQuery,
+  readBenchmarkRollup,
+  recordFromAgentSummary,
+  recordFromBenchmark,
+} from "../benchmark/history.ts";
+import { attachLocateBenchmark, benchmarkLocateComparison } from "../benchmark/locate-compare.ts";
+import {
+  MCP_BENCHMARK_SERVER_INSTRUCTIONS,
   MCP_EXPAND_TOOL_DESCRIPTION,
   MCP_FIND_RELATED_TOOL_DESCRIPTION,
+  MCP_LOCATE_TOOL_DESCRIPTION,
+  MCP_READ_BENCHMARK_TOOL_DESCRIPTION,
   MCP_SEARCH_TOOL_DESCRIPTION,
   MCP_SERVER_INSTRUCTIONS,
 } from "../installer/search-policy.ts";
+import { formatLiteralLocate, type LiteralMode } from "../literal.ts";
 import type { ContentType } from "../types.ts";
 import {
   clampMcpTopK,
+  DEFAULT_EXPAND_AFTER,
+  DEFAULT_EXPAND_BEFORE,
   DEFAULT_MCP_TOP_K,
   dedupeResultsByFile,
   expandChunksAtLine,
@@ -26,14 +40,31 @@ const REPO_DESCRIPTION =
   "Pass the project root for local workspaces. " +
   "The index is built on the first tool call and cached for the session.";
 
-export function createMcpServer(cache: IndexCache): MiruMcpServer {
+const BENCHMARK_LOCAL_ONLY_NOTE =
+  "Benchmark comparisons require a local repo path; git URL repos are skipped.";
+
+function withBenchmarkSkipped<T extends Record<string, unknown>>(
+  payload: T,
+): T & { benchmark_skipped: "local_repo_only"; note: string } {
+  return {
+    ...payload,
+    benchmark_skipped: "local_repo_only",
+    note: BENCHMARK_LOCAL_ONLY_NOTE,
+  };
+}
+
+export function createMcpServer(
+  cache: IndexCache,
+  options?: { benchmark?: boolean },
+): MiruMcpServer {
+  const benchmark = options?.benchmark ?? false;
   const server = new MiruMcpServer(
     {
       name: "miru",
       version: packageJson.version,
     },
     {
-      instructions: MCP_SERVER_INSTRUCTIONS,
+      instructions: benchmark ? MCP_BENCHMARK_SERVER_INSTRUCTIONS : MCP_SERVER_INSTRUCTIONS,
     },
   );
 
@@ -64,15 +95,116 @@ export function createMcpServer(cache: IndexCache): MiruMcpServer {
     async ({ query, repo, top_k: topK, dedupe_by_file: dedupeByFile }) => {
       try {
         const index = await getIndexForRepo(repo, cache);
-        let results = await index.search({ query, topK: clampMcpTopK(topK) });
+        const repoRoot = localRepoRoot(repo);
+        const k = clampMcpTopK(topK);
+
+        if (benchmark && repoRoot) {
+          const comparison = await benchmarkSearchComparison({
+            query,
+            repoPath: repoRoot,
+            index,
+            topK: k,
+            dedupeByFile: dedupeByFile !== false,
+          });
+          const results = comparison.results;
+          if (results.length === 0) {
+            return toolText(JSON.stringify({ error: "No results found." }));
+          }
+          try {
+            await appendBenchmarkQuery(recordFromBenchmark(repoRoot, comparison.benchmark));
+          } catch {
+            // History is best-effort; never fail the search on persist errors.
+          }
+          return toolText(
+            JSON.stringify(
+              attachSearchBenchmark(
+                formatResults(query, results, { repoRoot, snippet: true }),
+                comparison.benchmark,
+              ),
+            ),
+          );
+        }
+
+        let results = await index.search({ query, topK: k });
         if (dedupeByFile !== false) {
           results = dedupeResultsByFile(results);
         }
         if (results.length === 0) {
           return toolText(JSON.stringify({ error: "No results found." }));
         }
-        const repoRoot = localRepoRoot(repo);
-        return toolText(JSON.stringify(formatResults(query, results, { repoRoot, snippet: true })));
+        const payload = formatResults(query, results, { repoRoot, snippet: true });
+        if (benchmark && !repoRoot) {
+          return toolText(JSON.stringify(withBenchmarkSkipped(payload)));
+        }
+        return toolText(JSON.stringify(payload));
+      } catch (err) {
+        return toolText(err instanceof Error ? err.message : String(err));
+      }
+    },
+  );
+
+  server.registerTool(
+    "locate",
+    {
+      description: MCP_LOCATE_TOOL_DESCRIPTION,
+      inputSchema: {
+        literal: z
+          .string()
+          .min(1)
+          .describe("Exact substring to find (env var, symbol, error code, quoted text)."),
+        repo: z.string().describe(REPO_DESCRIPTION),
+        mode: z
+          .enum(["count", "locations", "lines"])
+          .optional()
+          .describe(
+            "count=totals only; locations=path:line; lines=path:line+text (default). Prefer count/locations when possible.",
+          ),
+        limit: z
+          .number()
+          .int()
+          .min(1)
+          .optional()
+          .describe(
+            "Optional cap on returned hits. Omit to return ALL matches (recommended). Do not fall back to Grep if n is large — use mode=count or locations instead.",
+          ),
+        ignore_case: z.boolean().optional().describe("Case-insensitive match (default false)."),
+      },
+    },
+    async ({ literal, repo, mode, limit, ignore_case: ignoreCase }) => {
+      try {
+        const index = await getIndexForRepo(repo, cache);
+        const locateOpts = {
+          mode: (mode as LiteralMode | undefined) ?? "lines",
+          ...(limit != null ? { limit } : {}),
+          ignore_case: ignoreCase,
+        };
+
+        if (benchmark) {
+          const repoRoot = localRepoRoot(repo);
+          if (repoRoot) {
+            const comparison = await benchmarkLocateComparison({
+              literal,
+              repoPath: repoRoot,
+              index,
+              locate: locateOpts,
+            });
+            try {
+              await appendBenchmarkQuery(recordFromAgentSummary(repoRoot, comparison.benchmark));
+            } catch {
+              // History is best-effort; never fail locate on persist errors.
+            }
+            return toolText(
+              JSON.stringify(attachLocateBenchmark(comparison.payload, comparison.benchmark)),
+            );
+          }
+        }
+
+        const result = index.locateLiteral(literal, locateOpts);
+        const payload = formatLiteralLocate(result);
+        if (benchmark && !localRepoRoot(repo)) {
+          return toolText(JSON.stringify(withBenchmarkSkipped(payload)));
+        }
+        return toolText(JSON.stringify(payload));
       } catch (err) {
         return toolText(err instanceof Error ? err.message : String(err));
       }
@@ -97,21 +229,21 @@ export function createMcpServer(cache: IndexCache): MiruMcpServer {
           .int()
           .min(0)
           .optional()
-          .describe("Extra chunks before the anchor (default 1)."),
+          .describe(`Extra chunks before the anchor (default ${DEFAULT_EXPAND_BEFORE}).`),
         after: z
           .number()
           .int()
           .min(0)
           .optional()
-          .describe("Extra chunks after the anchor (default 1)."),
+          .describe(`Extra chunks after the anchor (default ${DEFAULT_EXPAND_AFTER}).`),
       },
     },
     async ({ file_path: filePath, anchor_line: anchorLine, repo, before, after }) => {
       try {
         const index = await getIndexForRepo(repo, cache);
         const repoRoot = localRepoRoot(repo);
-        const beforeCount = before ?? 1;
-        const afterCount = after ?? 1;
+        const beforeCount = before ?? DEFAULT_EXPAND_BEFORE;
+        const afterCount = after ?? DEFAULT_EXPAND_AFTER;
         const { anchor, chunks: expanded } = expandChunksAtLine(
           index.chunks,
           filePath,
@@ -194,6 +326,31 @@ export function createMcpServer(cache: IndexCache): MiruMcpServer {
     },
   );
 
+  if (benchmark) {
+    server.registerTool(
+      "read_benchmark",
+      {
+        description: MCP_READ_BENCHMARK_TOOL_DESCRIPTION,
+        inputSchema: {
+          repo: z
+            .string()
+            .optional()
+            .describe(
+              "Optional local repo path or git URL to filter the rollup. Omit for overall totals.",
+            ),
+        },
+      },
+      async ({ repo }) => {
+        try {
+          const rollup = await readBenchmarkRollup({ repo });
+          return toolText(JSON.stringify(rollup));
+        } catch (err) {
+          return toolText(err instanceof Error ? err.message : String(err));
+        }
+      },
+    );
+  }
+
   return server;
 }
 

package/src/miru-index.ts

@@ -14,9 +14,10 @@ import { createIndexFromPath } from "./index/create.ts";
 import { applyIncrementalFileChanges } from "./index/incremental.ts";
 import { persistencePaths, saveIndexBundle } from "./index/persistence.ts";
 import type { SemanticIndex } from "./index/semantic-index.ts";
+import { type LiteralLocateOptions, type LiteralLocateResult, locateLiteral } from "./literal.ts";
 import { hybridSearch, searchSemanticOnly } from "./search.ts";
 import type { Chunk, ContentType, SearchResult } from "./types.ts";
-import { chunkKey, chunkToDict } from "./types.ts";
+import { chunkKey, chunkToDict, defaultContentTypes } from "./types.ts";
 import { computeSourceCacheKey, isGitUrl } from "./utils.ts";
 import { indexCacheEpoch } from "./version.ts";
 
@@ -120,7 +121,7 @@ export class MiruIndex {
     this.chunksInternal = options.chunks;
     this.embeddingModel = options.embeddingModel;
     this._root = options.root ?? null;
-    this.content = options.content ?? ["code"];
+    this.content = options.content ?? defaultContentTypes();
     this.loadedFromDiskFlag = options.loadedFromDisk ?? false;
     this.storedFileMtimes = new Map(Object.entries(options.storedFileMtimes ?? {}));
     this.fileMapping = new Map();
@@ -162,7 +163,7 @@ export class MiruIndex {
    */
   static async fromGit(
     url: string,
-    content: ContentType[] = ["code"],
+    content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
     ref?: string | null,
   ): Promise<MiruIndex> {
@@ -202,7 +203,7 @@ export class MiruIndex {
   /** Resolve a local path or git URL and delegate to `fromPath` or `fromGit`. */
   static async fromSource(
     source: string,
-    content: ContentType[] = ["code"],
+    content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
     ref?: string | null,
   ): Promise<MiruIndex> {
@@ -221,7 +222,7 @@ export class MiruIndex {
    */
   static async fromPath(
     path: string,
-    content: ContentType[] = ["code"],
+    content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
   ): Promise<MiruIndex> {
     const resolved = resolve(path);
@@ -271,7 +272,7 @@ export class MiruIndex {
   static async loadFromDisk(path: string, embeddingModel?: string): Promise<MiruIndex> {
     const model = embeddingModel ?? resolveEmbeddingModel();
     const { bm25, semantic, chunks, metadata } = await loadCachedIndex(path);
-    const content = (metadata.content_type as ContentType[]) ?? ["code"];
+    const content = (metadata.content_type as ContentType[]) ?? defaultContentTypes();
     const root = metadata.root_path ? String(metadata.root_path) : null;
     const storedFileMtimes = (metadata.file_mtimes as Record<string, number>) ?? {};
 
@@ -444,4 +445,12 @@ export class MiruIndex {
 
     return results.filter((r) => chunkKey(r.chunk) !== chunkKey(target)).slice(0, topK);
   }
+
+  /**
+   * Exact substring location over indexed chunks (no disk re-walk).
+   * Prefer for known literals; use `search` for meaning-based questions.
+   */
+  locateLiteral(literal: string, options?: LiteralLocateOptions): LiteralLocateResult {
+    return locateLiteral(this.chunksInternal, literal, options);
+  }
 }

package/src/snippet.ts

@@ -149,6 +149,9 @@ export function applySnippetsToResults(
   });
 }
 
-export function estimateResultTokens(results: SearchResult[]): number {
-  return results.reduce((sum, r) => sum + Math.floor(r.chunk.content.length / 4), 0);
-}
+export {
+  countTokens,
+  estimateResultTokens,
+  tokenCountMethod,
+  tokenizerJsonPath,
+} from "./token-count.ts";

package/src/token-count.ts

@@ -0,0 +1,62 @@
+import { existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { BertWordPieceTokenizer } from "./tokenizer/bert-wordpiece.ts";
+import { loadTokenizerFromFile } from "./tokenizer/load.ts";
+import type { SearchResult } from "./types.ts";
+
+const TOKENIZER_JSON_ENV = "MIRU_TOKENIZER_JSON";
+const PACKAGE_ROOT = resolve(fileURLToPath(new URL("..", import.meta.url)));
+const DEFAULT_TOKENIZER_JSON = join(PACKAGE_ROOT, "tokenizer", "tokenizer.json");
+
+let cached: BertWordPieceTokenizer | undefined;
+let cachedPath: string | null = null;
+
+function resolveTokenizerJsonPath(): string {
+  const fromEnv = process.env[TOKENIZER_JSON_ENV]?.trim();
+  if (fromEnv) {
+    return resolve(fromEnv);
+  }
+  return DEFAULT_TOKENIZER_JSON;
+}
+
+function loadTokenizer(): BertWordPieceTokenizer {
+  if (cached) {
+    return cached;
+  }
+
+  const path = resolveTokenizerJsonPath();
+  if (!existsSync(path)) {
+    throw new Error(
+      `Tokenizer not found at ${path}. Set ${TOKENIZER_JSON_ENV} or add tokenizer/tokenizer.json to the package.`,
+    );
+  }
+
+  cached = loadTokenizerFromFile(path);
+  cachedPath = path;
+  return cached;
+}
+
+/** Reset cached tokenizer (for tests). */
+export function resetTokenizerCache(): void {
+  cached = undefined;
+  cachedPath = null;
+}
+
+export function tokenizerJsonPath(): string {
+  loadTokenizer();
+  return cachedPath ?? resolveTokenizerJsonPath();
+}
+
+/** @deprecated Use tokenCountMethod() — kept for benchmark response compatibility. */
+export function tokenCountMethod(): "wordpiece" {
+  return "wordpiece";
+}
+
+export function countTokens(text: string): number {
+  return loadTokenizer().count(text);
+}
+
+export function estimateResultTokens(results: SearchResult[]): number {
+  return results.reduce((sum, result) => sum + countTokens(result.chunk.content), 0);
+}