@takara-ai/miru-code 1.1.3 → 1.2.2

package/src/mcp/server.ts

@@ -1,14 +1,28 @@
 import * as z from "zod";
 import packageJson from "../../package.json";
+import { attachSearchBenchmark, benchmarkSearchComparison } from "../benchmark/compare.ts";
 import {
+  appendBenchmarkQuery,
+  readBenchmarkRollup,
+  recordFromAgentSummary,
+  recordFromBenchmark,
+} from "../benchmark/history.ts";
+import { attachLocateBenchmark, benchmarkLocateComparison } from "../benchmark/locate-compare.ts";
+import {
+  MCP_BENCHMARK_SERVER_INSTRUCTIONS,
   MCP_EXPAND_TOOL_DESCRIPTION,
   MCP_FIND_RELATED_TOOL_DESCRIPTION,
+  MCP_LOCATE_TOOL_DESCRIPTION,
+  MCP_READ_BENCHMARK_TOOL_DESCRIPTION,
   MCP_SEARCH_TOOL_DESCRIPTION,
   MCP_SERVER_INSTRUCTIONS,
 } from "../installer/search-policy.ts";
+import { formatLiteralLocate, type LiteralMode } from "../literal.ts";
 import type { ContentType } from "../types.ts";
 import {
   clampMcpTopK,
+  DEFAULT_EXPAND_AFTER,
+  DEFAULT_EXPAND_BEFORE,
   DEFAULT_MCP_TOP_K,
   dedupeResultsByFile,
   expandChunksAtLine,
@@ -26,14 +40,31 @@ const REPO_DESCRIPTION =
   "Pass the project root for local workspaces. " +
   "The index is built on the first tool call and cached for the session.";
 
-export function createMcpServer(cache: IndexCache): MiruMcpServer {
+const BENCHMARK_LOCAL_ONLY_NOTE =
+  "Benchmark comparisons require a local repo path; git URL repos are skipped.";
+
+function withBenchmarkSkipped<T extends Record<string, unknown>>(
+  payload: T,
+): T & { benchmark_skipped: "local_repo_only"; note: string } {
+  return {
+    ...payload,
+    benchmark_skipped: "local_repo_only",
+    note: BENCHMARK_LOCAL_ONLY_NOTE,
+  };
+}
+
+export function createMcpServer(
+  cache: IndexCache,
+  options?: { benchmark?: boolean },
+): MiruMcpServer {
+  const benchmark = options?.benchmark ?? false;
   const server = new MiruMcpServer(
     {
       name: "miru",
       version: packageJson.version,
     },
     {
-      instructions: MCP_SERVER_INSTRUCTIONS,
+      instructions: benchmark ? MCP_BENCHMARK_SERVER_INSTRUCTIONS : MCP_SERVER_INSTRUCTIONS,
     },
   );
 
@@ -64,15 +95,116 @@ export function createMcpServer(cache: IndexCache): MiruMcpServer {
     async ({ query, repo, top_k: topK, dedupe_by_file: dedupeByFile }) => {
       try {
         const index = await getIndexForRepo(repo, cache);
-        let results = await index.search({ query, topK: clampMcpTopK(topK) });
+        const repoRoot = localRepoRoot(repo);
+        const k = clampMcpTopK(topK);
+
+        if (benchmark && repoRoot) {
+          const comparison = await benchmarkSearchComparison({
+            query,
+            repoPath: repoRoot,
+            index,
+            topK: k,
+            dedupeByFile: dedupeByFile !== false,
+          });
+          const results = comparison.results;
+          if (results.length === 0) {
+            return toolText(JSON.stringify({ error: "No results found." }));
+          }
+          try {
+            await appendBenchmarkQuery(recordFromBenchmark(repoRoot, comparison.benchmark));
+          } catch {
+            // History is best-effort; never fail the search on persist errors.
+          }
+          return toolText(
+            JSON.stringify(
+              attachSearchBenchmark(
+                formatResults(query, results, { repoRoot, snippet: true }),
+                comparison.benchmark,
+              ),
+            ),
+          );
+        }
+
+        let results = await index.search({ query, topK: k });
         if (dedupeByFile !== false) {
           results = dedupeResultsByFile(results);
         }
         if (results.length === 0) {
           return toolText(JSON.stringify({ error: "No results found." }));
         }
-        const repoRoot = localRepoRoot(repo);
-        return toolText(JSON.stringify(formatResults(query, results, { repoRoot, snippet: true })));
+        const payload = formatResults(query, results, { repoRoot, snippet: true });
+        if (benchmark && !repoRoot) {
+          return toolText(JSON.stringify(withBenchmarkSkipped(payload)));
+        }
+        return toolText(JSON.stringify(payload));
+      } catch (err) {
+        return toolText(err instanceof Error ? err.message : String(err));
+      }
+    },
+  );
+
+  server.registerTool(
+    "locate",
+    {
+      description: MCP_LOCATE_TOOL_DESCRIPTION,
+      inputSchema: {
+        literal: z
+          .string()
+          .min(1)
+          .describe("Exact substring to find (env var, symbol, error code, quoted text)."),
+        repo: z.string().describe(REPO_DESCRIPTION),
+        mode: z
+          .enum(["count", "locations", "lines"])
+          .optional()
+          .describe(
+            "count=totals only; locations=path:line; lines=path:line+text (default). Prefer count/locations when possible.",
+          ),
+        limit: z
+          .number()
+          .int()
+          .min(1)
+          .optional()
+          .describe(
+            "Optional cap on returned hits. Omit to return ALL matches (recommended). Do not fall back to Grep if n is large — use mode=count or locations instead.",
+          ),
+        ignore_case: z.boolean().optional().describe("Case-insensitive match (default false)."),
+      },
+    },
+    async ({ literal, repo, mode, limit, ignore_case: ignoreCase }) => {
+      try {
+        const index = await getIndexForRepo(repo, cache);
+        const locateOpts = {
+          mode: (mode as LiteralMode | undefined) ?? "lines",
+          ...(limit != null ? { limit } : {}),
+          ignore_case: ignoreCase,
+        };
+
+        if (benchmark) {
+          const repoRoot = localRepoRoot(repo);
+          if (repoRoot) {
+            const comparison = await benchmarkLocateComparison({
+              literal,
+              repoPath: repoRoot,
+              index,
+              locate: locateOpts,
+            });
+            try {
+              await appendBenchmarkQuery(recordFromAgentSummary(repoRoot, comparison.benchmark));
+            } catch {
+              // History is best-effort; never fail locate on persist errors.
+            }
+            return toolText(
+              JSON.stringify(attachLocateBenchmark(comparison.payload, comparison.benchmark)),
+            );
+          }
+        }
+
+        const result = index.locateLiteral(literal, locateOpts);
+        const payload = formatLiteralLocate(result);
+        if (benchmark && !localRepoRoot(repo)) {
+          return toolText(JSON.stringify(withBenchmarkSkipped(payload)));
+        }
+        return toolText(JSON.stringify(payload));
       } catch (err) {
         return toolText(err instanceof Error ? err.message : String(err));
       }
@@ -97,21 +229,21 @@ export function createMcpServer(cache: IndexCache): MiruMcpServer {
           .int()
           .min(0)
           .optional()
-          .describe("Extra chunks before the anchor (default 1)."),
+          .describe(`Extra chunks before the anchor (default ${DEFAULT_EXPAND_BEFORE}).`),
         after: z
           .number()
           .int()
           .min(0)
           .optional()
-          .describe("Extra chunks after the anchor (default 1)."),
+          .describe(`Extra chunks after the anchor (default ${DEFAULT_EXPAND_AFTER}).`),
       },
     },
     async ({ file_path: filePath, anchor_line: anchorLine, repo, before, after }) => {
       try {
         const index = await getIndexForRepo(repo, cache);
         const repoRoot = localRepoRoot(repo);
-        const beforeCount = before ?? 1;
-        const afterCount = after ?? 1;
+        const beforeCount = before ?? DEFAULT_EXPAND_BEFORE;
+        const afterCount = after ?? DEFAULT_EXPAND_AFTER;
         const { anchor, chunks: expanded } = expandChunksAtLine(
           index.chunks,
           filePath,
@@ -194,6 +326,31 @@ export function createMcpServer(cache: IndexCache): MiruMcpServer {
     },
   );
 
+  if (benchmark) {
+    server.registerTool(
+      "read_benchmark",
+      {
+        description: MCP_READ_BENCHMARK_TOOL_DESCRIPTION,
+        inputSchema: {
+          repo: z
+            .string()
+            .optional()
+            .describe(
+              "Optional local repo path or git URL to filter the rollup. Omit for overall totals.",
+            ),
+        },
+      },
+      async ({ repo }) => {
+        try {
+          const rollup = await readBenchmarkRollup({ repo });
+          return toolText(JSON.stringify(rollup));
+        } catch (err) {
+          return toolText(err instanceof Error ? err.message : String(err));
+        }
+      },
+    );
+  }
+
   return server;
 }
 

package/src/miru-index.ts

@@ -14,9 +14,10 @@ import { createIndexFromPath } from "./index/create.ts";
 import { applyIncrementalFileChanges } from "./index/incremental.ts";
 import { persistencePaths, saveIndexBundle } from "./index/persistence.ts";
 import type { SemanticIndex } from "./index/semantic-index.ts";
+import { type LiteralLocateOptions, type LiteralLocateResult, locateLiteral } from "./literal.ts";
 import { hybridSearch, searchSemanticOnly } from "./search.ts";
 import type { Chunk, ContentType, SearchResult } from "./types.ts";
-import { chunkKey, chunkToDict } from "./types.ts";
+import { chunkKey, chunkToDict, defaultContentTypes } from "./types.ts";
 import { computeSourceCacheKey, isGitUrl } from "./utils.ts";
 import { indexCacheEpoch } from "./version.ts";
 
@@ -120,7 +121,7 @@ export class MiruIndex {
     this.chunksInternal = options.chunks;
     this.embeddingModel = options.embeddingModel;
     this._root = options.root ?? null;
-    this.content = options.content ?? ["code"];
+    this.content = options.content ?? defaultContentTypes();
     this.loadedFromDiskFlag = options.loadedFromDisk ?? false;
     this.storedFileMtimes = new Map(Object.entries(options.storedFileMtimes ?? {}));
     this.fileMapping = new Map();
@@ -162,7 +163,7 @@ export class MiruIndex {
    */
   static async fromGit(
     url: string,
-    content: ContentType[] = ["code"],
+    content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
     ref?: string | null,
   ): Promise<MiruIndex> {
@@ -202,7 +203,7 @@ export class MiruIndex {
   /** Resolve a local path or git URL and delegate to `fromPath` or `fromGit`. */
   static async fromSource(
     source: string,
-    content: ContentType[] = ["code"],
+    content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
     ref?: string | null,
   ): Promise<MiruIndex> {
@@ -221,7 +222,7 @@ export class MiruIndex {
    */
   static async fromPath(
     path: string,
-    content: ContentType[] = ["code"],
+    content: ContentType[] = defaultContentTypes(),
     embeddingModel?: string,
   ): Promise<MiruIndex> {
     const resolved = resolve(path);
@@ -271,7 +272,7 @@ export class MiruIndex {
   static async loadFromDisk(path: string, embeddingModel?: string): Promise<MiruIndex> {
     const model = embeddingModel ?? resolveEmbeddingModel();
     const { bm25, semantic, chunks, metadata } = await loadCachedIndex(path);
-    const content = (metadata.content_type as ContentType[]) ?? ["code"];
+    const content = (metadata.content_type as ContentType[]) ?? defaultContentTypes();
     const root = metadata.root_path ? String(metadata.root_path) : null;
     const storedFileMtimes = (metadata.file_mtimes as Record<string, number>) ?? {};
 
@@ -444,4 +445,12 @@ export class MiruIndex {
 
     return results.filter((r) => chunkKey(r.chunk) !== chunkKey(target)).slice(0, topK);
   }
+
+  /**
+   * Exact substring location over indexed chunks (no disk re-walk).
+   * Prefer for known literals; use `search` for meaning-based questions.
+   */
+  locateLiteral(literal: string, options?: LiteralLocateOptions): LiteralLocateResult {
+    return locateLiteral(this.chunksInternal, literal, options);
+  }
 }

package/src/snippet.ts

@@ -149,6 +149,9 @@ export function applySnippetsToResults(
   });
 }
 
-export function estimateResultTokens(results: SearchResult[]): number {
-  return results.reduce((sum, r) => sum + Math.floor(r.chunk.content.length / 4), 0);
-}
+export {
+  countTokens,
+  estimateResultTokens,
+  tokenCountMethod,
+  tokenizerJsonPath,
+} from "./token-count.ts";

package/src/token-count.ts

@@ -0,0 +1,62 @@
+import { existsSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { BertWordPieceTokenizer } from "./tokenizer/bert-wordpiece.ts";
+import { loadTokenizerFromFile } from "./tokenizer/load.ts";
+import type { SearchResult } from "./types.ts";
+
+const TOKENIZER_JSON_ENV = "MIRU_TOKENIZER_JSON";
+const PACKAGE_ROOT = resolve(fileURLToPath(new URL("..", import.meta.url)));
+const DEFAULT_TOKENIZER_JSON = join(PACKAGE_ROOT, "tokenizer", "tokenizer.json");
+
+let cached: BertWordPieceTokenizer | undefined;
+let cachedPath: string | null = null;
+
+function resolveTokenizerJsonPath(): string {
+  const fromEnv = process.env[TOKENIZER_JSON_ENV]?.trim();
+  if (fromEnv) {
+    return resolve(fromEnv);
+  }
+  return DEFAULT_TOKENIZER_JSON;
+}
+
+function loadTokenizer(): BertWordPieceTokenizer {
+  if (cached) {
+    return cached;
+  }
+
+  const path = resolveTokenizerJsonPath();
+  if (!existsSync(path)) {
+    throw new Error(
+      `Tokenizer not found at ${path}. Set ${TOKENIZER_JSON_ENV} or add tokenizer/tokenizer.json to the package.`,
+    );
+  }
+
+  cached = loadTokenizerFromFile(path);
+  cachedPath = path;
+  return cached;
+}
+
+/** Reset cached tokenizer (for tests). */
+export function resetTokenizerCache(): void {
+  cached = undefined;
+  cachedPath = null;
+}
+
+export function tokenizerJsonPath(): string {
+  loadTokenizer();
+  return cachedPath ?? resolveTokenizerJsonPath();
+}
+
+/** @deprecated Use tokenCountMethod() — kept for benchmark response compatibility. */
+export function tokenCountMethod(): "wordpiece" {
+  return "wordpiece";
+}
+
+export function countTokens(text: string): number {
+  return loadTokenizer().count(text);
+}
+
+export function estimateResultTokens(results: SearchResult[]): number {
+  return results.reduce((sum, result) => sum + countTokens(result.chunk.content), 0);
+}

package/src/tokenizer/bert-wordpiece.ts

@@ -0,0 +1,219 @@
+export interface TokenizerJson {
+  normalizer?: {
+    type?: string;
+    clean_text?: boolean;
+    handle_chinese_chars?: boolean;
+    strip_accents?: boolean | null;
+    lowercase?: boolean;
+  };
+  pre_tokenizer?: { type?: string };
+  model: {
+    type?: string;
+    vocab: Record<string, number>;
+    unk_token?: string;
+    continuing_subword_prefix?: string;
+    max_input_chars_per_word?: number;
+  };
+}
+
+export interface BertWordPieceTokenizer {
+  encode(text: string): string[];
+  count(text: string): number;
+}
+
+function isControl(char: string): boolean {
+  const cp = char.codePointAt(0) ?? 0;
+  // TAB/LF/CR are whitespace in Bert clean_text, not stripped controls.
+  if (cp === 0x09 || cp === 0x0a || cp === 0x0d) {
+    return false;
+  }
+  return cp === 0 || cp === 0xfffd || (cp >= 0x00 && cp <= 0x1f) || (cp >= 0x7f && cp <= 0x9f);
+}
+
+function isWhitespace(char: string): boolean {
+  const cp = char.codePointAt(0) ?? 0;
+  return cp === 0x20 || cp === 0x09 || cp === 0x0a || cp === 0x0d;
+}
+
+function isPunctuation(char: string): boolean {
+  const cp = char.codePointAt(0) ?? 0;
+  if (
+    (cp >= 33 && cp <= 47) ||
+    (cp >= 58 && cp <= 64) ||
+    (cp >= 91 && cp <= 96) ||
+    (cp >= 123 && cp <= 126)
+  ) {
+    return true;
+  }
+  return /\p{P}/u.test(char);
+}
+
+function isChineseChar(cp: number): boolean {
+  return (
+    (cp >= 0x4e00 && cp <= 0x9fff) ||
+    (cp >= 0x3400 && cp <= 0x4dbf) ||
+    (cp >= 0x20000 && cp <= 0x2a6df) ||
+    (cp >= 0x2a700 && cp <= 0x2b73f) ||
+    (cp >= 0x2b740 && cp <= 0x2b81f) ||
+    (cp >= 0x2b820 && cp <= 0x2ceaf) ||
+    (cp >= 0xf900 && cp <= 0xfaff) ||
+    (cp >= 0x2f800 && cp <= 0x2fa1f)
+  );
+}
+
+function cleanText(text: string): string {
+  const output: string[] = [];
+  for (const char of text) {
+    if (isControl(char)) {
+      continue;
+    }
+    output.push(isWhitespace(char) ? " " : char);
+  }
+  return output.join("");
+}
+
+function tokenizeChineseChars(text: string): string {
+  const output: string[] = [];
+  for (const char of text) {
+    const cp = char.codePointAt(0) ?? 0;
+    if (isChineseChar(cp)) {
+      output.push(" ", char, " ");
+    } else {
+      output.push(char);
+    }
+  }
+  return output.join("");
+}
+
+function removeAccents(text: string): string {
+  return text.normalize("NFD").replace(/\p{M}/gu, "");
+}
+
+function whitespaceTokenize(text: string): string[] {
+  const trimmed = text.trim();
+  if (!trimmed) {
+    return [];
+  }
+  return trimmed.split(/\s+/);
+}
+
+function splitOnPunctuation(text: string): string[] {
+  const chars = [...text];
+  if (chars.length === 0) {
+    return [];
+  }
+
+  const output: string[][] = [];
+  let startNewWord = true;
+
+  for (const char of chars) {
+    if (isPunctuation(char)) {
+      output.push([char]);
+      startNewWord = true;
+      continue;
+    }
+    if (startNewWord) {
+      output.push([]);
+      startNewWord = false;
+    }
+    output[output.length - 1]?.push(char);
+  }
+
+  return output.map((parts) => parts.join("")).filter(Boolean);
+}
+
+function bertPreTokenize(text: string): string[] {
+  const words = whitespaceTokenize(text);
+  const splitTokens: string[] = [];
+  for (const word of words) {
+    splitTokens.push(...splitOnPunctuation(word));
+  }
+  return whitespaceTokenize(splitTokens.join(" "));
+}
+
+function wordpieceTokenize(
+  token: string,
+  vocab: Set<string>,
+  unkToken: string,
+  prefix: string,
+  maxInputCharsPerWord: number,
+): string[] {
+  if (token.length === 0) {
+    return [];
+  }
+  if (token.length > maxInputCharsPerWord) {
+    return [unkToken];
+  }
+
+  const chars = [...token];
+  const subTokens: string[] = [];
+  let start = 0;
+
+  while (start < chars.length) {
+    let end = chars.length;
+    let curSubstr: string | null = null;
+
+    while (start < end) {
+      let substr = chars.slice(start, end).join("");
+      if (start > 0) {
+        substr = `${prefix}${substr}`;
+      }
+      if (vocab.has(substr)) {
+        curSubstr = substr;
+        break;
+      }
+      end -= 1;
+    }
+
+    if (!curSubstr) {
+      return [unkToken];
+    }
+
+    subTokens.push(curSubstr);
+    start = end;
+  }
+
+  return subTokens;
+}
+
+export function createBertWordPieceTokenizer(json: TokenizerJson): BertWordPieceTokenizer {
+  const normalizer = json.normalizer ?? {};
+  const model = json.model;
+  const lowercase = normalizer.lowercase ?? true;
+  const stripAccents = normalizer.strip_accents ?? lowercase;
+  const vocab = new Set(Object.keys(model.vocab));
+  const unkToken = model.unk_token ?? "[UNK]";
+  const prefix = model.continuing_subword_prefix ?? "##";
+  const maxInputCharsPerWord = model.max_input_chars_per_word ?? 100;
+  const clean = normalizer.clean_text ?? true;
+  const handleChineseChars = normalizer.handle_chinese_chars ?? true;
+
+  function encode(text: string): string[] {
+    let normalized = text;
+    if (clean) {
+      normalized = cleanText(normalized);
+    }
+    if (handleChineseChars) {
+      normalized = tokenizeChineseChars(normalized);
+    }
+    if (lowercase) {
+      normalized = normalized.toLowerCase();
+      if (stripAccents) {
+        normalized = removeAccents(normalized);
+      }
+    }
+
+    const tokens: string[] = [];
+    for (const basic of bertPreTokenize(normalized)) {
+      tokens.push(...wordpieceTokenize(basic, vocab, unkToken, prefix, maxInputCharsPerWord));
+    }
+    return tokens;
+  }
+
+  return {
+    encode,
+    count(text: string): number {
+      return encode(text).length;
+    },
+  };
+}

package/src/tokenizer/load.ts

@@ -0,0 +1,14 @@
+import { readFileSync } from "node:fs";
+import {
+  type BertWordPieceTokenizer,
+  createBertWordPieceTokenizer,
+  type TokenizerJson,
+} from "./bert-wordpiece.ts";
+
+export function loadTokenizerFromFile(path: string): BertWordPieceTokenizer {
+  const json = JSON.parse(readFileSync(path, "utf-8")) as TokenizerJson;
+  if (json.model?.type !== "WordPiece") {
+    throw new Error(`Unsupported tokenizer model type: ${json.model?.type ?? "unknown"}`);
+  }
+  return createBertWordPieceTokenizer(json);
+}

package/src/types.ts

@@ -1,5 +1,13 @@
 export type ContentType = "code" | "docs" | "config";
 
+/** Default index corpus: source code plus config (json/yaml/toml/…). */
+export const DEFAULT_CONTENT_TYPES: readonly ContentType[] = ["code", "config"];
+
+/** Mutable copy of {@link DEFAULT_CONTENT_TYPES} for API defaults. */
+export function defaultContentTypes(): ContentType[] {
+  return [...DEFAULT_CONTENT_TYPES];
+}
+
 export interface Chunk {
   content: string;
   file_path: string;

package/src/utils.ts

@@ -1,7 +1,7 @@
 import { join, relative, resolve, sep } from "node:path";
 import { applySnippetsToResults, searchSnippetsEnabled } from "./snippet.ts";
 import type { Chunk, ContentType, SearchResult } from "./types.ts";
-import { chunkToDict } from "./types.ts";
+import { chunkToDict, defaultContentTypes } from "./types.ts";
 
 const GIT_URL_SCHEMES = [
   "https://",
@@ -103,6 +103,10 @@ export type ExpandResults = {
 export const DEFAULT_MCP_TOP_K = 3;
 export const MAX_MCP_TOP_K = 10;
 
+/** Extra chunks before/after the expand anchor — MCP `expand` tool defaults. */
+export const DEFAULT_EXPAND_BEFORE = 1;
+export const DEFAULT_EXPAND_AFTER = 1;
+
 /** Clamp MCP top_k to a sane range; omit for the default. */
 export function clampMcpTopK(topK?: number): number {
   const value = topK ?? DEFAULT_MCP_TOP_K;
@@ -246,6 +250,9 @@ export function formatExpandResults(
 }
 
 export function resolveContent(raw: string[]): ContentType[] {
+  if (raw.length === 0) {
+    return defaultContentTypes();
+  }
   if (raw.includes("all")) {
     return ["code", "docs", "config"];
   }