@takara-ai/miru-code 1.7.7 → 1.8.0

package/src/mcp/index-cache.ts

@@ -13,6 +13,7 @@ import {
 } from "../utils.ts";
 
 const CACHE_MAX_SIZE = 10;
+const BUN_WATCH_RECONCILE_INTERVAL_MS = 500;
 
 /** Directory names we skip for MCP fs.watch update triggers (aligned with file-walker). */
 const WATCH_IGNORED_DIR_NAMES = new Set([
@@ -35,7 +36,7 @@ const WATCH_IGNORED_DIR_NAMES = new Set([
   ".eggs",
 ]);
 
-type WatcherHandle = ReturnType<typeof watch>;
+type WatcherHandle = { close(): void };
 
 type CacheEntry = {
   source: string;
@@ -274,13 +275,19 @@ export class IndexCache {
 
       try {
         await index.applyFileChanges(paths);
-        if (!isGitUrl(source)) {
-          await index.saveToCache(resolve(source), { force: true });
-        }
       } catch {
         for (const p of paths) {
           entry.pendingPaths.add(p);
         }
+        return;
+      }
+      if (!isGitUrl(source)) {
+        try {
+          await index.saveToCache(resolve(source), { force: true });
+        } catch {
+          // The in-memory index is already current. A cache write failure must
+          // not requeue the change and repeatedly re-embed the same file.
+        }
       }
     };
 
@@ -351,10 +358,48 @@ export class IndexCache {
       return;
     }
 
-    const watcher = watch(resolved, { recursive: true }, (_event, filename) => {
-      this.noteFileChange(resolved, filename);
+    let nativeWatcher: ReturnType<typeof watch> | null = null;
+    try {
+      nativeWatcher = watch(resolved, { recursive: true }, (_event, filename) => {
+        this.noteFileChange(path, filename);
+      });
+    } catch {
+      // Recursive fs.watch is unavailable on some platforms. The reconciliation
+      // fallback below keeps local indexes current there.
+    }
+
+    // Bun 1.3 on macOS can create a recursive watcher without delivering file
+    // events. Reconcile periodically in Bun so a silent watcher cannot leave an
+    // MCP index stale. Native watchers remain the fast path everywhere else.
+    let reconcileInFlight = false;
+    const reconcile = (): void => {
+      if (reconcileInFlight) {
+        return;
+      }
+      const cacheKey = computeSourceCacheKey(path);
+      const entry = this.entries.get(cacheKey);
+      if (!entry?.index) {
+        return;
+      }
+      reconcileInFlight = true;
+      void this.checkAndQueueStaleFiles(path, entry.index, cacheKey).finally(() => {
+        reconcileInFlight = false;
+      });
+    };
+    const interval =
+      process.versions.bun || !nativeWatcher
+        ? setInterval(reconcile, BUN_WATCH_RECONCILE_INTERVAL_MS)
+        : null;
+    interval?.unref();
+
+    this.watchers.set(resolved, {
+      close: () => {
+        nativeWatcher?.close();
+        if (interval) {
+          clearInterval(interval);
+        }
+      },
     });
-    this.watchers.set(resolved, watcher);
   }
 
   get watcher(): WatcherHandle | null {

package/src/mcp/server.ts

@@ -1,3 +1,4 @@
+import { join } from "node:path";
 import * as z from "zod";
 import packageJson from "../../package.json";
 import { benchmarkSearchComparison, toAgentBenchmarkSummary } from "../benchmark/compare.ts";
@@ -9,6 +10,7 @@ import {
   recordFromBenchmark,
 } from "../benchmark/history.ts";
 import { benchmarkLocateComparison } from "../benchmark/locate-compare.ts";
+import { selectComparableLiteralSearchTool } from "../benchmark/rg-literal.ts";
 import { appendAgentBenchmark } from "../benchmark/summary.ts";
 import {
   MCP_BENCHMARK_SERVER_INSTRUCTIONS,
@@ -20,7 +22,7 @@ import {
   MCP_SERVER_INSTRUCTIONS,
 } from "../installer/search-policy.ts";
 import { formatLiteralLocate } from "../literal.ts";
-import type { ContentType } from "../types.ts";
+import type { Chunk, ContentType, SearchResult } from "../types.ts";
 import {
   clampMcpTopK,
   DEFAULT_EXPAND_AFTER,
@@ -50,6 +52,10 @@ const REPO_DESCRIPTION =
 
 const BENCHMARK_SKIP_NOTES = {
   local_repo_only: "Benchmark comparisons require a local repo path; git URL repos are skipped.",
+  limited_locate:
+    "Benchmark comparison skipped: locate.limit is global, while rg/grep limits are per file. Omit limit for a valid token and recall comparison.",
+  incompatible_literal_baseline:
+    "Benchmark comparison skipped: a comparable literal baseline requires rg or compatible grep. Windows findstr does not preserve locate scope, context, or count semantics.",
   grep_timeout:
     "Benchmark Grep baseline timed out; Miru results are still returned. " +
     "Raise MIRU_BENCHMARK_SEARCH_TIMEOUT (seconds) if needed.",
@@ -64,6 +70,45 @@ function withBenchmarkSkippedNote(body: string, reason: BenchmarkSkipReason): st
   })}`;
 }
 
+/**
+ * Use the source file for final snippet shaping when it is locally available.
+ * Index chunks can intentionally split a large declaration; source-backed
+ * snippets let the structural guard return a complete declaration instead.
+ */
+async function loadSnippetSources(
+  root: string | null,
+  results: SearchResult[],
+): Promise<Map<string, Chunk> | undefined> {
+  if (!root) {
+    return undefined;
+  }
+  const sources = new Map<string, Chunk>();
+  await Promise.all(
+    [...new Set(results.map((result) => result.chunk.file_path))].map(async (filePath) => {
+      try {
+        const content = await Bun.file(join(root, filePath)).text();
+        const indexed = results.find((result) => result.chunk.file_path === filePath)?.chunk;
+        // Synthetic index entries (for example package entry-point metadata) do
+        // not occur verbatim in the source file and must retain their own chunk.
+        if (!indexed || !content.includes(indexed.content)) {
+          return;
+        }
+        const lines = content.split("\n");
+        sources.set(filePath, {
+          content,
+          file_path: filePath,
+          start_line: 1,
+          end_line: lines.length,
+          language: indexed.language,
+        });
+      } catch {
+        // A stale index entry is still safe to format from its indexed chunk.
+      }
+    }),
+  );
+  return sources;
+}
+
 async function persistBenchmarkQuery(
   ...args: Parameters<typeof appendBenchmarkQuery>
 ): Promise<void> {
@@ -139,7 +184,11 @@ export function createMcpServer(
             }
             await persistBenchmarkQuery(recordFromBenchmark(repoRoot, comparison.benchmark));
             const body = formatResultsText(
-              formatResults(query, results, { repoRoot, snippet: true }),
+              formatResults(query, results, {
+                repoRoot,
+                snippet: true,
+                snippetSourceChunks: await loadSnippetSources(index.root, results),
+              }),
             );
             return toolText(
               appendAgentBenchmark(body, toAgentBenchmarkSummary(comparison.benchmark)),
@@ -157,7 +206,11 @@ export function createMcpServer(
         if (results.length === 0) {
           return toolText("No results found.");
         }
-        const payload = formatResults(query, results, { repoRoot, snippet: true });
+        const payload = formatResults(query, results, {
+          repoRoot,
+          snippet: true,
+          snippetSourceChunks: await loadSnippetSources(index.root, results),
+        });
         const body = formatResultsText(payload);
         return toolText(skip ? withBenchmarkSkippedNote(body, skip) : body);
       } catch (err) {
@@ -224,20 +277,27 @@ export function createMcpServer(
         let skip: BenchmarkSkipReason | undefined;
         const repoRoot = localRepoRoot(repo);
         if (benchmark && repoRoot && typeof literal === "string") {
-          const comparison = await withGrepTimeoutFallback(() =>
-            benchmarkLocateComparison({
-              literal,
-              repoPath: repoRoot,
-              index,
-              locate: locateOpts,
-            }),
-          );
-          if (comparison) {
-            await persistBenchmarkQuery(recordFromAgentSummary(repoRoot, comparison.benchmark));
-            const body = formatLiteralLocateText(comparison.payload);
-            return toolText(appendAgentBenchmark(body, comparison.benchmark));
+          if (locateOpts.limit != null) {
+            skip = "limited_locate";
+          } else if (!selectComparableLiteralSearchTool()) {
+            skip = "incompatible_literal_baseline";
+          }
+          if (!skip) {
+            const comparison = await withGrepTimeoutFallback(() =>
+              benchmarkLocateComparison({
+                literal,
+                repoPath: repoRoot,
+                index,
+                locate: locateOpts,
+              }),
+            );
+            if (comparison) {
+              await persistBenchmarkQuery(recordFromAgentSummary(repoRoot, comparison.benchmark));
+              const body = formatLiteralLocateText(comparison.payload);
+              return toolText(appendAgentBenchmark(body, comparison.benchmark));
+            }
+            skip = "grep_timeout";
           }
-          skip = "grep_timeout";
         } else if (benchmark && !repoRoot) {
           skip = "local_repo_only";
         }
@@ -356,6 +416,7 @@ export function createMcpServer(
             formatResults(`Chunks related to ${filePath}:${anchorLine}`, results, {
               repoRoot,
               snippet: true,
+              snippetSourceChunks: await loadSnippetSources(index.root, results),
             }),
           ),
         );

package/src/miru-index.ts

@@ -396,13 +396,15 @@ export class MiruIndex {
    */
   async search(options: {
     query: string;
+    /** Optional precomputed embedding for high-throughput callers. */
+    queryVector?: Float32Array;
     topK?: number;
     alpha?: number | null;
     filterLanguages?: string[];
     filterPaths?: string[];
     rerank?: boolean;
   }): Promise<SearchResult[]> {
-    const { query, topK = 10, alpha, filterLanguages, filterPaths, rerank } = options;
+    const { query, queryVector, topK = 10, alpha, filterLanguages, filterPaths, rerank } = options;
 
     if (!this.chunks.length || !query.trim()) {
       return [];
@@ -412,6 +414,7 @@ export class MiruIndex {
 
     return hybridSearch({
       query,
+      queryVector,
       embeddings: this.embeddings,
       semanticIndex: this.semanticIndex,
       bm25Index: this.bm25Index,

package/src/ranking/boosting.ts

@@ -2,6 +2,7 @@ import { basename, dirname } from "node:path";
 import { splitIdentifier } from "../tokens.ts";
 import type { Chunk } from "../types.ts";
 import { chunkKey } from "../types.ts";
+import { rankingConfig } from "./config.ts";
 import { searchImprovementsEnabled } from "./features.ts";
 import {
   boostExactStemMatches,
@@ -16,7 +17,6 @@ const EMBEDDED_SYMBOL_RE =
   /\b(?:[A-Z][a-z][a-zA-Z0-9]*[A-Z][a-zA-Z0-9]*|[a-z][a-zA-Z0-9]*[A-Z][a-zA-Z0-9]+)\b/g;
 
 const EMBEDDED_STEM_MIN_LEN = 4;
-const EMBEDDED_SYMBOL_BOOST_SCALE = 0.5;
 
 const DEFINITION_KEYWORDS = [
   "class",
@@ -49,10 +49,6 @@ const SQL_DEFINITION_KEYWORDS = [
   "CREATE FUNCTION",
 ];
 
-const DEFINITION_BOOST_MULTIPLIER = 3.0;
-const STEM_BOOST_MULTIPLIER = 1.0;
-const FILE_COHERENCE_BOOST_FRAC = 0.2;
-
 const STOPWORDS = new Set(
   "a an and are as at be by do does for from has have how if in is it not of on or the to was what when where which who why with".split(
     " ",
@@ -147,7 +143,7 @@ export function boostMultiChunkFiles(
   }
 
   const maxFileSum = Math.max(...fileSum.values());
-  const boostUnit = maxScore * FILE_COHERENCE_BOOST_FRAC;
+  const boostUnit = maxScore * rankingConfig().fileCoherenceBoost;
 
   for (const [fp, key] of bestChunk) {
     const current = scores.get(key) ?? 0;
@@ -192,7 +188,7 @@ function boostSymbolDefinitions(
   if (symbolName !== query.trim()) {
     names.add(query.trim());
   }
-  const boostUnit = maxScore * DEFINITION_BOOST_MULTIPLIER;
+  const boostUnit = maxScore * rankingConfig().definitionBoost;
 
   for (const key of [...boosted.keys()]) {
     const chunk = chunksByKey.get(key);
@@ -235,7 +231,8 @@ function boostEmbeddedSymbols(
     return;
   }
 
-  const boostUnit = maxScore * DEFINITION_BOOST_MULTIPLIER * EMBEDDED_SYMBOL_BOOST_SCALE;
+  const tuning = rankingConfig();
+  const boostUnit = maxScore * tuning.definitionBoost * tuning.embeddedSymbolScale;
   const symbolsLower = new Set([...names].map((s) => s.toLowerCase()));
 
   for (const key of [...boosted.keys()]) {
@@ -309,7 +306,7 @@ function boostStemMatches(
     return;
   }
 
-  const boost = maxScore * STEM_BOOST_MULTIPLIER;
+  const boost = maxScore * rankingConfig().exactStemBoost;
   const pathCache = new Map<string, Set<string>>();
 
   for (const key of [...boosted.keys()]) {

package/src/ranking/config.ts

@@ -0,0 +1,35 @@
+/** Optional ranking overrides, primarily for reproducible offline calibration. */
+export interface RankingConfig {
+  rrfK: number;
+  candidateMultiplier: number;
+  locationCandidateMultiplier: number;
+  fileCoherenceBoost: number;
+  definitionBoost: number;
+  embeddedSymbolScale: number;
+  exactStemBoost: number;
+  locationBoost: number;
+  pathPenalties: boolean;
+  fileSaturationDecay: number;
+}
+
+function numberEnv(name: string, fallback: number, min: number, max: number): number {
+  const value = Number.parseFloat(process.env[name] ?? "");
+  return Number.isFinite(value) && value >= min && value <= max ? value : fallback;
+}
+
+export function rankingConfig(): RankingConfig {
+  return {
+    // Validated on the cached 3k CodeSearchNet corpus. Overrides preserve a
+    // straightforward rollback/A-B path for repository-specific evaluation.
+    rrfK: numberEnv("MIRU_RANK_RRF_K", 10, 1, 500),
+    candidateMultiplier: numberEnv("MIRU_RANK_CANDIDATE_MULTIPLIER", 5, 1, 50),
+    locationCandidateMultiplier: numberEnv("MIRU_RANK_LOCATION_CANDIDATE_MULTIPLIER", 10, 1, 100),
+    fileCoherenceBoost: numberEnv("MIRU_RANK_FILE_COHERENCE_BOOST", 0, 0, 2),
+    definitionBoost: numberEnv("MIRU_RANK_DEFINITION_BOOST", 3, 0, 10),
+    embeddedSymbolScale: numberEnv("MIRU_RANK_EMBEDDED_SYMBOL_SCALE", 0.5, 0, 2),
+    exactStemBoost: numberEnv("MIRU_RANK_EXACT_STEM_BOOST", 0.75, 0, 3),
+    locationBoost: numberEnv("MIRU_RANK_LOCATION_BOOST", 2.5, 0, 10),
+    pathPenalties: process.env.MIRU_RANK_PATH_PENALTIES !== "0",
+    fileSaturationDecay: numberEnv("MIRU_RANK_FILE_SATURATION_DECAY", 0.5, 0.05, 1),
+  };
+}

package/src/ranking/location.ts

@@ -1,6 +1,7 @@
 import { basename } from "node:path";
 import type { Chunk } from "../types.ts";
 import { chunkKey } from "../types.ts";
+import { rankingConfig } from "./config.ts";
 
 const LOCATION_QUERY_RE =
   /\b(where(?:'s| is)?|entry\s*point|bootstrap|starts?|live[s]?|located|defined|wiring)\b/i;
@@ -16,7 +17,6 @@ const ENTRY_POINT_CONTENT_RE =
 
 const PACKAGE_ENTRY_RE = /^\[package entry\]/;
 
-const LOCATION_BOOST_MULTIPLIER = 2.5;
 const INSTALLER_LOCATION_PENALTY = 0.35;
 
 /** Too common as filenames to get exact-stem boosts from incidental query tokens. */
@@ -126,8 +126,9 @@ export function boostLocationSignals(
   }
 
   const entryPointQuery = isEntryPointQuery(query);
-  const packageBoost = maxScore * (entryPointQuery ? LOCATION_BOOST_MULTIPLIER : 0.75);
-  const codeBoost = maxScore * LOCATION_BOOST_MULTIPLIER;
+  const tuning = rankingConfig();
+  const packageBoost = maxScore * (entryPointQuery ? tuning.locationBoost : 0.75);
+  const codeBoost = maxScore * tuning.locationBoost;
 
   for (const chunk of allChunks) {
     if (!chunkLooksLikeEntryPoint(chunk)) {
@@ -205,7 +206,7 @@ export function boostExactStemMatches(
     return;
   }
 
-  const boost = maxScore * 0.75;
+  const boost = maxScore * rankingConfig().exactStemBoost;
   const preferImplementation = hasSpecificImplementationTarget(query, chunksByKey);
 
   for (const key of [...boosted.keys()]) {

package/src/ranking/penalties.ts

@@ -1,5 +1,6 @@
 import { basename } from "node:path";
 import type { Chunk } from "../types.ts";
+import { rankingConfig } from "./config.ts";
 
 const TEST_FILE_RE =
   /(?:^|\/)(?:test_[^/]*\.py|[^/]*_test\.py|[^/]*_test\.go|[^/]*Tests?\.java|[^/]*Test\.php|[^/]*_spec\.rb|[^/]*_test\.rb|[^/]*\.test\.[jt]sx?|[^/]*\.spec\.[jt]sx?|[^/]*Tests?\.kt|[^/]*Spec\.kt|[^/]*Tests?\.swift|[^/]*Spec\.swift|[^/]*Tests?\.cs|test_[^/]*\.cpp|[^/]*_test\.cpp|test_[^/]*\.c|[^/]*_test\.c|[^/]*Spec\.scala|[^/]*Suite\.scala|[^/]*Test\.scala|[^/]*_test\.dart|test_[^/]*\.dart|[^/]*_spec\.lua|[^/]*_test\.lua|test_[^/]*\.lua|test_helpers?[^/]*\.\w+)$/;
@@ -15,7 +16,6 @@ const MILD_PENALTY = 0.7;
 
 const REEXPORT_FILENAMES = new Set(["__init__.py", "package-info.java"]);
 const FILE_SATURATION_THRESHOLD = 1;
-const FILE_SATURATION_DECAY = 0.5;
 
 function filePathPenalty(filePath: string): number {
   const normalised = filePath.replace(/\\/g, "/");
@@ -49,6 +49,7 @@ export function rerankTopk(
   }
 
   const penaltyCache = new Map<string, number>();
+  const tuning = rankingConfig();
   const penalised = new Map<string, number>();
 
   for (const [key, score] of scores) {
@@ -57,7 +58,7 @@ export function rerankTopk(
       continue;
     }
     let pen = score;
-    if (penalisePaths) {
+    if (penalisePaths && tuning.pathPenalties) {
       let mult = penaltyCache.get(chunk.file_path);
       if (mult === undefined) {
         mult = filePathPenalty(chunk.file_path);
@@ -88,7 +89,7 @@ export function rerankTopk(
     let effScore = penScore;
     if (already >= FILE_SATURATION_THRESHOLD) {
       const excess = already - FILE_SATURATION_THRESHOLD + 1;
-      effScore *= FILE_SATURATION_DECAY ** excess;
+      effScore *= tuning.fileSaturationDecay ** excess;
     }
 
     selected.push({ score: effScore, key });

package/src/ranking/weighting.ts

@@ -1,7 +1,10 @@
 import { isSymbolQuery } from "./boosting.ts";
 
 const ALPHA_SYMBOL = 0.3;
-const ALPHA_NL = 0.5;
+// Natural-language code queries benefit from a modest lexical preference:
+// CodeSearchNet factorial sweep (3k full validation) improved exact-match@1,
+// hit@3, and MRR at 0.25 versus the former 0.50.
+const ALPHA_NL = 0.25;
 
 export function resolveAlpha(query: string, alpha: number | null | undefined): number {
   if (alpha != null) {

package/src/search.ts

@@ -4,6 +4,7 @@ import type { SemanticIndex } from "./index/semantic-index.ts";
 import { selectorToMask } from "./index/sparse.ts";
 import { selectTopKScoreIndices } from "./index/topk.ts";
 import { applyQueryBoost, boostMultiChunkFiles } from "./ranking/boosting.ts";
+import { rankingConfig } from "./ranking/config.ts";
 import { searchImprovementsEnabled } from "./ranking/features.ts";
 import { isLocationQuery } from "./ranking/location.ts";
 import { rerankTopk } from "./ranking/penalties.ts";
@@ -12,16 +13,14 @@ import { tokenize } from "./tokens.ts";
 import type { Chunk, SearchResult } from "./types.ts";
 import { chunkKey } from "./types.ts";
 
-const RRF_K = 60;
-
-function rrfScores(scores: Map<string, number>): Map<string, number> {
+function rrfScores(scores: Map<string, number>, rrfK: number): Map<string, number> {
   if (scores.size === 0) {
     return scores;
   }
   const ranked = [...scores.entries()].sort((a, b) => b[1] - a[1]);
   const out = new Map<string, number>();
   ranked.forEach(([key], i) => {
-    out.set(key, 1.0 / (RRF_K + i + 1));
+    out.set(key, 1.0 / (rrfK + i + 1));
   });
   return out;
 }
@@ -73,6 +72,8 @@ async function searchBm25(
 
 export async function hybridSearch(options: {
   query: string;
+  /** Reuse a caller-cached query vector when evaluating or batching searches. */
+  queryVector?: Float32Array;
   embeddings: EmbeddingBackend;
   semanticIndex: SemanticIndex;
   bm25Index: BM25Index;
@@ -84,6 +85,7 @@ export async function hybridSearch(options: {
 }): Promise<SearchResult[]> {
   const {
     query,
+    queryVector,
     embeddings,
     semanticIndex,
     bm25Index,
@@ -95,11 +97,14 @@ export async function hybridSearch(options: {
   } = options;
 
   const alphaWeight = resolveAlpha(query, alpha);
+  const tuning = rankingConfig();
   const candidateCount =
-    searchImprovementsEnabled() && isLocationQuery(query) ? topK * 10 : topK * 5;
+    searchImprovementsEnabled() && isLocationQuery(query)
+      ? topK * tuning.locationCandidateMultiplier
+      : topK * tuning.candidateMultiplier;
   const chunksByKey = new Map(chunks.map((c) => [chunkKey(c), c]));
 
-  const queryVecPromise = embeddings.embedQuery(query);
+  const queryVecPromise = queryVector ? Promise.resolve(queryVector) : embeddings.embedQuery(query);
   const bm25Promise = searchBm25(query, bm25Index, chunks, candidateCount, selector);
   const queryVec = await queryVecPromise;
   const bm25Hits = await bm25Promise;
@@ -123,8 +128,8 @@ export async function hybridSearch(options: {
     }
   }
 
-  const normalizedSemantic = rrfScores(semanticScores);
-  const normalizedBm25 = rrfScores(bm25Scores);
+  const normalizedSemantic = rrfScores(semanticScores, tuning.rrfK);
+  const normalizedBm25 = rrfScores(bm25Scores, tuning.rrfK);
 
   const allKeys = new Set([...normalizedSemantic.keys(), ...normalizedBm25.keys()]);
   const sortedKeys = [...allKeys].sort((a, b) => {

package/src/setup.ts

@@ -192,6 +192,7 @@ export async function runSageMakerSetup(options: RunSetupOptions = {}): Promise<
     const config: SageMakerEmbeddingConfig = {
       endpointName: parsed.endpointName,
       region: parsed.region,
+      profile,
       normalize: true,
       truncate: true,
       truncationDirection: "Right",