@takara-ai/miru-code 0.1.3 → 0.1.4

package/src/index/incremental.ts

@@ -0,0 +1,130 @@
+import { join, relative, resolve } from "node:path";
+import { chunkSource } from "../chunking/chunking.ts";
+import type { EmbeddingBackend } from "../embeddings/openai.ts";
+import { tokenize } from "../tokens.ts";
+import type { Chunk, ContentType } from "../types.ts";
+import { BM25Index } from "./bm25.ts";
+import { detectLanguage, getExtensions, getFileStatus, readFileText } from "./files.ts";
+import type { SemanticIndex } from "./semantic-index.ts";
+import { enrichForBm25 } from "./sparse.ts";
+import { buildSemanticIndex } from "./vector-storage.ts";
+import { vectorAt } from "./vectors.ts";
+
+export function normalizeRelativePath(path: string): string {
+  return path.replace(/\\/g, "/").replace(/^\.\/+/, "");
+}
+
+async function chunksForFile(root: string, relativePath: string): Promise<Chunk[]> {
+  const rel = normalizeRelativePath(relativePath);
+  const absolute = join(root, rel);
+  try {
+    const status = await getFileStatus(absolute);
+    if (status !== "valid") {
+      return [];
+    }
+    const source = await readFileText(absolute);
+    const language = detectLanguage(absolute);
+    return chunkSource(source, rel, language);
+  } catch {
+    return [];
+  }
+}
+
+function isIndexableRelativePath(relativePath: string, extensions: Set<string>): boolean {
+  const rel = normalizeRelativePath(relativePath);
+  const name = rel.split("/").pop() ?? rel;
+  if (name.toLowerCase() === "dockerfile") {
+    return true;
+  }
+  const dot = name.lastIndexOf(".");
+  const ext = dot >= 0 ? name.slice(dot).toLowerCase() : "";
+  return extensions.has(ext);
+}
+
+/**
+ * Replace chunks for the given repo-relative paths only: remove old chunks,
+ * embed new ones, rebuild BM25 + semantic indexes from the merged vector set.
+ */
+export async function applyIncrementalFileChanges(options: {
+  root: string;
+  content: ContentType[];
+  embeddings: EmbeddingBackend;
+  chunks: Chunk[];
+  semanticIndex: SemanticIndex;
+  relativePaths: readonly string[];
+}): Promise<{ chunks: Chunk[]; bm25: BM25Index; semantic: SemanticIndex }> {
+  const root = resolve(options.root);
+  const extensions = new Set(getExtensions(options.content).map((e) => e.toLowerCase()));
+  const targets = new Set(
+    options.relativePaths
+      .map(normalizeRelativePath)
+      .filter((p) => p.length > 0 && isIndexableRelativePath(p, extensions)),
+  );
+
+  if (targets.size === 0) {
+    return {
+      chunks: options.chunks,
+      bm25: rebuildBm25(options.chunks),
+      semantic: options.semanticIndex,
+    };
+  }
+
+  const keptChunks: Chunk[] = [];
+  const keptVectors: Float32Array[] = [];
+
+  for (let i = 0; i < options.chunks.length; i++) {
+    const chunk = options.chunks[i];
+    if (!chunk) {
+      continue;
+    }
+    const rel = normalizeRelativePath(chunk.file_path);
+    if (targets.has(rel)) {
+      continue;
+    }
+    keptChunks.push(chunk);
+    keptVectors.push(vectorAt(options.semanticIndex, i));
+  }
+
+  const addedChunks: Chunk[] = [];
+  for (const rel of targets) {
+    addedChunks.push(...(await chunksForFile(root, rel)));
+  }
+
+  let addedVectors: Float32Array[] = [];
+  if (addedChunks.length > 0) {
+    addedVectors = await options.embeddings.embedDocuments(addedChunks.map((c) => c.content));
+  }
+
+  const chunks = [...keptChunks, ...addedChunks];
+  const vectors = [...keptVectors, ...addedVectors];
+
+  if (chunks.length === 0) {
+    throw new Error(`No indexed chunks remain under ${root}.`);
+  }
+
+  if (vectors.length !== chunks.length) {
+    throw new Error(`Vector count ${vectors.length} does not match chunk count ${chunks.length}.`);
+  }
+
+  return {
+    chunks,
+    bm25: rebuildBm25(chunks),
+    semantic: buildSemanticIndex(vectors),
+  };
+}
+
+function rebuildBm25(chunks: Chunk[]): BM25Index {
+  const bm25 = new BM25Index();
+  bm25.index(chunks.map((c) => tokenize(enrichForBm25(c))));
+  return bm25;
+}
+
+/** Map an absolute changed path to a repo-relative path for chunk keys. */
+export function relativePathFromRoot(root: string, absoluteOrRelative: string): string {
+  const resolvedRoot = resolve(root);
+  const candidate = resolve(resolvedRoot, absoluteOrRelative);
+  if (candidate.startsWith(`${resolvedRoot}/`) || candidate === resolvedRoot) {
+    return normalizeRelativePath(relative(resolvedRoot, candidate));
+  }
+  return normalizeRelativePath(absoluteOrRelative);
+}

package/src/index/quantize.ts

@@ -86,6 +86,30 @@ export class QuantizedVectorIndex implements SemanticIndex {
     return this.codes.length * this.dim + this.scales.byteLength;
   }
 
+  vectorAt(docIndex: number): Float32Array {
+    const codes = this.codes[docIndex];
+    const scale = this.scales[docIndex];
+    if (!codes || scale === undefined) {
+      throw new Error(`Missing quantized vector at index ${docIndex}`);
+    }
+    const out = new Float32Array(codes.length);
+    for (let i = 0; i < codes.length; i++) {
+      out[i] = (codes[i] ?? 0) * scale;
+    }
+    let norm = 0;
+    for (let i = 0; i < out.length; i++) {
+      const v = out[i] ?? 0;
+      norm += v * v;
+    }
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+      for (let i = 0; i < out.length; i++) {
+        out[i] = (out[i] ?? 0) / norm;
+      }
+    }
+    return out;
+  }
+
   query(
     queryVector: Float32Array,
     k: number,

package/src/index/vectors.ts

@@ -0,0 +1,18 @@
+import { VectorIndex } from "./dense.ts";
+import { QuantizedVectorIndex } from "./quantize.ts";
+import type { SemanticIndex } from "./semantic-index.ts";
+
+/** Recover a float embedding for incremental re-indexing (unchanged chunks). */
+export function vectorAt(index: SemanticIndex, docIndex: number): Float32Array {
+  if (index instanceof VectorIndex) {
+    const vec = index.getVectors()[docIndex];
+    if (!vec) {
+      throw new Error(`Missing vector at index ${docIndex}`);
+    }
+    return vec;
+  }
+  if (index instanceof QuantizedVectorIndex) {
+    return index.vectorAt(docIndex);
+  }
+  throw new Error("Unsupported semantic index type");
+}

package/src/mcp/index-cache.ts

@@ -1,14 +1,13 @@
 import { watch } from "node:fs";
 import { resolve } from "node:path";
+import { normalizeRelativePath, relativePathFromRoot } from "../index/incremental.ts";
 import { MiruIndex } from "../miru-index.ts";
-import { envOptionalInt } from "../env.ts";
 import type { ContentType } from "../types.ts";
 import { computeSourceCacheKey, isGitUrl } from "../utils.ts";
 
 const CACHE_MAX_SIZE = 10;
-const DEFAULT_REINDEX_DEBOUNCE_MS = 3000;
 
-/** Directory names we skip for MCP fs.watch re-index triggers (aligned with file-walker). */
+/** Directory names we skip for MCP fs.watch update triggers (aligned with file-walker). */
 const WATCH_IGNORED_DIR_NAMES = new Set([
   ".git",
   ".hg",
@@ -29,28 +28,20 @@ const WATCH_IGNORED_DIR_NAMES = new Set([
   ".eggs",
 ]);
 
-class StaleIndexError extends Error {
-  constructor() {
-    super("Index build superseded by a newer change");
-    this.name = "StaleIndexError";
-  }
-}
-
 type CacheEntry = {
-  generation: number;
+  index: MiruIndex | null;
   task: Promise<MiruIndex> | null;
+  pendingPaths: Set<string>;
+  flushQueued: boolean;
+  updateChain: Promise<void>;
 };
 
-function resolveReindexDebounceMs(): number {
-  return envOptionalInt(["MIRU_MCP_REINDEX_DEBOUNCE_MS"], 0) ?? DEFAULT_REINDEX_DEBOUNCE_MS;
-}
-
 export function mcpWatchEnabled(): boolean {
   const raw = process.env.MIRU_MCP_WATCH ?? process.env.SEMBLE_MCP_WATCH;
   return raw !== "0" && raw !== "false";
 }
 
-/** Returns true when a watch event path should not trigger a re-index. */
+/** Returns true when a watch event path should not trigger an update. */
 export function shouldIgnoreWatchPath(relativePath: string | null | undefined): boolean {
   if (!relativePath) {
     return false;
@@ -67,25 +58,12 @@ export function shouldIgnoreWatchPath(relativePath: string | null | undefined):
 export class IndexCache {
   private readonly content: ContentType[];
   private readonly entries = new Map<string, CacheEntry>();
-  private readonly reindexTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private watcher: ReturnType<typeof watch> | null = null;
 
   constructor(content: ContentType[] = ["code"]) {
     this.content = content;
   }
 
-  evict(source: string, ref?: string | null): void {
-    this.invalidate(computeSourceCacheKey(source, ref));
-  }
-
-  private invalidate(cacheKey: string): void {
-    const entry = this.entries.get(cacheKey);
-    if (entry) {
-      entry.generation++;
-      entry.task = null;
-    }
-  }
-
   private ensureEntry(cacheKey: string): CacheEntry {
     let entry = this.entries.get(cacheKey);
     if (!entry) {
@@ -95,87 +73,168 @@ export class IndexCache {
           this.clearEntry(oldest);
         }
       }
-      entry = { generation: 0, task: null };
+      entry = {
+        index: null,
+        task: null,
+        pendingPaths: new Set(),
+        flushQueued: false,
+        updateChain: Promise.resolve(),
+      };
       this.entries.set(cacheKey, entry);
     }
     return entry;
   }
 
   private clearEntry(cacheKey: string): void {
-    const timer = this.reindexTimers.get(cacheKey);
-    if (timer) {
-      clearTimeout(timer);
-      this.reindexTimers.delete(cacheKey);
-    }
     this.entries.delete(cacheKey);
   }
 
   private startBuild(
     source: string,
     ref: string | null | undefined,
-    entry: CacheEntry,
-    generationAtStart: number,
+    cacheKey: string,
   ): Promise<MiruIndex> {
-    return (async () => {
+    const task = (async () => {
       const index = await MiruIndex.fromSource(source, this.content, undefined, ref);
-      if (entry.generation !== generationAtStart) {
-        throw new StaleIndexError();
-      }
       if (!isGitUrl(source)) {
         await index.saveToDefaultCache(resolve(source));
       }
       return index;
     })();
+
+    const entry = this.ensureEntry(cacheKey);
+    entry.task = task;
+    void task
+      .then((index) => {
+        entry.index = index;
+        void this.flushFileUpdates(cacheKey, source);
+        return index;
+      })
+      .catch(() => {
+        if (entry.task === task) {
+          entry.task = null;
+        }
+      });
+
+    return task;
   }
 
   async get(source: string, ref?: string | null): Promise<MiruIndex> {
     const cacheKey = computeSourceCacheKey(source, ref);
+    const entry = this.ensureEntry(cacheKey);
+
+    if (!entry.task) {
+      this.startBuild(source, ref, cacheKey);
+    }
+
+    const index = await entry.task;
+    if (!index) {
+      throw new Error(`Failed to load index for ${source}`);
+    }
+    await entry.updateChain;
+    return index;
+  }
+
+  private scheduleFlush(cacheKey: string, source: string, entry: CacheEntry): void {
+    if (!entry.flushQueued) {
+      entry.flushQueued = true;
+      queueMicrotask(() => {
+        this.flushFileUpdates(cacheKey, source);
+      });
+    }
+  }
+
+  private queueIndexedPaths(source: string, index: MiruIndex): void {
+    const cacheKey = computeSourceCacheKey(source);
+    const entry = this.ensureEntry(cacheKey);
+    for (const chunk of index.chunks) {
+      entry.pendingPaths.add(normalizeRelativePath(chunk.file_path));
+    }
+    this.scheduleFlush(cacheKey, source, entry);
+  }
+
+  /** macOS recursive fs.watch often omits filename; refresh all indexed paths incrementally. */
+  private noteAmbiguousDirectoryChange(source: string): void {
+    const cacheKey = computeSourceCacheKey(source);
+    const entry = this.ensureEntry(cacheKey);
+
+    if (entry.index) {
+      this.queueIndexedPaths(source, entry.index);
+      return;
+    }
+
+    if (entry.task) {
+      void entry.task.then((index) => {
+        entry.index = index;
+        this.queueIndexedPaths(source, index);
+      });
+    }
+  }
+
+  private noteFileChange(source: string, filename: string | null | undefined): void {
+    if (!filename) {
+      this.noteAmbiguousDirectoryChange(source);
+      return;
+    }
+    if (shouldIgnoreWatchPath(filename)) {
+      return;
+    }
+
+    const cacheKey = computeSourceCacheKey(source);
+    const entry = this.ensureEntry(cacheKey);
+    const rel = relativePathFromRoot(source, filename);
+    if (!rel) {
+      return;
+    }
+    entry.pendingPaths.add(rel);
+    this.scheduleFlush(cacheKey, source, entry);
+  }
 
-    for (;;) {
-      const entry = this.ensureEntry(cacheKey);
-      const generationAtStart = entry.generation;
+  private flushFileUpdates(cacheKey: string, source: string): void {
+    const entry = this.entries.get(cacheKey);
+    if (!entry) {
+      return;
+    }
+    entry.flushQueued = false;
 
-      if (!entry.task) {
-        entry.task = this.startBuild(source, ref, entry, generationAtStart);
+    const run = async (): Promise<void> => {
+      const paths = [...entry.pendingPaths];
+      entry.pendingPaths.clear();
+      if (paths.length === 0) {
+        return;
       }
 
-      const task = entry.task;
-      try {
-        const index = await task;
-        if (entry.generation !== generationAtStart) {
-          continue;
+      let index = entry.index;
+      if (!index && entry.task) {
+        try {
+          index = await entry.task;
+        } catch {
+          for (const p of paths) {
+            entry.pendingPaths.add(p);
+          }
+          return;
         }
-        return index;
-      } catch (err) {
-        if (entry.task === task) {
-          entry.task = null;
+      }
+      if (!index) {
+        for (const p of paths) {
+          entry.pendingPaths.add(p);
         }
-        if (err instanceof StaleIndexError) {
-          continue;
+        return;
+      }
+
+      try {
+        await index.applyFileChanges(paths);
+        if (!isGitUrl(source)) {
+          await index.persistToCache(resolve(source));
         }
-        if (this.entries.get(cacheKey) === entry) {
-          this.entries.delete(cacheKey);
+      } catch {
+        for (const p of paths) {
+          entry.pendingPaths.add(p);
         }
-        throw err;
       }
-    }
-  }
-
-  private scheduleReindex(source: string, ref?: string | null): void {
-    const cacheKey = computeSourceCacheKey(source, ref);
-    const existing = this.reindexTimers.get(cacheKey);
-    if (existing) {
-      clearTimeout(existing);
-    }
+    };
 
-    const debounceMs = resolveReindexDebounceMs();
-    const timer = setTimeout(() => {
-      this.reindexTimers.delete(cacheKey);
-      this.invalidate(cacheKey);
-      void this.get(source, ref).catch(() => undefined);
-    }, debounceMs);
-
-    this.reindexTimers.set(cacheKey, timer);
+    entry.updateChain = entry.updateChain.then(run, run);
   }
 
   startWatcher(path: string): void {
@@ -185,20 +244,14 @@ export class IndexCache {
     }
 
     this.watcher = watch(resolved, { recursive: true }, (_event, filename) => {
-      if (shouldIgnoreWatchPath(filename)) {
-        return;
-      }
-      this.scheduleReindex(resolved);
+      this.noteFileChange(resolved, filename);
     });
   }
 
   close(): void {
-    for (const timer of this.reindexTimers.values()) {
-      clearTimeout(timer);
-    }
-    this.reindexTimers.clear();
     this.watcher?.close();
     this.watcher = null;
+    this.entries.clear();
   }
 }
 

package/src/miru-index.ts

@@ -10,6 +10,7 @@ import {
 import { cloneGitRepository } from "./git.ts";
 import type { BM25Index } from "./index/bm25.ts";
 import { createIndexFromPath } from "./index/create.ts";
+import { applyIncrementalFileChanges } from "./index/incremental.ts";
 import { persistencePaths, saveIndexBundle } from "./index/persistence.ts";
 import type { SemanticIndex } from "./index/semantic-index.ts";
 import { hybridSearch, searchSemanticOnly } from "./search.ts";
@@ -19,16 +20,24 @@ import { computeSourceCacheKey, isGitUrl } from "./utils.ts";
 
 export class MiruIndex {
   readonly embeddings: EmbeddingBackend;
-  readonly chunks: Chunk[];
-  readonly loadedFromDisk: boolean;
+  private chunksInternal: Chunk[];
+  private bm25Index: BM25Index;
+  private semanticIndex: SemanticIndex;
+  private loadedFromDiskFlag: boolean;
 
-  private readonly bm25Index: BM25Index;
-  private readonly semanticIndex: SemanticIndex;
-  private readonly embeddingModel: string;
+  readonly embeddingModel: string;
   private readonly root: string | null;
   private readonly content: ContentType[];
-  private readonly fileMapping: Map<string, number[]>;
-  private readonly languageMapping: Map<string, number[]>;
+  private fileMapping: Map<string, number[]>;
+  private languageMapping: Map<string, number[]>;
+
+  get chunks(): Chunk[] {
+    return this.chunksInternal;
+  }
+
+  get loadedFromDisk(): boolean {
+    return this.loadedFromDiskFlag;
+  }
 
   constructor(options: {
     embeddings: EmbeddingBackend;
@@ -43,16 +52,21 @@ export class MiruIndex {
     this.embeddings = options.embeddings;
     this.bm25Index = options.bm25Index;
     this.semanticIndex = options.semanticIndex;
-    this.chunks = options.chunks;
+    this.chunksInternal = options.chunks;
     this.embeddingModel = options.embeddingModel;
     this.root = options.root ?? null;
     this.content = options.content ?? ["code"];
-    this.loadedFromDisk = options.loadedFromDisk ?? false;
+    this.loadedFromDiskFlag = options.loadedFromDisk ?? false;
+    this.fileMapping = new Map();
+    this.languageMapping = new Map();
+    this.rebuildMappings();
+  }
 
+  private rebuildMappings(): void {
     this.fileMapping = new Map();
     this.languageMapping = new Map();
-    for (let i = 0; i < this.chunks.length; i++) {
-      const chunk = this.chunks[i];
+    for (let i = 0; i < this.chunksInternal.length; i++) {
+      const chunk = this.chunksInternal[i];
       if (!chunk) {
         continue;
       }
@@ -203,11 +217,40 @@ export class MiruIndex {
   }
 
   async saveToDefaultCache(sourcePath: string): Promise<void> {
-    if (!this.loadedFromDisk) {
+    if (!this.loadedFromDiskFlag) {
       await this.save(findIndexCachePath(sourcePath));
     }
   }
 
+  async persistToCache(sourcePath: string): Promise<void> {
+    await this.save(findIndexCachePath(sourcePath));
+  }
+
+  /** Re-chunk and re-embed only the given repo-relative paths; drop their old chunks. */
+  async applyFileChanges(relativePaths: readonly string[]): Promise<void> {
+    if (!this.root) {
+      throw new Error("Incremental update requires a local index with root_path set.");
+    }
+    if (relativePaths.length === 0) {
+      return;
+    }
+
+    const updated = await applyIncrementalFileChanges({
+      root: this.root,
+      content: this.content,
+      embeddings: this.embeddings,
+      chunks: this.chunksInternal,
+      semanticIndex: this.semanticIndex,
+      relativePaths,
+    });
+
+    this.chunksInternal = updated.chunks;
+    this.bm25Index = updated.bm25;
+    this.semanticIndex = updated.semantic;
+    this.loadedFromDiskFlag = false;
+    this.rebuildMappings();
+  }
+
   private getSelector(filterLanguages?: string[], filterPaths?: string[]): number[] | undefined {
     const selector: number[] = [];
     for (const lang of filterLanguages ?? []) {