@takara-ai/miru-code 0.1.3 → 0.2.0

package/src/embeddings/validate.ts

@@ -0,0 +1,94 @@
+import {
+  resolveEmbeddingBaseUrl,
+  resolveEmbeddingDimensions,
+  resolveEmbeddingModel,
+} from "./openai.ts";
+
+export interface ValidateApiKeyResult {
+  valid: boolean;
+  status?: number;
+  message: string;
+}
+
+export async function validateEmbeddingApiKey(options: {
+  apiKey: string;
+  baseUrl?: string;
+  model?: string;
+  dimensions?: number;
+}): Promise<ValidateApiKeyResult> {
+  const baseUrl = (options.baseUrl ?? resolveEmbeddingBaseUrl()).replace(/\/$/, "");
+  const model = options.model ?? resolveEmbeddingModel();
+  const dimensions = options.dimensions ?? resolveEmbeddingDimensions(model);
+  const endpoint = `${baseUrl}/embeddings`;
+
+  const body: Record<string, unknown> = {
+    model,
+    input: "miru setup validation",
+  };
+  if (dimensions != null) {
+    body.dimensions = dimensions;
+  }
+
+  let response: Response;
+  try {
+    response = await fetch(endpoint, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${options.apiKey}`,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return {
+      valid: false,
+      message: `Could not reach embedding API at ${baseUrl}: ${message}`,
+    };
+  }
+
+  if (!response.ok) {
+    const text = await response.text();
+    if (response.status === 401 || response.status === 403) {
+      return {
+        valid: false,
+        status: response.status,
+        message: "Invalid API key (authentication failed).",
+      };
+    }
+    return {
+      valid: false,
+      status: response.status,
+      message: `Embedding API returned ${response.status}: ${text.slice(0, 200)}`,
+    };
+  }
+
+  try {
+    const payload = (await response.json()) as {
+      data?: Array<{ embedding?: number[] }>;
+    };
+    const embedding = payload.data?.[0]?.embedding;
+    if (!embedding || embedding.length === 0) {
+      return {
+        valid: false,
+        status: response.status,
+        message: "Embedding API returned an empty response.",
+      };
+    }
+    if (dimensions != null && embedding.length !== dimensions) {
+      return {
+        valid: false,
+        status: response.status,
+        message: `Expected ${dimensions} embedding dimensions, got ${embedding.length}.`,
+      };
+    }
+  } catch {
+    return {
+      valid: false,
+      status: response.status,
+      message: "Embedding API returned invalid JSON.",
+    };
+  }
+
+  return { valid: true, status: response.status, message: "API key is valid." };
+}

package/src/env.ts

@@ -38,12 +38,14 @@ const EMBEDDING_API_KEY_NAMES = [
   "SEMBLE_OPENAI_API_KEY",
 ] as const;
 
+export { EMBEDDING_API_KEY_NAMES };
+
 export function resolveEmbeddingApiKey(): string {
   const key = envFirstString([...EMBEDDING_API_KEY_NAMES], "");
   if (!key) {
     throw new Error(
-      "Embedding API key required. Set TAKARA_API_KEY, OPENAI_API_KEY, MIRU_OPENAI_API_KEY, or SEMBLE_OPENAI_API_KEY " +
-        "in your MCP server env (Cursor mcp.json) or .env.local.",
+      "Embedding API key required. Run `miru setup`, set TAKARA_API_KEY, OPENAI_API_KEY, " +
+        "MIRU_OPENAI_API_KEY, or SEMBLE_OPENAI_API_KEY in your MCP server env or .env.local.",
     );
   }
   return key;

package/src/index/dense.ts

@@ -22,6 +22,14 @@ export class VectorIndex implements SemanticIndex {
     return this.vectors;
   }
 
+  vectorAt(docIndex: number): Float32Array {
+    const vec = this.vectors[docIndex];
+    if (!vec) {
+      throw new Error(`Missing vector at index ${docIndex}`);
+    }
+    return vec;
+  }
+
   memoryBytes(): number {
     return this.vectors.length * this.dimensions * 4;
   }

package/src/index/incremental.ts

@@ -0,0 +1,130 @@
+import { join, relative, resolve } from "node:path";
+import { chunkSource } from "../chunking/chunking.ts";
+import type { EmbeddingBackend } from "../embeddings/openai.ts";
+import { tokenize } from "../tokens.ts";
+import type { Chunk, ContentType } from "../types.ts";
+import { BM25Index } from "./bm25.ts";
+import { detectLanguage, getExtensions, getFileStatus, readFileText } from "./files.ts";
+import type { SemanticIndex } from "./semantic-index.ts";
+import { enrichForBm25 } from "./sparse.ts";
+import { buildSemanticIndex } from "./vector-storage.ts";
+import { vectorAt } from "./vectors.ts";
+
+export function normalizeRelativePath(path: string): string {
+  return path.replace(/\\/g, "/").replace(/^\.\/+/, "");
+}
+
+async function chunksForFile(root: string, relativePath: string): Promise<Chunk[]> {
+  const rel = normalizeRelativePath(relativePath);
+  const absolute = join(root, rel);
+  try {
+    const status = await getFileStatus(absolute);
+    if (status !== "valid") {
+      return [];
+    }
+    const source = await readFileText(absolute);
+    const language = detectLanguage(absolute);
+    return chunkSource(source, rel, language);
+  } catch {
+    return [];
+  }
+}
+
+function isIndexableRelativePath(relativePath: string, extensions: Set<string>): boolean {
+  const rel = normalizeRelativePath(relativePath);
+  const name = rel.split("/").pop() ?? rel;
+  if (name.toLowerCase() === "dockerfile") {
+    return true;
+  }
+  const dot = name.lastIndexOf(".");
+  const ext = dot >= 0 ? name.slice(dot).toLowerCase() : "";
+  return extensions.has(ext);
+}
+
+/**
+ * Replace chunks for the given repo-relative paths only: remove old chunks,
+ * embed new ones, rebuild BM25 + semantic indexes from the merged vector set.
+ */
+export async function applyIncrementalFileChanges(options: {
+  root: string;
+  content: ContentType[];
+  embeddings: EmbeddingBackend;
+  chunks: Chunk[];
+  semanticIndex: SemanticIndex;
+  relativePaths: readonly string[];
+}): Promise<{ chunks: Chunk[]; bm25: BM25Index; semantic: SemanticIndex }> {
+  const root = resolve(options.root);
+  const extensions = new Set(getExtensions(options.content).map((e) => e.toLowerCase()));
+  const targets = new Set(
+    options.relativePaths
+      .map(normalizeRelativePath)
+      .filter((p) => p.length > 0 && isIndexableRelativePath(p, extensions)),
+  );
+
+  if (targets.size === 0) {
+    return {
+      chunks: options.chunks,
+      bm25: rebuildBm25(options.chunks),
+      semantic: options.semanticIndex,
+    };
+  }
+
+  const keptChunks: Chunk[] = [];
+  const keptVectors: Float32Array[] = [];
+
+  for (let i = 0; i < options.chunks.length; i++) {
+    const chunk = options.chunks[i];
+    if (!chunk) {
+      continue;
+    }
+    const rel = normalizeRelativePath(chunk.file_path);
+    if (targets.has(rel)) {
+      continue;
+    }
+    keptChunks.push(chunk);
+    keptVectors.push(vectorAt(options.semanticIndex, i));
+  }
+
+  const addedChunks: Chunk[] = [];
+  for (const rel of targets) {
+    addedChunks.push(...(await chunksForFile(root, rel)));
+  }
+
+  let addedVectors: Float32Array[] = [];
+  if (addedChunks.length > 0) {
+    addedVectors = await options.embeddings.embedDocuments(addedChunks.map((c) => c.content));
+  }
+
+  const chunks = [...keptChunks, ...addedChunks];
+  const vectors = [...keptVectors, ...addedVectors];
+
+  if (chunks.length === 0) {
+    throw new Error(`No indexed chunks remain under ${root}.`);
+  }
+
+  if (vectors.length !== chunks.length) {
+    throw new Error(`Vector count ${vectors.length} does not match chunk count ${chunks.length}.`);
+  }
+
+  return {
+    chunks,
+    bm25: rebuildBm25(chunks),
+    semantic: buildSemanticIndex(vectors),
+  };
+}
+
+function rebuildBm25(chunks: Chunk[]): BM25Index {
+  const bm25 = new BM25Index();
+  bm25.index(chunks.map((c) => tokenize(enrichForBm25(c))));
+  return bm25;
+}
+
+/** Map an absolute changed path to a repo-relative path for chunk keys. */
+export function relativePathFromRoot(root: string, absoluteOrRelative: string): string {
+  const resolvedRoot = resolve(root);
+  const candidate = resolve(resolvedRoot, absoluteOrRelative);
+  if (candidate.startsWith(`${resolvedRoot}/`) || candidate === resolvedRoot) {
+    return normalizeRelativePath(relative(resolvedRoot, candidate));
+  }
+  return normalizeRelativePath(absoluteOrRelative);
+}

package/src/index/quantize.ts

@@ -86,6 +86,30 @@ export class QuantizedVectorIndex implements SemanticIndex {
     return this.codes.length * this.dim + this.scales.byteLength;
   }
 
+  vectorAt(docIndex: number): Float32Array {
+    const codes = this.codes[docIndex];
+    const scale = this.scales[docIndex];
+    if (!codes || scale === undefined) {
+      throw new Error(`Missing quantized vector at index ${docIndex}`);
+    }
+    const out = new Float32Array(codes.length);
+    for (let i = 0; i < codes.length; i++) {
+      out[i] = (codes[i] ?? 0) * scale;
+    }
+    let norm = 0;
+    for (let i = 0; i < out.length; i++) {
+      const v = out[i] ?? 0;
+      norm += v * v;
+    }
+    norm = Math.sqrt(norm);
+    if (norm > 0) {
+      for (let i = 0; i < out.length; i++) {
+        out[i] = (out[i] ?? 0) / norm;
+      }
+    }
+    return out;
+  }
+
   query(
     queryVector: Float32Array,
     k: number,

package/src/index/vectors.ts

@@ -0,0 +1,18 @@
+import { VectorIndex } from "./dense.ts";
+import { QuantizedVectorIndex } from "./quantize.ts";
+import type { SemanticIndex } from "./semantic-index.ts";
+
+/** Recover a float embedding for incremental re-indexing (unchanged chunks). */
+export function vectorAt(index: SemanticIndex, docIndex: number): Float32Array {
+  if (index instanceof VectorIndex) {
+    const vec = index.getVectors()[docIndex];
+    if (!vec) {
+      throw new Error(`Missing vector at index ${docIndex}`);
+    }
+    return vec;
+  }
+  if (index instanceof QuantizedVectorIndex) {
+    return index.vectorAt(docIndex);
+  }
+  throw new Error("Unsupported semantic index type");
+}

package/src/mcp/index-cache.ts

@@ -1,14 +1,13 @@
 import { watch } from "node:fs";
 import { resolve } from "node:path";
+import { normalizeRelativePath, relativePathFromRoot } from "../index/incremental.ts";
 import { MiruIndex } from "../miru-index.ts";
-import { envOptionalInt } from "../env.ts";
 import type { ContentType } from "../types.ts";
 import { computeSourceCacheKey, isGitUrl } from "../utils.ts";
 
 const CACHE_MAX_SIZE = 10;
-const DEFAULT_REINDEX_DEBOUNCE_MS = 3000;
 
-/** Directory names we skip for MCP fs.watch re-index triggers (aligned with file-walker). */
+/** Directory names we skip for MCP fs.watch update triggers (aligned with file-walker). */
 const WATCH_IGNORED_DIR_NAMES = new Set([
   ".git",
   ".hg",
@@ -29,28 +28,20 @@ const WATCH_IGNORED_DIR_NAMES = new Set([
   ".eggs",
 ]);
 
-class StaleIndexError extends Error {
-  constructor() {
-    super("Index build superseded by a newer change");
-    this.name = "StaleIndexError";
-  }
-}
-
 type CacheEntry = {
-  generation: number;
+  index: MiruIndex | null;
   task: Promise<MiruIndex> | null;
+  pendingPaths: Set<string>;
+  flushQueued: boolean;
+  updateChain: Promise<void>;
 };
 
-function resolveReindexDebounceMs(): number {
-  return envOptionalInt(["MIRU_MCP_REINDEX_DEBOUNCE_MS"], 0) ?? DEFAULT_REINDEX_DEBOUNCE_MS;
-}
-
 export function mcpWatchEnabled(): boolean {
   const raw = process.env.MIRU_MCP_WATCH ?? process.env.SEMBLE_MCP_WATCH;
   return raw !== "0" && raw !== "false";
 }
 
-/** Returns true when a watch event path should not trigger a re-index. */
+/** Returns true when a watch event path should not trigger an update. */
 export function shouldIgnoreWatchPath(relativePath: string | null | undefined): boolean {
   if (!relativePath) {
     return false;
@@ -66,24 +57,14 @@ export function shouldIgnoreWatchPath(relativePath: string | null | undefined):
 
 export class IndexCache {
   private readonly content: ContentType[];
+  private readonly defaultRef: string | null;
   private readonly entries = new Map<string, CacheEntry>();
-  private readonly reindexTimers = new Map<string, ReturnType<typeof setTimeout>>();
   private watcher: ReturnType<typeof watch> | null = null;
+  private watchedSource: string | null = null;
 
-  constructor(content: ContentType[] = ["code"]) {
+  constructor(content: ContentType[] = ["code"], defaultRef: string | null = null) {
     this.content = content;
-  }
-
-  evict(source: string, ref?: string | null): void {
-    this.invalidate(computeSourceCacheKey(source, ref));
-  }
-
-  private invalidate(cacheKey: string): void {
-    const entry = this.entries.get(cacheKey);
-    if (entry) {
-      entry.generation++;
-      entry.task = null;
-    }
+    this.defaultRef = defaultRef;
   }
 
   private ensureEntry(cacheKey: string): CacheEntry {
@@ -95,87 +76,182 @@ export class IndexCache {
           this.clearEntry(oldest);
         }
       }
-      entry = { generation: 0, task: null };
+      entry = {
+        index: null,
+        task: null,
+        pendingPaths: new Set(),
+        flushQueued: false,
+        updateChain: Promise.resolve(),
+      };
       this.entries.set(cacheKey, entry);
     }
     return entry;
   }
 
   private clearEntry(cacheKey: string): void {
-    const timer = this.reindexTimers.get(cacheKey);
-    if (timer) {
-      clearTimeout(timer);
-      this.reindexTimers.delete(cacheKey);
-    }
     this.entries.delete(cacheKey);
   }
 
   private startBuild(
     source: string,
     ref: string | null | undefined,
-    entry: CacheEntry,
-    generationAtStart: number,
+    cacheKey: string,
   ): Promise<MiruIndex> {
-    return (async () => {
+    const task = (async () => {
       const index = await MiruIndex.fromSource(source, this.content, undefined, ref);
-      if (entry.generation !== generationAtStart) {
-        throw new StaleIndexError();
-      }
       if (!isGitUrl(source)) {
         await index.saveToDefaultCache(resolve(source));
       }
       return index;
     })();
+
+    const entry = this.ensureEntry(cacheKey);
+    entry.task = task;
+    void task
+      .then((index) => {
+        entry.index = index;
+        this.maybeStartWatcher(source);
+        void this.flushFileUpdates(cacheKey, source);
+        return index;
+      })
+      .catch(() => {
+        if (entry.task === task) {
+          entry.task = null;
+        }
+      });
+
+    return task;
   }
 
   async get(source: string, ref?: string | null): Promise<MiruIndex> {
-    const cacheKey = computeSourceCacheKey(source, ref);
+    const resolvedRef = ref ?? this.defaultRef;
+    const cacheKey = computeSourceCacheKey(source, resolvedRef);
+    const entry = this.ensureEntry(cacheKey);
+
+    if (!entry.task) {
+      this.startBuild(source, resolvedRef, cacheKey);
+    }
+
+    const index = await entry.task;
+    if (!index) {
+      throw new Error(`Failed to load index for ${source}`);
+    }
+    await entry.updateChain;
+    return index;
+  }
+
+  private scheduleFlush(cacheKey: string, source: string, entry: CacheEntry): void {
+    if (!entry.flushQueued) {
+      entry.flushQueued = true;
+      queueMicrotask(() => {
+        this.flushFileUpdates(cacheKey, source);
+      });
+    }
+  }
+
+  private queueIndexedPaths(source: string, index: MiruIndex): void {
+    const cacheKey = computeSourceCacheKey(source);
+    const entry = this.ensureEntry(cacheKey);
+    for (const chunk of index.chunks) {
+      entry.pendingPaths.add(normalizeRelativePath(chunk.file_path));
+    }
+    this.scheduleFlush(cacheKey, source, entry);
+  }
+
+  /** macOS recursive fs.watch often omits filename; refresh all indexed paths incrementally. */
+  private noteAmbiguousDirectoryChange(source: string): void {
+    const cacheKey = computeSourceCacheKey(source);
+    const entry = this.ensureEntry(cacheKey);
 
-    for (;;) {
-      const entry = this.ensureEntry(cacheKey);
-      const generationAtStart = entry.generation;
+    if (entry.index) {
+      this.queueIndexedPaths(source, entry.index);
+      return;
+    }
 
-      if (!entry.task) {
-        entry.task = this.startBuild(source, ref, entry, generationAtStart);
+    if (entry.task) {
+      void entry.task.then((index) => {
+        entry.index = index;
+        this.queueIndexedPaths(source, index);
+      });
+    }
+  }
+
+  private noteFileChange(source: string, filename: string | null | undefined): void {
+    if (!filename) {
+      this.noteAmbiguousDirectoryChange(source);
+      return;
+    }
+    if (shouldIgnoreWatchPath(filename)) {
+      return;
+    }
+
+    const cacheKey = computeSourceCacheKey(source);
+    const entry = this.ensureEntry(cacheKey);
+    const rel = relativePathFromRoot(source, filename);
+    if (!rel) {
+      return;
+    }
+    entry.pendingPaths.add(rel);
+    this.scheduleFlush(cacheKey, source, entry);
+  }
+
+  private flushFileUpdates(cacheKey: string, source: string): void {
+    const entry = this.entries.get(cacheKey);
+    if (!entry) {
+      return;
+    }
+    entry.flushQueued = false;
+
+    const run = async (): Promise<void> => {
+      const paths = [...entry.pendingPaths];
+      entry.pendingPaths.clear();
+      if (paths.length === 0) {
+        return;
       }
 
-      const task = entry.task;
-      try {
-        const index = await task;
-        if (entry.generation !== generationAtStart) {
-          continue;
+      let index = entry.index;
+      if (!index && entry.task) {
+        try {
+          index = await entry.task;
+        } catch {
+          for (const p of paths) {
+            entry.pendingPaths.add(p);
+          }
+          return;
         }
-        return index;
-      } catch (err) {
-        if (entry.task === task) {
-          entry.task = null;
+      }
+      if (!index) {
+        for (const p of paths) {
+          entry.pendingPaths.add(p);
         }
-        if (err instanceof StaleIndexError) {
-          continue;
+        return;
+      }
+
+      try {
+        await index.applyFileChanges(paths);
+        if (!isGitUrl(source)) {
+          await index.persistToCache(resolve(source));
         }
-        if (this.entries.get(cacheKey) === entry) {
-          this.entries.delete(cacheKey);
+      } catch {
+        for (const p of paths) {
+          entry.pendingPaths.add(p);
         }
-        throw err;
       }
-    }
+    };
+
+    entry.updateChain = entry.updateChain.then(run, run);
   }
 
-  private scheduleReindex(source: string, ref?: string | null): void {
-    const cacheKey = computeSourceCacheKey(source, ref);
-    const existing = this.reindexTimers.get(cacheKey);
-    if (existing) {
-      clearTimeout(existing);
+  private maybeStartWatcher(source: string): void {
+    if (!mcpWatchEnabled() || isGitUrl(source)) {
+      return;
     }
-
-    const debounceMs = resolveReindexDebounceMs();
-    const timer = setTimeout(() => {
-      this.reindexTimers.delete(cacheKey);
-      this.invalidate(cacheKey);
-      void this.get(source, ref).catch(() => undefined);
-    }, debounceMs);
-
-    this.reindexTimers.set(cacheKey, timer);
+    const resolved = resolve(source);
+    if (this.watchedSource === resolved) {
+      return;
+    }
+    this.watchedSource = resolved;
+    this.startWatcher(resolved);
   }
 
   startWatcher(path: string): void {
@@ -185,47 +261,40 @@ export class IndexCache {
     }
 
     this.watcher = watch(resolved, { recursive: true }, (_event, filename) => {
-      if (shouldIgnoreWatchPath(filename)) {
-        return;
-      }
-      this.scheduleReindex(resolved);
+      this.noteFileChange(resolved, filename);
     });
   }
 
   close(): void {
-    for (const timer of this.reindexTimers.values()) {
-      clearTimeout(timer);
-    }
-    this.reindexTimers.clear();
     this.watcher?.close();
     this.watcher = null;
+    this.watchedSource = null;
+    this.entries.clear();
   }
 }
 
 export async function getIndexForRepo(
   repo: string | null | undefined,
-  defaultSource: string | null,
   cache: IndexCache,
   ref?: string | null,
 ): Promise<MiruIndex> {
-  if (repo && isGitUrl(repo) && !repo.startsWith("https://") && !repo.startsWith("http://")) {
+  if (!repo) {
     throw new Error(
-      `Only https://, http://, or local directory paths are accepted as \`repo\`. Got: ${repo}`,
+      "Pass an https:// or http:// git URL or local directory path as `repo` (project root for local workspaces).",
     );
   }
 
-  const source = repo ?? defaultSource;
-  if (!source) {
+  if (isGitUrl(repo) && !repo.startsWith("https://") && !repo.startsWith("http://")) {
     throw new Error(
-      "No repo specified and no default index. Pass an https:// or http:// git URL or local directory path as `repo`.",
+      `Only https://, http://, or local directory paths are accepted as \`repo\`. Got: ${repo}`,
     );
   }
 
   try {
-    return await cache.get(source, ref);
+    return await cache.get(repo, ref);
   } catch (err) {
     const message = err instanceof Error ? err.message : String(err);
-    throw new Error(`Failed to index ${source}: ${message}`);
+    throw new Error(`Failed to index ${repo}: ${message}`);
   }
 }