@takara-ai/miru-code 0.8.0 → 1.0.3

package/README.md

@@ -114,7 +114,46 @@ Pass the **project root** as `repo` for local projects. Prefer these over Grep,
 
 Hybrid search: Takara embeddings + BM25 + fusion + reranking. Indexes **code**, **docs**, **config**, or **all** via `--content`.
 
-Disk cache: `~/Library/Caches/miru` (macOS), `~/.cache/miru` (Linux). MCP watches local files and updates incrementally; run `miru clear .` after big refactors when using CLI only.
+Disk cache: `~/Library/Caches/miru` (macOS), `~/.cache/miru` (Linux). MCP watches local files and updates incrementally; run `miru clear .` after big refactors when using CLI only. Upgrading Miru invalidates stale indexes automatically when the package version epoch changes.
+
+### Chunking & languages
+
+Miru chunks source in tiers: **AST** (tree-sitter, default) → **structural** heuristics → **line** splits.
+
+**AST chunking** — 22 languages (syntax-aware boundaries via vendored `web-tree-sitter` grammars):
+
+| Language | Typical extensions |
+|----------|-------------------|
+| bash | `.sh`, `.bash`, `.zsh` |
+| c | `.c` |
+| cpp | `.cpp`, `.h`, `.hpp`, etc. |
+| csharp | `.cs` |
+| css | `.css` |
+| dart | `.dart` |
+| elixir | `.ex`, `.exs` |
+| embeddedtemplate | ERB-style templates |
+| go | `.go` |
+| haskell | `.hs` |
+| html | `.html` |
+| java | `.java` |
+| javascript | `.js`, `.jsx`, `.mjs`, `.cjs` |
+| json | `.json` |
+| ocaml | `.ml`, etc. |
+| php | `.php` |
+| python | `.py`, `.pyi` |
+| ruby | `.rb` |
+| rust | `.rs` |
+| scala | `.scala` |
+| solidity | `.sol` |
+| typescript | `.ts`, `.mts`, `.cts` |
+
+**Also shipped:** `.tsx` uses a dedicated TSX grammar; extra OCaml and PHP-only grammars are vendored alongside the main set.
+
+**Structural fallback** (brace/indent heuristics when AST is unavailable): python, go, typescript, javascript, cpp, c.
+
+**Line fallback:** everything else that gets indexed (kotlin, swift, vue, sql, etc.) — still searchable, coarser chunks.
+
+Set `MIRU_AST_CHUNKING=0` to disable AST and use structural → lines only.
 
 ## CLI reference
 
@@ -149,10 +188,12 @@ const results = await index.search({ query: "BM25 tokenize", topK: 10 });
 |----------|-------|
 | `TAKARA_API_KEY` | Required |
 | `MIRU_OPENAI_BASE_URL` | Default `https://infer.takara.ai/v1` |
+| `MIRU_OPENAI_EMBEDDING_MODEL` | Default `ds1-miru-int8` |
 | `MIRU_WORKSPACE_ROOT` | Restrict MCP local `repo` paths to this directory |
 | `MIRU_MAX_INDEX_FILES` | Cap files indexed per operation |
 | `MIRU_ALLOW_HTTP_GIT` | Set `1` to allow plain `http://` git clones |
 | `MIRU_MCP_WATCH` | Set `0` to disable MCP file watch |
+| `MIRU_AST_CHUNKING` | Set `0` to disable tree-sitter AST chunking |
 | `NO_COLOR` | Disable CLI colors |
 
 See `.env.example` for more.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@takara-ai/miru-code",
-  "version": "0.8.0",
+  "version": "1.0.3",
   "description": "Miru (見る) — hybrid code search for agents. Bun, TypeScript, Takara embeddings.",
   "type": "module",
   "license": "MIT",

package/src/embeddings/openai.ts

@@ -1,15 +1,16 @@
 import { mapPool, resolveWorkerConcurrency } from "../concurrency.ts";
 import { envFirstString, envOptionalInt, resolveEmbeddingApiKey } from "../env.ts";
 
-const DEFAULT_MODEL = "ds1-potion-code-16m";
+const DEFAULT_MODEL = "ds1-miru-int8";
 const DEFAULT_BASE_URL = "https://infer.takara.ai/v1";
 const DEFAULT_BATCH_SIZE = 32;
 const DEFAULT_MAX_EMBED_CHARS = 1300;
 const WINDOW_OVERLAP_CHARS = 120;
 
-/** Native output size for Takara ds1-potion-code-16m (override with MIRU_EMBEDDING_DIMENSIONS). */
+/** Native output size per model (override with MIRU_EMBEDDING_DIMENSIONS). */
 const MODEL_DEFAULT_DIMENSIONS: Record<string, number> = {
   "ds1-potion-code-16m": 256,
+  "ds1-miru-int8": 256,
 };
 
 export interface EmbeddingTransportStats {
@@ -123,9 +124,47 @@ export function resolveEmbeddingBaseUrl(): string {
   );
 }
 
+interface Int8Embedding {
+  dtype: "i8";
+  values: number[];
+  scale: number;
+  zero_point: number;
+}
+
+type EmbeddingPayload = number[] | Int8Embedding;
+
 interface EmbeddingResponseItem {
   index: number;
-  embedding: number[];
+  embedding: EmbeddingPayload;
+}
+
+function isInt8Embedding(embedding: EmbeddingPayload): embedding is Int8Embedding {
+  return (
+    typeof embedding === "object" &&
+    embedding !== null &&
+    !Array.isArray(embedding) &&
+    embedding.dtype === "i8" &&
+    Array.isArray(embedding.values)
+  );
+}
+
+export function embeddingDimensions(embedding: EmbeddingPayload): number {
+  return isInt8Embedding(embedding) ? embedding.values.length : embedding.length;
+}
+
+export function dequantizeEmbedding(embedding: EmbeddingPayload): Float32Array {
+  if (Array.isArray(embedding)) {
+    return new Float32Array(embedding);
+  }
+  if (isInt8Embedding(embedding)) {
+    const { values, scale, zero_point } = embedding;
+    const out = new Float32Array(values.length);
+    for (let i = 0; i < values.length; i++) {
+      out[i] = ((values[i] ?? 0) - zero_point) * scale;
+    }
+    return out;
+  }
+  throw new Error("Embedding API returned unsupported embedding format");
 }
 
 interface EmbeddingResponse {
@@ -402,7 +441,7 @@ function vectorsFromResponse(data: EmbeddingResponseItem[], expected: number): F
   const byIndex = new Map<number, Float32Array>();
   for (const item of data) {
     if (item.index >= 0 && item.index < expected) {
-      byIndex.set(item.index, new Float32Array(item.embedding));
+      byIndex.set(item.index, dequantizeEmbedding(item.embedding));
     }
   }
   if (byIndex.size !== expected) {

package/src/embeddings/validate.ts

@@ -1,4 +1,5 @@
 import {
+  embeddingDimensions,
   resolveEmbeddingBaseUrl,
   resolveEmbeddingDimensions,
   resolveEmbeddingModel,
@@ -65,21 +66,29 @@ export async function validateEmbeddingApiKey(options: {
 
   try {
     const payload = (await response.json()) as {
-      data?: Array<{ embedding?: number[] }>;
+      data?: Array<{ embedding?: Parameters<typeof embeddingDimensions>[0] }>;
     };
     const embedding = payload.data?.[0]?.embedding;
-    if (!embedding || embedding.length === 0) {
+    if (!embedding) {
       return {
         valid: false,
         status: response.status,
         message: "Embedding API returned an empty response.",
       };
     }
-    if (dimensions != null && embedding.length !== dimensions) {
+    const dim = embeddingDimensions(embedding);
+    if (dim === 0) {
       return {
         valid: false,
         status: response.status,
-        message: `Expected ${dimensions} embedding dimensions, got ${embedding.length}.`,
+        message: "Embedding API returned an empty response.",
+      };
+    }
+    if (dimensions != null && dim !== dimensions) {
+      return {
+        valid: false,
+        status: response.status,
+        message: `Expected ${dimensions} embedding dimensions, got ${dim}.`,
       };
     }
   } catch {

package/src/help.ts

@@ -55,6 +55,8 @@ export function printEnvHelp(): void {
   writeStdout("      Takara bearer token for embeddings");
   writeStdout("  MIRU_OPENAI_BASE_URL");
   writeStdout("      Default: https://infer.takara.ai/v1");
+  writeStdout("  MIRU_OPENAI_EMBEDDING_MODEL");
+  writeStdout("      Default: ds1-miru-int8");
   writeStdout("  MIRU_CONCURRENCY");
   writeStdout("      Parallel workers (default: CPUs − 2)");
   writeStdout("");