@takara-ai/miru-code 0.6.1 → 0.7.0

package/grammars/manifest.json

@@ -0,0 +1,56 @@
+{
+  "version": 1,
+  "runtime": "web-tree-sitter@^0.26.9",
+  "languages": {
+    "bash": "tree-sitter-bash.wasm",
+    "c": "tree-sitter-c.wasm",
+    "cpp": "tree-sitter-cpp.wasm",
+    "csharp": "tree-sitter-c_sharp.wasm",
+    "css": "tree-sitter-css.wasm",
+    "dart": "tree-sitter-dart.wasm",
+    "elixir": "tree-sitter-elixir.wasm",
+    "embeddedtemplate": "tree-sitter-embedded_template.wasm",
+    "go": "tree-sitter-go.wasm",
+    "haskell": "tree-sitter-haskell.wasm",
+    "html": "tree-sitter-html.wasm",
+    "java": "tree-sitter-java.wasm",
+    "javascript": "tree-sitter-javascript.wasm",
+    "json": "tree-sitter-json.wasm",
+    "ocaml": "tree-sitter-ocaml.wasm",
+    "php": "tree-sitter-php.wasm",
+    "python": "tree-sitter-python.wasm",
+    "ruby": "tree-sitter-ruby.wasm",
+    "rust": "tree-sitter-rust.wasm",
+    "scala": "tree-sitter-scala.wasm",
+    "solidity": "tree-sitter-solidity.wasm",
+    "typescript": "tree-sitter-typescript.wasm"
+  },
+  "files": [
+    "tree-sitter-bash.wasm",
+    "tree-sitter-c.wasm",
+    "tree-sitter-c_sharp.wasm",
+    "tree-sitter-cpp.wasm",
+    "tree-sitter-css.wasm",
+    "tree-sitter-dart.wasm",
+    "tree-sitter-elixir.wasm",
+    "tree-sitter-embedded_template.wasm",
+    "tree-sitter-go.wasm",
+    "tree-sitter-haskell.wasm",
+    "tree-sitter-html.wasm",
+    "tree-sitter-java.wasm",
+    "tree-sitter-javascript.wasm",
+    "tree-sitter-json.wasm",
+    "tree-sitter-ocaml.wasm",
+    "tree-sitter-ocaml_interface.wasm",
+    "tree-sitter-ocaml_type.wasm",
+    "tree-sitter-php.wasm",
+    "tree-sitter-php_only.wasm",
+    "tree-sitter-python.wasm",
+    "tree-sitter-ruby.wasm",
+    "tree-sitter-rust.wasm",
+    "tree-sitter-scala.wasm",
+    "tree-sitter-solidity.wasm",
+    "tree-sitter-tsx.wasm",
+    "tree-sitter-typescript.wasm"
+  ]
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@takara-ai/miru-code",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "Miru (見る) — hybrid code search for agents. Bun, TypeScript, Takara embeddings.",
   "type": "module",
   "license": "MIT",
@@ -36,6 +36,7 @@
   },
   "files": [
     "src",
+    "grammars",
     "README.md",
     "LICENSE"
   ],
@@ -49,7 +50,8 @@
     "typecheck": "tsc --noEmit",
     "lint": "biome check .",
     "lint:fix": "biome check --write .",
-    "prepublishOnly": "bun run typecheck && bun test",
+    "vendor-grammars": "bun run scripts/vendor-grammars.ts",
+    "prepublishOnly": "bun run vendor-grammars && bun run typecheck && bun test",
     "benchmark:ab": "bun run scripts/search-ab-local.ts",
     "benchmark:ab-multi": "bun run scripts/search-ab-multi.ts",
     "benchmark:recall": "bun run scripts/benchmark-recall-tokens.ts",
@@ -61,11 +63,34 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
     "ignore": "^7.0.5",
+    "web-tree-sitter": "^0.26.9",
     "zod": "^4.4.3"
   },
   "devDependencies": {
     "@biomejs/biome": "^2.4.16",
     "@types/bun": "latest",
+    "tree-sitter-bash": "^0.25.1",
+    "tree-sitter-c": "^0.24.1",
+    "tree-sitter-c-sharp": "^0.23.1",
+    "tree-sitter-cpp": "^0.23.4",
+    "tree-sitter-css": "^0.25.0",
+    "tree-sitter-dart": "^1.0.0",
+    "tree-sitter-elixir": "^0.3.4",
+    "tree-sitter-embedded-template": "^0.25.0",
+    "tree-sitter-go": "^0.25.0",
+    "tree-sitter-haskell": "^0.23.1",
+    "tree-sitter-html": "^0.23.2",
+    "tree-sitter-java": "^0.23.5",
+    "tree-sitter-javascript": "^0.25.0",
+    "tree-sitter-json": "^0.24.8",
+    "tree-sitter-ocaml": "^0.24.2",
+    "tree-sitter-php": "^0.23.12",
+    "tree-sitter-python": "^0.25.0",
+    "tree-sitter-ruby": "^0.23.1",
+    "tree-sitter-rust": "^0.24.0",
+    "tree-sitter-scala": "^0.24.0",
+    "tree-sitter-solidity": "^1.2.13",
+    "tree-sitter-typescript": "^0.23.2",
     "typescript": "^5.9.3"
   }
 }

package/src/chunking/ast.ts

@@ -0,0 +1,119 @@
+import type { Node } from "web-tree-sitter";
+import { Parser } from "web-tree-sitter";
+import { getLanguageForFile } from "./grammars.ts";
+import { type ChunkBoundary, mergeAdjacentChunks } from "./lines.ts";
+
+const RECURSION_DEPTH = 500;
+const MIN_CHUNK_SIZE = 50;
+
+function astChunkingEnabled(): boolean {
+  return process.env.MIRU_AST_CHUNKING !== "0";
+}
+
+function mergeNodeInner(node: Node, desiredLength: number, depth: number): ChunkBoundary[] {
+  if (node.childCount === 0) {
+    return [{ start: node.startIndex, end: node.endIndex }];
+  }
+
+  const length = node.endIndex - node.startIndex;
+  if (depth > RECURSION_DEPTH) {
+    return [{ start: node.startIndex, end: node.endIndex }];
+  }
+  if (length < MIN_CHUNK_SIZE) {
+    return [{ start: node.startIndex, end: node.endIndex }];
+  }
+
+  const groups: ChunkBoundary[] = [];
+  const children = node.children;
+  let index = 0;
+
+  while (index < children.length) {
+    const child = children[index];
+    if (!child) {
+      break;
+    }
+
+    const start = child.startIndex;
+    let end = child.endIndex;
+    let groupLength = end - start;
+    index += 1;
+
+    if (groupLength > desiredLength) {
+      groups.push(...mergeNodeInner(child, desiredLength, depth + 1));
+      continue;
+    }
+
+    while (index < children.length) {
+      const nextChild = children[index];
+      if (!nextChild) {
+        break;
+      }
+      const childLength = nextChild.endIndex - nextChild.startIndex;
+      if (groupLength + childLength > desiredLength) {
+        break;
+      }
+      end = nextChild.endIndex;
+      groupLength += childLength;
+      index += 1;
+    }
+
+    groups.push({ start, end });
+  }
+
+  return groups;
+}
+
+function mergeNode(node: Node, desiredLength: number): ChunkBoundary[] {
+  const rawChunks = mergeNodeInner(node, desiredLength, 0);
+  return mergeAdjacentChunks(rawChunks, desiredLength);
+}
+
+function byteBoundariesToCharBoundaries(
+  source: string,
+  byteBoundaries: ChunkBoundary[],
+): ChunkBoundary[] {
+  const sourceBytes = new TextEncoder().encode(source);
+  const decoder = new TextDecoder();
+
+  return byteBoundaries.map((boundary) => ({
+    start: decoder.decode(sourceBytes.subarray(0, boundary.start)).length,
+    end: decoder.decode(sourceBytes.subarray(0, boundary.end)).length,
+  }));
+}
+
+/** AST-aware chunk boundaries via vendored tree-sitter grammars; null to fall back. */
+export async function chunkAst(
+  source: string,
+  filePath: string,
+  language: string | null,
+  desiredLength: number,
+): Promise<ChunkBoundary[] | null> {
+  if (!source.trim() || !astChunkingEnabled()) {
+    return null;
+  }
+
+  const languageObj = await getLanguageForFile(filePath, language);
+  if (!languageObj) {
+    return null;
+  }
+
+  const parser = new Parser();
+  parser.setLanguage(languageObj);
+
+  let tree: ReturnType<Parser["parse"]>;
+  try {
+    tree = parser.parse(source);
+  } catch {
+    return null;
+  }
+  if (!tree) {
+    return null;
+  }
+
+  try {
+    const byteBoundaries = mergeNode(tree.rootNode, desiredLength);
+    return byteBoundariesToCharBoundaries(source, byteBoundaries);
+  } finally {
+    tree.delete();
+  }
+}

package/src/chunking/chunking.ts

@@ -1,16 +1,22 @@
 import type { Chunk } from "../types.ts";
+import { chunkAst } from "./ast.ts";
 import { chunkLines } from "./lines.ts";
 import { chunkStructural } from "./structural.ts";
 
 /** Match upstream hybrid-search chunking target length. */
 const DESIRED_CHUNK_LENGTH_CHARS = 1500;
 
-export function chunkSource(source: string, filePath: string, language: string | null): Chunk[] {
+export async function chunkSource(
+  source: string,
+  filePath: string,
+  language: string | null,
+): Promise<Chunk[]> {
   if (!source.trim()) {
     return [];
   }
 
   const boundaries =
+    (await chunkAst(source, filePath, language, DESIRED_CHUNK_LENGTH_CHARS)) ??
     chunkStructural(source, language, DESIRED_CHUNK_LENGTH_CHARS) ??
     chunkLines(source, DESIRED_CHUNK_LENGTH_CHARS);
   const chunks: Chunk[] = [];

package/src/chunking/grammars.ts

@@ -0,0 +1,99 @@
+import { existsSync, readFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { Language, Parser } from "web-tree-sitter";
+
+const require = createRequire(import.meta.url);
+const packageRoot = join(dirname(fileURLToPath(import.meta.url)), "../..");
+
+export type GrammarManifest = {
+  version: number;
+  runtime: string;
+  languages: Record<string, string>;
+  files: string[];
+};
+
+export const grammarManifest: GrammarManifest = JSON.parse(
+  readFileSync(join(packageRoot, "grammars/manifest.json"), "utf8"),
+) as GrammarManifest;
+
+/** Absolute path to the vendored grammars directory. */
+export function grammarsDir(): string {
+  return join(packageRoot, "grammars");
+}
+
+/** Resolve vendored wasm path for a Miru language id, if we ship a grammar for it. */
+export function wasmPathForLanguage(language: string | null): string | null {
+  if (!language) {
+    return null;
+  }
+  const file = grammarManifest.languages[language];
+  if (!file) {
+    return null;
+  }
+  return join(grammarsDir(), file);
+}
+
+/** Pick grammar wasm for a file (e.g. tsx vs typescript). */
+export function wasmPathForFile(filePath: string, language: string | null): string | null {
+  if (!language) {
+    return null;
+  }
+  if (language === "typescript" && filePath.toLowerCase().endsWith(".tsx")) {
+    const tsxPath = join(grammarsDir(), "tree-sitter-tsx.wasm");
+    if (existsSync(tsxPath)) {
+      return tsxPath;
+    }
+  }
+  return wasmPathForLanguage(language);
+}
+
+/** Absolute path to web-tree-sitter's runtime wasm (0.26.x). */
+export function webTreeSitterRuntimePath(): string {
+  const pkgDir = dirname(require.resolve("web-tree-sitter/package.json"));
+  return join(pkgDir, "web-tree-sitter.wasm");
+}
+
+export function hasVendoredGrammar(language: string | null): boolean {
+  return wasmPathForLanguage(language) !== null;
+}
+
+let parserInitPromise: Promise<void> | null = null;
+const languageCache = new Map<string, Promise<Language>>();
+
+/** Initialize web-tree-sitter once per process (safe to call repeatedly). */
+export async function ensureParserInit(): Promise<void> {
+  if (!parserInitPromise) {
+    parserInitPromise = Parser.init({ locateFile: () => webTreeSitterRuntimePath() });
+  }
+  await parserInitPromise;
+}
+
+async function loadLanguage(wasmPath: string): Promise<Language> {
+  let pending = languageCache.get(wasmPath);
+  if (!pending) {
+    pending = (async () => {
+      await ensureParserInit();
+      return Language.load(wasmPath);
+    })();
+    languageCache.set(wasmPath, pending);
+  }
+  return pending;
+}
+
+/** Load a cached Language for the given file, or null if unsupported / missing. */
+export async function getLanguageForFile(
+  filePath: string,
+  language: string | null,
+): Promise<Language | null> {
+  const wasmPath = wasmPathForFile(filePath, language);
+  if (!wasmPath || !existsSync(wasmPath)) {
+    return null;
+  }
+  try {
+    return await loadLanguage(wasmPath);
+  } catch {
+    return null;
+  }
+}

package/src/cli.ts

@@ -36,6 +36,7 @@ import {
   resolveContent,
   resolveSearchPath,
 } from "./utils.ts";
+import { maybeNotifyUpdate, miruVersion } from "./version.ts";
 
 loadEnvFiles();
 normalizeTakaraApiKeyEnv();
@@ -54,6 +55,8 @@ const CLI_COMMANDS = new Set([
   "help",
   "-h",
   "--help",
+  "-v",
+  "--version",
 ]);
 
 const AGENTS = new Set<AgentId>(AGENT_IDS);
@@ -269,6 +272,11 @@ async function runCli(argv: string[]): Promise<void> {
     return;
   }
 
+  if (command === "-v" || command === "--version") {
+    console.log(miruVersion());
+    return;
+  }
+
   if (command === "help") {
     const topic = rest[0];
     if (!topic) {
@@ -444,13 +452,24 @@ async function runMcpWithCredentials(argv: string[]): Promise<void> {
 async function main(): Promise<void> {
   const argv = process.argv.slice(2);
   const first = argv[0];
+
+  if (first === "-v" || first === "--version") {
+    console.log(miruVersion());
+    return;
+  }
+
   if (first === "hook-guard") {
     process.exit(await runSearchGuardFromStdin());
   }
+
+  const updateNotice = maybeNotifyUpdate();
+
   if (first && CLI_COMMANDS.has(first)) {
-    await runCli(argv);
+    await Promise.all([runCli(argv), updateNotice]);
     return;
   }
+
+  await updateNotice;
   await runMcpWithCredentials(argv);
 }
 

package/src/embeddings/openai.ts

@@ -31,17 +31,11 @@ export interface EmbeddingBackend {
 }
 
 export function resolveEmbeddingModel(): string {
-  return envFirstString(
-    ["MIRU_OPENAI_EMBEDDING_MODEL", "OPENAI_EMBEDDING_MODEL"],
-    DEFAULT_MODEL,
-  );
+  return envFirstString(["MIRU_OPENAI_EMBEDDING_MODEL", "OPENAI_EMBEDDING_MODEL"], DEFAULT_MODEL);
 }
 
 export function resolveMaxEmbedChars(): number {
-  return (
-    envOptionalInt(["MIRU_MAX_EMBED_CHARS"], 256) ??
-    DEFAULT_MAX_EMBED_CHARS
-  );
+  return envOptionalInt(["MIRU_MAX_EMBED_CHARS"], 256) ?? DEFAULT_MAX_EMBED_CHARS;
 }
 
 function splitIntoWindows(text: string, maxChars: number): string[] {
@@ -123,10 +117,10 @@ export function resolveEmbeddingBatchSize(): number {
 }
 
 export function resolveEmbeddingBaseUrl(): string {
-  return envFirstString(
-    ["MIRU_OPENAI_BASE_URL", "OPENAI_BASE_URL"],
-    DEFAULT_BASE_URL,
-  ).replace(/\/$/, "");
+  return envFirstString(["MIRU_OPENAI_BASE_URL", "OPENAI_BASE_URL"], DEFAULT_BASE_URL).replace(
+    /\/$/,
+    "",
+  );
 }
 
 interface EmbeddingResponseItem {

package/src/git.ts

@@ -6,8 +6,7 @@ import { envOptionalInt } from "./env.ts";
 const DEFAULT_CLONE_TIMEOUT_SEC = 60;
 
 export async function cloneGitRepository(url: string, ref?: string | null): Promise<string> {
-  const timeoutSec =
-    envOptionalInt(["MIRU_CLONE_TIMEOUT"], 1) ?? DEFAULT_CLONE_TIMEOUT_SEC;
+  const timeoutSec = envOptionalInt(["MIRU_CLONE_TIMEOUT"], 1) ?? DEFAULT_CLONE_TIMEOUT_SEC;
   const dir = await mkdtemp(join(tmpdir(), "miru-git-"));
   const args = ["clone", "--depth", "1"];
   if (ref) {

package/src/help.ts

@@ -45,7 +45,7 @@ export function printMainHelp(): void {
   writeStdout("  miru setup && miru install");
   writeStdout('  miru search "auth middleware" ./src');
   writeStdout("");
-  hint("miru help <command>  ·  miru -h for environment variables");
+  hint("miru help <command>  ·  miru -h for environment variables  ·  miru -v for version");
   writeStdout("");
 }
 

package/src/index/create.ts

@@ -1,5 +1,6 @@
 import { relative, resolve } from "node:path";
 import { chunkSource } from "../chunking/chunking.ts";
+import { ensureParserInit } from "../chunking/grammars.ts";
 import { resolveWorkerConcurrency } from "../concurrency.ts";
 import type { EmbeddingBackend } from "../embeddings/openai.ts";
 import { envOptionalInt } from "../env.ts";
@@ -18,17 +19,11 @@ const DEFAULT_PIPELINE_EMBED_BATCH = 64;
 const DEFAULT_PIPELINE_EMBED_INFLIGHT = 8;
 
 function resolvePipelineEmbedBatch(): number {
-  return (
-    envOptionalInt(["MIRU_PIPELINE_EMBED_BATCH"], 1) ??
-    DEFAULT_PIPELINE_EMBED_BATCH
-  );
+  return envOptionalInt(["MIRU_PIPELINE_EMBED_BATCH"], 1) ?? DEFAULT_PIPELINE_EMBED_BATCH;
 }
 
 function resolvePipelineEmbedInflight(): number {
-  return (
-    envOptionalInt(["MIRU_PIPELINE_EMBED_INFLIGHT"], 1) ??
-    DEFAULT_PIPELINE_EMBED_INFLIGHT
-  );
+  return envOptionalInt(["MIRU_PIPELINE_EMBED_INFLIGHT"], 1) ?? DEFAULT_PIPELINE_EMBED_INFLIGHT;
 }
 
 function resolveMaxIndexFiles(): number | undefined {
@@ -54,6 +49,8 @@ export async function createIndexFromPath(
     embeddings.resetStats();
   }
 
+  await ensureParserInit();
+
   const resolved = resolve(path);
   const root = displayRoot ? resolve(displayRoot) : resolved;
   const extensions = getExtensions(content);
@@ -80,7 +77,7 @@ export async function createIndexFromPath(
       }
       const source = await readFileText(filePath);
       const chunkPath = displayRoot ? relative(root, filePath).replace(/\\/g, "/") : filePath;
-      return chunkSource(source, chunkPath, language);
+      return await chunkSource(source, chunkPath, language);
     } catch {
       fileErrors++;
       return [];

package/src/index/incremental.ts

@@ -24,7 +24,7 @@ async function chunksForFile(root: string, relativePath: string): Promise<Chunk[
     }
     const source = await readFileText(absolute);
     const language = detectLanguage(absolute);
-    return chunkSource(source, rel, language);
+    return await chunkSource(source, rel, language);
   } catch {
     return [];
   }

package/src/index/vector-storage.ts

@@ -6,9 +6,7 @@ export type SemanticStorage = "int8" | "float32";
 
 /** Default: int8 quantized vectors (~4x less RAM). Set MIRU_FLOAT_VECTORS=1 for float32. */
 export function resolveSemanticStorage(): SemanticStorage {
-  return process.env.MIRU_FLOAT_VECTORS === "1"
-    ? "float32"
-    : "int8";
+  return process.env.MIRU_FLOAT_VECTORS === "1" ? "float32" : "int8";
 }
 
 export function semanticStorageFromMetadata(metadata: Record<string, unknown>): SemanticStorage {

package/src/version.ts

@@ -0,0 +1,119 @@
+import { readFile, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import packageJson from "../package.json";
+import { resolveCacheFolder } from "./cache.ts";
+import { hint, writeStderr } from "./cli-ui.ts";
+
+const PACKAGE_NAME = "@takara-ai/miru-code";
+const REGISTRY_URL = "https://registry.npmjs.org/@takara-ai%2Fmiru-code";
+const UPDATE_CHECK_FILENAME = "update-check.json";
+const CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000;
+const FETCH_TIMEOUT_MS = 2_000;
+
+interface UpdateCheckCache {
+  checkedAt: number;
+  latest: string;
+}
+
+export function miruVersion(): string {
+  return packageJson.version;
+}
+
+export function isVersionNewer(latest: string, current: string): boolean {
+  const parse = (v: string): [number, number, number] | null => {
+    const match = v.trim().match(/^(\d+)\.(\d+)\.(\d+)/);
+    if (!match) {
+      return null;
+    }
+    return [Number(match[1]), Number(match[2]), Number(match[3])];
+  };
+
+  const a = parse(latest);
+  const b = parse(current);
+  if (!a || !b) {
+    return latest !== current;
+  }
+
+  for (let i = 0; i < 3; i++) {
+    if (a[i]! > b[i]!) {
+      return true;
+    }
+    if (a[i]! < b[i]!) {
+      return false;
+    }
+  }
+  return false;
+}
+
+function updateCheckPath(): string {
+  return join(resolveCacheFolder(), UPDATE_CHECK_FILENAME);
+}
+
+async function readUpdateCheckCache(): Promise<UpdateCheckCache | null> {
+  try {
+    const raw = await readFile(updateCheckPath(), "utf-8");
+    const parsed = JSON.parse(raw) as UpdateCheckCache;
+    if (typeof parsed.checkedAt !== "number" || typeof parsed.latest !== "string") {
+      return null;
+    }
+    return parsed;
+  } catch {
+    return null;
+  }
+}
+
+async function writeUpdateCheckCache(latest: string): Promise<void> {
+  const payload: UpdateCheckCache = { checkedAt: Date.now(), latest };
+  await writeFile(updateCheckPath(), `${JSON.stringify(payload)}\n`, "utf-8");
+}
+
+export async function fetchLatestPublishedVersion(): Promise<string> {
+  const response = await fetch(REGISTRY_URL, {
+    headers: { accept: "application/json" },
+    signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+  });
+  if (!response.ok) {
+    throw new Error(`registry ${response.status}`);
+  }
+  const body = (await response.json()) as { "dist-tags"?: { latest?: string } };
+  const latest = body["dist-tags"]?.latest;
+  if (!latest) {
+    throw new Error("missing dist-tags.latest");
+  }
+  return latest;
+}
+
+/** Print update hint to stderr when a newer package is on npm (at most once per day). */
+export async function maybeNotifyUpdate(): Promise<void> {
+  if (process.env.MIRU_NO_UPDATE_CHECK === "1") {
+    return;
+  }
+
+  const current = miruVersion();
+  const cached = await readUpdateCheckCache();
+  if (cached && Date.now() - cached.checkedAt < CHECK_INTERVAL_MS) {
+    if (isVersionNewer(cached.latest, current)) {
+      writeUpdateNotice(cached.latest, current);
+    }
+    return;
+  }
+
+  try {
+    const latest = await fetchLatestPublishedVersion();
+    await writeUpdateCheckCache(latest);
+    if (isVersionNewer(latest, current)) {
+      writeUpdateNotice(latest, current);
+    }
+  } catch {
+    // Offline or registry unreachable — skip silently.
+  }
+}
+
+function writeUpdateNotice(latest: string, current: string): void {
+  writeStderr();
+  hint(
+    `Update available: ${latest} (you have ${current}). Run: bun add -g ${PACKAGE_NAME}`,
+    process.stderr,
+  );
+  writeStderr();
+}