@takara-ai/miru-code 0.1.0

package/src/cache.ts

@@ -0,0 +1,141 @@
+import { createHash } from "node:crypto";
+import { rm } from "node:fs/promises";
+import { join, resolve } from "node:path";
+import { resolveEmbeddingDimensions, resolveEmbeddingModel } from "./embeddings/openai.ts";
+import {
+  loadBm25,
+  loadSemantic,
+  pathsExist,
+  persistencePaths,
+  semanticIndexMatchesStorage,
+} from "./index/persistence.ts";
+import { resolveSemanticStorage, semanticStorageFromMetadata } from "./index/vector-storage.ts";
+import type { Chunk, ContentType } from "./types.ts";
+import { chunkFromDict } from "./types.ts";
+
+export function resolveCacheFolder(): string {
+  const name = "miru";
+  const home = process.env.HOME ?? process.env.USERPROFILE ?? "";
+  let base: string;
+  if (process.platform === "win32") {
+    base = process.env.LOCALAPPDATA ?? process.env.APPDATA ?? join(home, "AppData", "Local");
+    base = join(base, name, "Cache");
+  } else if (process.platform === "darwin") {
+    base = join(home, "Library", "Caches", name);
+  } else {
+    base = process.env.XDG_CACHE_HOME
+      ? join(process.env.XDG_CACHE_HOME, name)
+      : join(home, ".cache", name);
+  }
+  return base;
+}
+
+export function findIndexCachePath(path: string): string {
+  const normalized = resolve(path);
+  const subdir = createHash("sha256").update(normalized, "utf-8").digest("hex");
+  return join(resolveCacheFolder(), subdir, "index");
+}
+
+function metadataMatches(
+  metadata: Record<string, unknown>,
+  embeddingModel: string,
+  content: ContentType[],
+): boolean {
+  try {
+    const stored = metadata.content_type as ContentType[];
+    const model = metadata.embedding_model as string;
+    if (model !== embeddingModel) {
+      return false;
+    }
+    const expectedDims = resolveEmbeddingDimensions(embeddingModel);
+    const storedDims = metadata.embedding_dimensions as number | undefined;
+    if (expectedDims != null && storedDims !== expectedDims) {
+      return false;
+    }
+    if (semanticStorageFromMetadata(metadata) !== resolveSemanticStorage()) {
+      return false;
+    }
+    const a = new Set(stored);
+    const b = new Set(content);
+    if (a.size !== b.size) {
+      return false;
+    }
+    for (const c of a) {
+      if (!b.has(c)) {
+        return false;
+      }
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export async function getValidatedCache(
+  path: string,
+  embeddingModel: string | undefined,
+  content: ContentType[],
+): Promise<string | null> {
+  const indexPath = findIndexCachePath(path);
+  const paths = persistencePaths(indexPath);
+  const missing = await pathsExist(paths);
+  if (missing.length > 0) {
+    return null;
+  }
+
+  const model = embeddingModel ?? resolveEmbeddingModel();
+  const metadata = JSON.parse(await Bun.file(paths.metadata).text()) as Record<string, unknown>;
+  if (!metadataMatches(metadata, model, content)) {
+    return null;
+  }
+
+  const semanticMeta = JSON.parse(await Bun.file(`${paths.semanticIndex}/meta.json`).text()) as {
+    dimensions: number;
+  };
+  const expectedDims = resolveEmbeddingDimensions(model);
+  if (expectedDims != null && semanticMeta.dimensions !== expectedDims) {
+    return null;
+  }
+  if (!(await semanticIndexMatchesStorage(paths.semanticIndex))) {
+    return null;
+  }
+
+  const rootPath = metadata.root_path as string | null;
+  if (rootPath) {
+    const filePaths = metadata.file_paths as string[];
+    try {
+      await Bun.file(rootPath).stat();
+      for (const rel of filePaths) {
+        try {
+          await Bun.file(join(rootPath, rel)).stat();
+        } catch {
+          return null;
+        }
+      }
+    } catch {
+      return null;
+    }
+  }
+
+  return indexPath;
+}
+
+export async function loadCachedIndex(indexPath: string): Promise<{
+  bm25: Awaited<ReturnType<typeof loadBm25>>;
+  semantic: Awaited<ReturnType<typeof loadSemantic>>;
+  chunks: Chunk[];
+  metadata: Record<string, unknown>;
+}> {
+  const paths = persistencePaths(indexPath);
+  const bm25 = await loadBm25(paths.bm25Index);
+  const semantic = await loadSemantic(paths.semanticIndex);
+  const chunkData = JSON.parse(await Bun.file(paths.chunks).text()) as Record<string, unknown>[];
+  const chunks = chunkData.map(chunkFromDict);
+  const metadata = JSON.parse(await Bun.file(paths.metadata).text()) as Record<string, unknown>;
+  return { bm25, semantic, chunks, metadata };
+}
+
+export async function clearCache(path: string): Promise<void> {
+  const indexPath = findIndexCachePath(path);
+  await rm(indexPath, { recursive: true, force: true });
+}

package/src/chunking/chunking.ts

@@ -0,0 +1,36 @@
+import type { Chunk } from "../types.ts";
+import { chunkLines } from "./lines.ts";
+import { chunkStructural } from "./structural.ts";
+
+/** Match upstream hybrid-search chunking target length. */
+const DESIRED_CHUNK_LENGTH_CHARS = 1500;
+
+export function chunkSource(source: string, filePath: string, language: string | null): Chunk[] {
+  if (!source.trim()) {
+    return [];
+  }
+
+  const boundaries =
+    chunkStructural(source, language, DESIRED_CHUNK_LENGTH_CHARS) ??
+    chunkLines(source, DESIRED_CHUNK_LENGTH_CHARS);
+  const chunks: Chunk[] = [];
+  const prefixNewlineCounts = new Uint32Array(source.length + 1);
+  for (let i = 0; i < source.length; i++) {
+    const prevCount = prefixNewlineCounts[i] ?? 0;
+    prefixNewlineCounts[i + 1] = prevCount + (source[i] === "\n" ? 1 : 0);
+  }
+
+  for (const boundary of boundaries) {
+    const endIndex = Math.max(boundary.end - 1, boundary.start);
+    const text = source.slice(boundary.start, endIndex + 1);
+    chunks.push({
+      content: text,
+      file_path: filePath,
+      start_line: (prefixNewlineCounts[boundary.start] ?? 0) + 1,
+      end_line: (prefixNewlineCounts[endIndex] ?? 0) + 1,
+      language,
+    });
+  }
+
+  return chunks;
+}

package/src/chunking/lines.ts

@@ -0,0 +1,79 @@
+export interface ChunkBoundary {
+  start: number;
+  end: number;
+}
+
+export interface LineGroup extends ChunkBoundary {
+  text: string;
+}
+
+export function mergeAdjacentChunks(
+  chunks: ChunkBoundary[],
+  desiredLength: number,
+): ChunkBoundary[] {
+  if (chunks.length === 0) {
+    return [];
+  }
+
+  const first = chunks[0];
+  if (!first) {
+    return [];
+  }
+
+  const merged: ChunkBoundary[] = [];
+  let currentStart = first.start;
+  let currentEnd = first.end;
+  let currentLength = currentEnd - currentStart;
+
+  for (const group of chunks.slice(1)) {
+    const start = group.start;
+    const end = group.end;
+    const length = end - start;
+
+    if (currentLength + length > desiredLength) {
+      merged.push({ start: currentStart, end: currentEnd });
+      currentStart = start;
+      currentEnd = end;
+      currentLength = length;
+      continue;
+    }
+
+    currentEnd = end;
+    currentLength += length;
+  }
+
+  merged.push({ start: currentStart, end: currentEnd });
+  return merged;
+}
+
+export function splitLinesKeepEnds(source: string): LineGroup[] {
+  const groups: LineGroup[] = [];
+  const re = /.*(?:\r\n|\n|\r|$)/g;
+  let match = re.exec(source);
+  while (match !== null) {
+    const text = match[0] ?? "";
+    if (text.length === 0) {
+      break;
+    }
+    const start = match.index;
+    const end = start + text.length;
+    groups.push({ start, end, text });
+    if (re.lastIndex >= source.length) {
+      break;
+    }
+    match = re.exec(source);
+  }
+  return groups;
+}
+
+export function chunkLines(source: string, desiredLength: number): ChunkBoundary[] {
+  if (!source.trim()) {
+    return [];
+  }
+
+  const lineGroups = splitLinesKeepEnds(source).map(({ start, end }) => ({
+    start,
+    end,
+  }));
+  return mergeAdjacentChunks(lineGroups, desiredLength);
+}

package/src/chunking/structural.ts

@@ -0,0 +1,244 @@
+import {
+  type ChunkBoundary,
+  type LineGroup,
+  mergeAdjacentChunks,
+  splitLinesKeepEnds,
+} from "./lines.ts";
+
+const SUPPORTED_STRUCTURAL_LANGUAGES = new Set(["python", "go", "typescript", "javascript"]);
+
+export function chunkStructural(
+  source: string,
+  language: string | null,
+  desiredLength: number,
+): ChunkBoundary[] | null {
+  if (!language || !SUPPORTED_STRUCTURAL_LANGUAGES.has(language)) {
+    return null;
+  }
+
+  const lines = splitLinesKeepEnds(source);
+  if (lines.length === 0) {
+    return [];
+  }
+
+  let units: ChunkBoundary[];
+  if (language === "python") {
+    units = pythonUnits(lines);
+  } else {
+    units = braceUnits(lines, language);
+  }
+
+  if (units.length === 0) {
+    return null;
+  }
+  return mergeAdjacentChunks(units, desiredLength);
+}
+
+function lineIndent(text: string): number {
+  let i = 0;
+  while (i < text.length && (text[i] === " " || text[i] === "\t")) {
+    i++;
+  }
+  return i;
+}
+
+function stripLine(text: string): string {
+  return text.replace(/[\r\n]+$/, "");
+}
+
+function _isBlank(text: string): boolean {
+  return stripLine(text).trim().length === 0;
+}
+
+function _isCommentLine(text: string, language: string): boolean {
+  const trimmed = stripLine(text).trim();
+  if (language === "python") {
+    return trimmed.startsWith("#");
+  }
+  return trimmed.startsWith("//");
+}
+
+function pythonUnits(lines: LineGroup[]): ChunkBoundary[] {
+  const units: ChunkBoundary[] = [];
+  let firstDeclStart: number | null = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const raw = stripLine(lines[i]?.text);
+    const trimmed = raw.trim();
+    if (!trimmed.match(/^(async\s+def|def|class)\b/)) {
+      continue;
+    }
+    if (firstDeclStart === null) {
+      firstDeclStart = lines[i]?.start;
+    }
+
+    const declIndent = lineIndent(raw);
+
+    if (trimmed.startsWith("class ")) {
+      const classEndLine = findPythonBlockEnd(lines, i, declIndent);
+      const methodStarts: number[] = [];
+      for (let j = i + 1; j <= classEndLine; j++) {
+        const innerRaw = stripLine(lines[j]?.text);
+        const innerTrim = innerRaw.trim();
+        if (!innerTrim.match(/^(async\s+def|def)\b/)) {
+          continue;
+        }
+        if (lineIndent(innerRaw) > declIndent) {
+          methodStarts.push(j);
+        }
+      }
+
+      if (methodStarts.length > 0) {
+        const firstMethodStart = methodStarts[0];
+        if (firstMethodStart === undefined) {
+          continue;
+        }
+        const firstMethodEnd = findPythonBlockEnd(
+          lines,
+          firstMethodStart,
+          lineIndent(stripLine(lines[firstMethodStart]?.text)),
+        );
+        units.push({ start: lines[i]?.start, end: lines[firstMethodEnd]?.end });
+        for (let m = 1; m < methodStarts.length; m++) {
+          const methodStart = methodStarts[m];
+          if (methodStart === undefined) {
+            continue;
+          }
+          const methodIndent = lineIndent(stripLine(lines[methodStart]?.text));
+          const methodEnd = findPythonBlockEnd(lines, methodStart, methodIndent);
+          units.push({ start: lines[methodStart]?.start, end: lines[methodEnd]?.end });
+        }
+      } else {
+        units.push({ start: lines[i]?.start, end: lines[classEndLine]?.end });
+      }
+      continue;
+    }
+
+    let startLine = i;
+    while (startLine - 1 >= 0) {
+      const prev = stripLine(lines[startLine - 1]?.text).trim();
+      if (prev.startsWith("@")) {
+        startLine--;
+        continue;
+      }
+      break;
+    }
+
+    const endLine = findPythonBlockEnd(lines, i, declIndent);
+
+    // Keep top-level defs and class blocks; methods are represented by their class block.
+    if (declIndent > 0) {
+      continue;
+    }
+
+    const start = lines[startLine]?.start;
+    const end = lines[endLine]?.end;
+    if (end > start) {
+      units.push({ start, end });
+    }
+  }
+
+  if (firstDeclStart !== null && firstDeclStart > 0) {
+    units.unshift({ start: 0, end: firstDeclStart });
+  }
+
+  return dedupeAndSort(units);
+}
+
+function findPythonBlockEnd(lines: LineGroup[], start: number, indent: number): number {
+  let endLine = lines.length - 1;
+  for (let j = start + 1; j < lines.length; j++) {
+    const nextRaw = stripLine(lines[j]?.text);
+    if (!nextRaw.trim() || nextRaw.trim().startsWith("#")) {
+      continue;
+    }
+    if (lineIndent(nextRaw) <= indent) {
+      endLine = j - 1;
+      break;
+    }
+  }
+  return endLine;
+}
+
+function declPattern(language: string): RegExp {
+  if (language === "go") {
+    return /^\s*(func\b|type\b.*\b(struct|interface)\b)/;
+  }
+  return /^\s*(export\s+)?(async\s+function\b|function\b|class\b|interface\b|type\b|(?:const|let|var)\s+[A-Za-z_$][\w$]*\s*=\s*(?:async\s*)?(?:\([^)]*\)|[A-Za-z_$][\w$]*)(?:\s*:\s*[^=]+)?\s*=>)/;
+}
+
+function braceUnits(lines: LineGroup[], language: string): ChunkBoundary[] {
+  const units: ChunkBoundary[] = [];
+  const full = lines.map((l) => l.text).join("");
+  const re = declPattern(language);
+  let firstDeclStart: number | null = null;
+
+  for (let i = 0; i < lines.length; i++) {
+    const text = stripLine(lines[i]?.text);
+    if (!re.test(text)) {
+      continue;
+    }
+    if (firstDeclStart === null) {
+      firstDeclStart = lines[i]?.start;
+    }
+
+    const startOffset = lines[i]?.start;
+    const openPos = full.indexOf("{", startOffset);
+    if (openPos === -1) {
+      continue;
+    }
+
+    const limit = i + 60 < lines.length ? lines[i + 60]?.end : full.length;
+    if (openPos >= limit) {
+      continue;
+    }
+
+    let depth = 0;
+    let endPos = -1;
+    for (let p = openPos; p < full.length; p++) {
+      const ch = full[p];
+      if (ch === "{") {
+        depth++;
+      } else if (ch === "}") {
+        depth--;
+        if (depth === 0) {
+          endPos = p + 1;
+          break;
+        }
+      }
+    }
+    if (endPos > startOffset) {
+      let extendedEnd = endPos;
+      while (
+        extendedEnd < full.length &&
+        (full[extendedEnd] === ";" || full[extendedEnd] === " " || full[extendedEnd] === "\t")
+      ) {
+        extendedEnd++;
+      }
+      if (extendedEnd < full.length && full[extendedEnd] === "\n") {
+        extendedEnd++;
+      }
+      units.push({ start: startOffset, end: extendedEnd });
+    }
+  }
+
+  if (firstDeclStart !== null && firstDeclStart > 0) {
+    units.unshift({ start: 0, end: firstDeclStart });
+  }
+
+  return dedupeAndSort(units);
+}
+
+function dedupeAndSort(units: ChunkBoundary[]): ChunkBoundary[] {
+  const out: ChunkBoundary[] = [];
+  const seen = new Set<string>();
+  for (const u of units.sort((a, b) => a.start - b.start || a.end - b.end)) {
+    const key = `${u.start}:${u.end}`;
+    if (seen.has(key)) {
+      continue;
+    }
+    seen.add(key);
+    out.push(u);
+  }
+  return out;
+}