@winci/local-rag 0.2.1

package/src/indexing/indexer.ts

@@ -0,0 +1,263 @@
+import { relative } from "path";
+import { createHash } from "crypto";
+import { readFile } from "fs/promises";
+import { Glob } from "bun";
+import { parseFile } from "./parse";
+import { embedBatch } from "../embeddings/embed";
+import { chunkText, KNOWN_EXTENSIONS, type ChunkImport, type ChunkExport } from "./chunker";
+import { RagDB } from "../db";
+import { type RagConfig } from "../config";
+import { resolveImports } from "../graph/resolver";
+import { log } from "../utils/log";
+import { type EmbeddedChunk } from "../types";
+
+function aggregateGraphData(chunks: { imports?: ChunkImport[]; exports?: ChunkExport[] }[]): {
+  imports: { name: string; source: string }[];
+  exports: { name: string; type: string }[];
+} {
+  const importMap = new Map<string, string>();
+  const exportMap = new Map<string, string>();
+
+  for (const chunk of chunks) {
+    if (chunk.imports) {
+      for (const imp of chunk.imports) {
+        if (!importMap.has(imp.source)) {
+          importMap.set(imp.source, imp.name);
+        }
+      }
+    }
+    if (chunk.exports) {
+      for (const exp of chunk.exports) {
+        if (!exportMap.has(exp.name)) {
+          exportMap.set(exp.name, exp.type);
+        }
+      }
+    }
+  }
+
+  return {
+    imports: Array.from(importMap, ([source, name]) => ({ name, source })),
+    exports: Array.from(exportMap, ([name, type]) => ({ name, type })),
+  };
+}
+
+export interface IndexResult {
+  indexed: number;
+  skipped: number;
+  pruned: number;
+  errors: string[];
+}
+
+async function fileHash(filePath: string): Promise<string> {
+  const content = await readFile(filePath);
+  return createHash("sha256").update(content).digest("hex");
+}
+
+function matchesAny(filePath: string, globs: Glob[]): boolean {
+  return globs.some((g) => g.match(filePath));
+}
+
+async function collectFiles(
+  directory: string,
+  config: RagConfig,
+  onWarning?: (msg: string) => void
+): Promise<string[]> {
+  const excludeGlobs = config.exclude.map((pat) => new Glob(pat));
+
+  async function scanPattern(pattern: string): Promise<string[]> {
+    const files: string[] = [];
+    const glob = new Glob(pattern);
+    try {
+      for await (const file of glob.scan({ cwd: directory, absolute: true })) {
+        const rel = relative(directory, file);
+        if (!matchesAny(rel, excludeGlobs)) {
+          files.push(file);
+        }
+      }
+    } catch (err: any) {
+      if (err.code === "EPERM" || err.code === "EACCES") {
+        onWarning?.(`Skipping inaccessible path (${err.code}): ${err.path ?? pattern}`);
+      } else {
+        throw err;
+      }
+    }
+    return files;
+  }
+
+  const results = await Promise.all(config.include.map(scanPattern));
+  return [...new Set(results.flat())];
+}
+
+interface ProcessFileOptions {
+  config: RagConfig;
+  /** Base directory for relative path display */
+  baseDir?: string;
+  onProgress?: (msg: string, opts?: { transient?: boolean }) => void;
+  signal?: AbortSignal;
+}
+
+/**
+ * Shared file processing pipeline: hash → parse → chunk → embed → write to DB.
+ * Streams DB writes alongside embedding to cap memory at one batch (~50 chunks)
+ * instead of buffering all embeddings.
+ */
+async function processFile(
+  filePath: string,
+  db: RagDB,
+  opts: ProcessFileOptions
+): Promise<"indexed" | "skipped"> {
+  const { config, baseDir, onProgress, signal } = opts;
+  const batchSize = config.indexBatchSize ?? 50;
+
+  const hash = await fileHash(filePath);
+  const existing = db.getFileByPath(filePath);
+
+  if (existing && existing.hash === hash) {
+    return "skipped";
+  }
+
+  const relPath = baseDir ? relative(baseDir, filePath) : filePath;
+  onProgress?.(`Indexing ${relPath}`);
+
+  const parsed = await parseFile(filePath);
+
+  if (!KNOWN_EXTENSIONS.has(parsed.extension)) {
+    onProgress?.(`Skipped (unsupported extension "${parsed.extension}"): ${relPath}`);
+    return "skipped";
+  }
+
+  if (!parsed.content.trim()) {
+    return "skipped";
+  }
+
+  const chunks = await chunkText(
+    parsed.content,
+    parsed.extension,
+    config.chunkSize,
+    config.chunkOverlap,
+    filePath
+  );
+
+  if (chunks.length > 10000) {
+    log.warn(`Large file: ${relPath} produced ${chunks.length} chunks`, "indexer");
+  }
+
+  // Stream: embed each batch and write to DB immediately (caps memory at one batch)
+  const DB_BATCH = 500;
+  const fileId = db.upsertFileStart(filePath, hash);
+  let chunkOffset = 0;
+  let pendingDbChunks: EmbeddedChunk[] = [];
+
+  for (let i = 0; i < chunks.length; i += batchSize) {
+    if (signal?.aborted) break;
+
+    const batch = chunks.slice(i, i + batchSize);
+    const embeddings = await embedBatch(batch.map(c => c.text), config.indexThreads);
+
+    for (let j = 0; j < batch.length; j++) {
+      const chunk = batch[j];
+      const primaryExport = chunk.exports?.[0];
+      pendingDbChunks.push({
+        snippet: chunk.text,
+        embedding: embeddings[j],
+        entityName: primaryExport?.name ?? null,
+        chunkType: primaryExport?.type ?? null,
+        startLine: chunk.startLine ?? null,
+        endLine: chunk.endLine ?? null,
+      });
+    }
+
+    // Flush to DB when we hit DB_BATCH size or on last iteration
+    if (pendingDbChunks.length >= DB_BATCH || i + batchSize >= chunks.length) {
+      if (signal?.aborted) break;
+      db.insertChunkBatch(fileId, pendingDbChunks, chunkOffset);
+      onProgress?.(`Writing ${Math.min(chunkOffset + pendingDbChunks.length, chunks.length)}/${chunks.length} chunks for ${relPath}`, { transient: true });
+      chunkOffset += pendingDbChunks.length;
+      pendingDbChunks = [];
+      await Bun.sleep(0);
+    }
+  }
+
+  if (signal?.aborted) return "skipped";
+
+  // Store graph metadata
+  const graphData = aggregateGraphData(chunks);
+  db.upsertFileGraph(fileId, graphData.imports, graphData.exports);
+
+  onProgress?.(`Indexed: ${relPath} (${chunks.length} chunks)`);
+  return "indexed";
+}
+
+/**
+ * Index a single file. Returns true if the file was re-indexed, false if skipped.
+ */
+export async function indexFile(
+  filePath: string,
+  db: RagDB,
+  config: RagConfig
+): Promise<"indexed" | "skipped" | "error"> {
+  try {
+    return await processFile(filePath, db, { config });
+  } catch (err) {
+    log.warn(`Failed to index ${filePath}: ${err instanceof Error ? err.message : err}`, "indexFile");
+    return "error";
+  }
+}
+
+export async function indexDirectory(
+  directory: string,
+  db: RagDB,
+  config: RagConfig,
+  onProgress?: (msg: string, opts?: { transient?: boolean }) => void,
+  signal?: AbortSignal
+): Promise<IndexResult> {
+  const result: IndexResult = { indexed: 0, skipped: 0, pruned: 0, errors: [] };
+
+  if (signal?.aborted) return result;
+
+  const matchedFiles = await collectFiles(directory, config, onProgress);
+
+  onProgress?.(`Found ${matchedFiles.length} files to index`);
+
+  for (const filePath of matchedFiles) {
+    if (signal?.aborted) break;
+
+    try {
+      const status = await processFile(filePath, db, {
+        config,
+        baseDir: directory,
+        onProgress,
+        signal,
+      });
+
+      if (status === "indexed") {
+        result.indexed++;
+      } else {
+        result.skipped++;
+      }
+    } catch (err) {
+      const msg = `Error indexing ${filePath}: ${err instanceof Error ? err.message : err}`;
+      result.errors.push(msg);
+      onProgress?.(msg);
+    }
+  }
+
+  if (signal?.aborted) return result;
+
+  // Prune files that no longer exist
+  const existingPaths = new Set(matchedFiles);
+  result.pruned = db.pruneDeleted(existingPaths);
+  if (result.pruned > 0) {
+    onProgress?.(`Pruned ${result.pruned} deleted files from index`);
+  }
+
+  // Resolve import paths across all files
+  if (result.indexed > 0) {
+    const resolved = resolveImports(db, directory);
+    if (resolved > 0) {
+      onProgress?.(`Resolved ${resolved} import paths`);
+    }
+  }
+
+  return result;
+}

package/src/indexing/parse.ts

@@ -0,0 +1,99 @@
+import matter from "gray-matter";
+import { readFile } from "fs/promises";
+import { extname } from "path";
+
+export interface ParsedFile {
+  path: string;
+  content: string;
+  frontmatter: Record<string, unknown> | null;
+  extension: string;
+}
+
+const MARKDOWN_EXTENSIONS = new Set([".md", ".mdx", ".markdown"]);
+
+// Files with no real extension matched by exact basename.
+const EXACT_NAME_MAP = new Map<string, string>([
+  ["makefile", ".makefile"],
+  ["gnumakefile", ".makefile"],
+  ["vagrantfile", ".vagrantfile"],
+  ["gemfile", ".gemfile"],
+  ["rakefile", ".rakefile"],
+  ["brewfile", ".brewfile"],
+  ["procfile", ".procfile"],
+]);
+
+// Files whose basename starts with a known prefix (e.g. Dockerfile.dev,
+// Jenkinsfile.staging). Each entry is [lowerPrefix, virtualExtension].
+const PREFIX_NAME_MAP: [string, string][] = [
+  ["dockerfile", ".dockerfile"],
+  ["jenkinsfile", ".jenkinsfile"],
+];
+
+function resolveExtension(rawExt: string, basename: string): string {
+  if (rawExt) {
+    // Even with an extension, check prefixes: Dockerfile.dev should win.
+    for (const [prefix, virtualExt] of PREFIX_NAME_MAP) {
+      if (basename === prefix || basename.startsWith(prefix + ".")) {
+        return virtualExt;
+      }
+    }
+    return rawExt;
+  }
+  // No extension — check exact names first, then prefixes.
+  if (EXACT_NAME_MAP.has(basename)) return EXACT_NAME_MAP.get(basename)!;
+  for (const [prefix, virtualExt] of PREFIX_NAME_MAP) {
+    if (basename === prefix || basename.startsWith(prefix + ".")) {
+      return virtualExt;
+    }
+  }
+  return rawExt;
+}
+
+export async function parseFile(filePath: string): Promise<ParsedFile> {
+  const raw = await readFile(filePath, "utf-8");
+  const rawExt = extname(filePath).toLowerCase();
+  const basename = filePath.split("/").pop()?.toLowerCase() ?? "";
+  const ext = resolveExtension(rawExt, basename);
+
+  if (MARKDOWN_EXTENSIONS.has(ext)) {
+    const { data, content } = matter(raw);
+    const hasFrontmatter = Object.keys(data).length > 0;
+    return {
+      path: filePath,
+      content: hasFrontmatter
+        ? buildWeightedText(data, content)
+        : content.trim(),
+      frontmatter: hasFrontmatter ? data : null,
+      extension: ext,
+    };
+  }
+
+  return {
+    path: filePath,
+    content: raw.trim(),
+    frontmatter: null,
+    extension: ext,
+  };
+}
+
+function buildWeightedText(
+  frontmatter: Record<string, unknown>,
+  body: string
+): string {
+  const parts: string[] = [];
+
+  if (frontmatter.name) parts.push(`${frontmatter.name}`);
+  if (frontmatter.description)
+    parts.push(`description: ${frontmatter.description}`);
+  if (frontmatter.type) parts.push(`type: ${frontmatter.type}`);
+  if (frontmatter.tags) {
+    const tags = Array.isArray(frontmatter.tags)
+      ? frontmatter.tags.join(", ")
+      : frontmatter.tags;
+    parts.push(`tags: ${tags}`);
+  }
+
+  if (body.trim()) parts.push(body.trim());
+
+  return parts.join("\n");
+}

package/src/indexing/watcher.ts

@@ -0,0 +1,84 @@
+import { watch } from "fs";
+import { resolve, relative } from "path";
+import { existsSync } from "fs";
+import { Glob } from "bun";
+import { indexFile } from "./indexer";
+import { type RagConfig } from "../config";
+import { type RagDB } from "../db";
+import { resolveImportsForFile, buildPathToIdMap, buildIdToPathMap } from "../graph/resolver";
+
+const DEBOUNCE_MS = 2000;
+
+export interface Watcher {
+  close(): void;
+}
+
+export function startWatcher(
+  directory: string,
+  db: RagDB,
+  config: RagConfig,
+  onEvent?: (msg: string) => void
+): Watcher {
+  const pending = new Map<string, NodeJS.Timeout>();
+
+  // Pre-compile globs once instead of per-event
+  const excludeGlobs = config.exclude.map((pat) => new Glob(pat));
+  const includeGlobs = config.include.map((pat) => new Glob(pat));
+
+  function matchesAny(filePath: string, globs: Glob[]): boolean {
+    return globs.some((g) => g.match(filePath));
+  }
+
+  const fsWatcher = watch(directory, { recursive: true }, (_event, filename) => {
+    if (!filename) return;
+
+    const rel = filename.toString();
+
+    if (matchesAny(rel, excludeGlobs)) return;
+    if (!matchesAny(rel, includeGlobs)) return;
+
+    const absPath = resolve(directory, rel);
+
+    const existing = pending.get(absPath);
+    if (existing) clearTimeout(existing);
+
+    pending.set(
+      absPath,
+      setTimeout(async () => {
+        pending.delete(absPath);
+
+        if (!existsSync(absPath)) {
+          const removed = db.removeFile(absPath);
+          if (removed) {
+            onEvent?.(`Removed deleted file: ${rel}`);
+          }
+          return;
+        }
+
+        const result = await indexFile(absPath, db, config);
+        if (result === "indexed") {
+          const file = db.getFileByPath(absPath);
+          if (file) {
+            // Build lookups once and reuse for all resolve calls
+            const pathToId = buildPathToIdMap(db);
+            const idToPath = buildIdToPathMap(pathToId);
+            resolveImportsForFile(db, file.id, directory, pathToId, idToPath);
+            for (const importerId of db.getImportersOf(file.id)) {
+              resolveImportsForFile(db, importerId, directory, pathToId, idToPath);
+            }
+          }
+          onEvent?.(`Re-indexed: ${rel}`);
+        }
+      }, DEBOUNCE_MS)
+    );
+  });
+
+  onEvent?.(`Watching ${directory} for changes`);
+  return {
+    close() {
+      for (const timer of pending.values()) clearTimeout(timer);
+      pending.clear();
+      fsWatcher.close();
+    },
+  };
+}

package/src/main.ts

@@ -0,0 +1,8 @@
+#!/usr/bin/env bun
+
+import { main } from "./cli";
+
+main().catch((err) => {
+  console.error(err.message);
+  process.exit(1);
+});

package/src/search/benchmark.ts

@@ -0,0 +1,139 @@
+import { readFile } from "fs/promises";
+import { resolve } from "path";
+import { RagDB } from "../db";
+import { search, type DedupedResult } from "./hybrid";
+import { loadConfig } from "../config";
+
+export interface BenchmarkQuery {
+  query: string;
+  expected: string[]; // file paths (relative or absolute)
+}
+
+export interface BenchmarkResult {
+  query: string;
+  expected: string[];
+  results: { path: string; score: number }[];
+  recall: number; // fraction of expected files found in top-K
+  reciprocalRank: number; // 1/rank of first expected file (0 if none found)
+  hit: boolean; // at least one expected file found
+}
+
+export interface BenchmarkSummary {
+  total: number;
+  recallAtK: number; // average recall across queries
+  mrr: number; // mean reciprocal rank
+  zeroMissRate: number; // fraction of queries that missed all expected files
+  results: BenchmarkResult[];
+}
+
+export async function loadBenchmarkQueries(path: string): Promise<BenchmarkQuery[]> {
+  const raw = await readFile(path, "utf-8");
+  const parsed = JSON.parse(raw);
+
+  if (!Array.isArray(parsed)) {
+    throw new Error("Benchmark file must be a JSON array of { query, expected } objects");
+  }
+
+  for (const entry of parsed) {
+    if (!entry.query || !Array.isArray(entry.expected) || entry.expected.length === 0) {
+      throw new Error(`Invalid benchmark entry: ${JSON.stringify(entry)}. Each entry needs "query" (string) and "expected" (string[])`);
+    }
+  }
+
+  return parsed;
+}
+
+function normalizePath(p: string, projectDir: string): string {
+  // If already absolute, return as-is; otherwise resolve relative to project
+  if (p.startsWith("/")) return p;
+  return resolve(projectDir, p);
+}
+
+export async function runBenchmark(
+  queries: BenchmarkQuery[],
+  db: RagDB,
+  projectDir: string,
+  topK: number = 5,
+  hybridWeight?: number
+): Promise<BenchmarkSummary> {
+  const config = await loadConfig(projectDir);
+  const weight = hybridWeight ?? config.hybridWeight;
+
+  const results: BenchmarkResult[] = [];
+
+  for (const q of queries) {
+    const searchResults = await search(q.query, db, topK, 0, weight);
+
+    const resultPaths = searchResults.map((r) => r.path);
+    const expectedNormalized = q.expected.map((p) => normalizePath(p, projectDir));
+
+    // Recall: fraction of expected files found in results
+    const found = expectedNormalized.filter((e) =>
+      resultPaths.some((r) => r === e || r.endsWith(e) || e.endsWith(r))
+    );
+    const recall = found.length / expectedNormalized.length;
+
+    // Reciprocal rank: 1/rank of first expected file in results
+    let reciprocalRank = 0;
+    for (let i = 0; i < resultPaths.length; i++) {
+      const matchesExpected = expectedNormalized.some(
+        (e) => resultPaths[i] === e || resultPaths[i].endsWith(e) || e.endsWith(resultPaths[i])
+      );
+      if (matchesExpected) {
+        reciprocalRank = 1 / (i + 1);
+        break;
+      }
+    }
+
+    results.push({
+      query: q.query,
+      expected: q.expected,
+      results: searchResults.map((r) => ({ path: r.path, score: r.score })),
+      recall,
+      reciprocalRank,
+      hit: found.length > 0,
+    });
+  }
+
+  const total = results.length;
+  const recallAtK = total > 0 ? results.reduce((s, r) => s + r.recall, 0) / total : 0;
+  const mrr = total > 0 ? results.reduce((s, r) => s + r.reciprocalRank, 0) / total : 0;
+  const misses = results.filter((r) => !r.hit).length;
+  const zeroMissRate = total > 0 ? misses / total : 0;
+
+  return { total, recallAtK, mrr, zeroMissRate, results };
+}
+
+export function formatBenchmarkReport(summary: BenchmarkSummary, topK: number = 5): string {
+  const lines: string[] = [];
+
+  lines.push(`Benchmark results (${summary.total} queries, top-${topK}):`);
+  lines.push(`  Recall@${topK}:      ${(summary.recallAtK * 100).toFixed(1)}%`);
+  lines.push(`  MRR:            ${summary.mrr.toFixed(3)}`);
+  lines.push(`  Zero-miss rate: ${(summary.zeroMissRate * 100).toFixed(1)}% (${summary.results.filter((r) => !r.hit).length} queries)`);
+
+  // Show failures
+  const failures = summary.results.filter((r) => !r.hit);
+  if (failures.length > 0) {
+    lines.push("\nMissed queries (no expected file in results):");
+    for (const f of failures) {
+      lines.push(`  "${f.query}"`);
+      lines.push(`    expected: ${f.expected.join(", ")}`);
+      const got = f.results.length > 0
+        ? f.results.map((r) => r.path).join(", ")
+        : "(no results)";
+      lines.push(`    got:      ${got}`);
+    }
+  }
+
+  // Show partial hits (recall < 1 but > 0)
+  const partials = summary.results.filter((r) => r.hit && r.recall < 1);
+  if (partials.length > 0) {
+    lines.push("\nPartial matches (some expected files missing):");
+    for (const p of partials) {
+      lines.push(`  "${p.query}" — recall: ${(p.recall * 100).toFixed(0)}%`);
+    }
+  }
+
+  return lines.join("\n");
+}