@winci/local-rag 0.2.1

package/src/db/types.ts

@@ -0,0 +1,85 @@
+export interface StoredChunk {
+  id: number;
+  fileId: number;
+  chunkIndex: number;
+  snippet: string;
+  entityName: string | null;
+  chunkType: string | null;
+  startLine: number | null;
+  endLine: number | null;
+}
+
+export interface StoredFile {
+  id: number;
+  path: string;
+  hash: string;
+  indexedAt: string;
+}
+
+export interface SearchResult {
+  path: string;
+  score: number;
+  snippet: string;
+  chunkIndex: number;
+  entityName: string | null;
+  chunkType: string | null;
+}
+
+export interface ChunkSearchResult {
+  path: string;
+  score: number;
+  content: string;
+  chunkIndex: number;
+  entityName: string | null;
+  chunkType: string | null;
+  startLine: number | null;
+  endLine: number | null;
+}
+
+export interface UsageResult {
+  path: string;
+  line: number | null;
+  snippet: string;
+}
+
+export interface AnnotationRow {
+  id: number;
+  path: string;
+  symbolName: string | null;
+  note: string;
+  author: string | null;
+  createdAt: string;
+  updatedAt: string;
+}
+
+export interface SymbolResult {
+  path: string;
+  symbolName: string;
+  symbolType: string;
+  snippet: string | null;
+  chunkIndex: number | null;
+}
+
+export interface CheckpointRow {
+  id: number;
+  sessionId: string;
+  turnIndex: number;
+  timestamp: string;
+  type: string;
+  title: string;
+  summary: string;
+  filesInvolved: string[];
+  tags: string[];
+}
+
+export interface ConversationSearchResult {
+  turnId: number;
+  turnIndex: number;
+  sessionId: string;
+  timestamp: string;
+  summary: string;
+  snippet: string;
+  toolsUsed: string[];
+  filesReferenced: string[];
+  score: number;
+}

package/src/embeddings/embed.ts

@@ -0,0 +1,73 @@
+import {
+  env,
+  pipeline,
+  type FeatureExtractionPipeline,
+} from "@huggingface/transformers";
+import { join } from "node:path";
+import { homedir, cpus } from "node:os";
+import { rmSync } from "node:fs";
+
+// Use a stable cache directory so models survive bunx temp dir cleanup
+const CACHE_DIR = join(homedir(), ".cache", "local-rag", "models");
+env.cacheDir = CACHE_DIR;
+
+const MODEL_ID = "Xenova/all-MiniLM-L6-v2";
+const EMBEDDING_DIM = 384;
+
+let extractor: FeatureExtractionPipeline | null = null;
+
+function defaultThreadCount(): number {
+  return Math.max(2, Math.floor(cpus().length / 3));
+}
+
+export async function getEmbedder(threads?: number): Promise<FeatureExtractionPipeline> {
+  if (!extractor) {
+    const numThreads = threads ?? defaultThreadCount();
+    const pipelineOptions = {
+      dtype: "fp32" as const,
+      session_options: {
+        intraOpNumThreads: numThreads,
+        interOpNumThreads: numThreads,
+      },
+    };
+    try {
+      extractor = await pipeline("feature-extraction", MODEL_ID, pipelineOptions);
+    } catch (err) {
+      // If the cached model is corrupted, delete it and retry once
+      const msg = (err as Error).message || "";
+      if (msg.includes("Protobuf parsing failed") || msg.includes("Load model")) {
+        const modelDir = join(CACHE_DIR, ...MODEL_ID.split("/"));
+        rmSync(modelDir, { recursive: true, force: true });
+        extractor = await pipeline("feature-extraction", MODEL_ID, pipelineOptions);
+      } else {
+        throw err;
+      }
+    }
+  }
+  return extractor;
+}
+
+export async function embed(text: string, threads?: number): Promise<Float32Array> {
+  const model = await getEmbedder(threads);
+  const output = await model(text, { pooling: "mean", normalize: true });
+  return new Float32Array(output.data as Float64Array);
+}
+
+export async function embedBatch(texts: string[], threads?: number): Promise<Float32Array[]> {
+  if (texts.length === 0) return [];
+  const model = await getEmbedder(threads);
+  const output = await model(texts, { pooling: "mean", normalize: true });
+  const flat = new Float32Array(output.data as Float64Array);
+  const result: Float32Array[] = [];
+  for (let i = 0; i < texts.length; i++) {
+    result.push(flat.slice(i * EMBEDDING_DIM, (i + 1) * EMBEDDING_DIM));
+  }
+  return result;
+}
+
+/** Reset the singleton — only for testing */
+export function resetEmbedder(): void {
+  extractor = null;
+}
+
+export { EMBEDDING_DIM };

package/src/graph/resolver.ts

@@ -0,0 +1,305 @@
+import { dirname, resolve, relative, basename } from "path";
+import { RagDB } from "../db";
+
+// Extensions to try when resolving relative imports
+const RESOLVE_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx"];
+const INDEX_FILES = RESOLVE_EXTENSIONS.map((ext) => `/index${ext}`);
+
+/**
+ * Resolve all unresolved imports in the database by matching import specifiers
+ * to indexed file paths.
+ */
+export function resolveImports(db: RagDB, projectDir: string): number {
+  const unresolved = db.getUnresolvedImports();
+  const pathToId = buildPathToIdMap(db);
+
+  let resolvedCount = 0;
+
+  for (const imp of unresolved) {
+    // Skip bare/external specifiers (no ./ or ../ prefix)
+    if (!imp.source.startsWith(".") && !imp.source.startsWith("/")) {
+      continue;
+    }
+
+    const importerDir = dirname(imp.filePath);
+    const basePath = resolve(importerDir, imp.source);
+
+    const resolved = tryResolvePath(basePath, pathToId);
+    if (resolved !== null) {
+      db.resolveImport(imp.id, resolved);
+      resolvedCount++;
+    }
+  }
+
+  return resolvedCount;
+}
+
+/**
+ * Resolve imports for a single file (used by watcher after re-indexing).
+ * Accepts optional prebuilt maps to avoid repeated full-table scans
+ * when resolving multiple files in sequence.
+ */
+export function resolveImportsForFile(
+  db: RagDB,
+  fileId: number,
+  projectDir: string,
+  pathToId?: Map<string, number>,
+  idToPath?: Map<number, string>
+): void {
+  if (!pathToId) {
+    pathToId = buildPathToIdMap(db);
+  }
+  if (!idToPath) {
+    idToPath = buildIdToPathMap(pathToId);
+  }
+
+  const imports = db.getImportsForFile(fileId);
+  const filePath = idToPath.get(fileId);
+  if (!filePath) return;
+
+  const importerDir = dirname(filePath);
+
+  for (const imp of imports) {
+    if (imp.resolvedFileId !== null) continue;
+    if (!imp.source.startsWith(".") && !imp.source.startsWith("/")) continue;
+
+    const basePath = resolve(importerDir, imp.source);
+    const resolved = tryResolvePath(basePath, pathToId);
+    if (resolved !== null) {
+      db.resolveImport(imp.id, resolved);
+    }
+  }
+}
+
+/** Build path → fileId and fileId → path lookups from all indexed files. */
+export function buildPathToIdMap(db: RagDB): Map<string, number> {
+  const allFiles = db.getAllFilePaths();
+  const map = new Map<string, number>();
+  for (const f of allFiles) {
+    map.set(f.path, f.id);
+  }
+  return map;
+}
+
+export function buildIdToPathMap(pathToId: Map<string, number>): Map<number, string> {
+  const map = new Map<number, string>();
+  for (const [path, id] of pathToId) {
+    map.set(id, path);
+  }
+  return map;
+}
+
+function tryResolvePath(basePath: string, pathToId: Map<string, number>): number | null {
+  // Exact match
+  if (pathToId.has(basePath)) return pathToId.get(basePath)!;
+
+  // Try adding extensions
+  for (const ext of RESOLVE_EXTENSIONS) {
+    const withExt = basePath + ext;
+    if (pathToId.has(withExt)) return pathToId.get(withExt)!;
+  }
+
+  // Try index files (e.g. ./utils → ./utils/index.ts)
+  for (const idx of INDEX_FILES) {
+    const withIndex = basePath + idx;
+    if (pathToId.has(withIndex)) return pathToId.get(withIndex)!;
+  }
+
+  return null;
+}
+
+export interface GraphOptions {
+  zoom?: "file" | "directory";
+  focus?: string;
+  maxNodes?: number;
+  maxHops?: number;
+  showExternals?: boolean;
+  projectDir: string;
+}
+
+interface GraphNode {
+  id: number;
+  path: string;
+  exports: { name: string; type: string }[];
+}
+
+interface GraphEdge {
+  fromId: number;
+  fromPath: string;
+  toId: number;
+  toPath: string;
+  source: string;
+}
+
+/**
+ * Generate a structured text dependency map optimized for AI agent consumption.
+ * Replaces the old Mermaid format with a more parseable, information-dense output.
+ */
+export function generateProjectMap(
+  db: RagDB,
+  options: GraphOptions
+): string {
+  const {
+    zoom = "file",
+    focus,
+    maxNodes = 50,
+    maxHops = 2,
+    projectDir,
+  } = options;
+
+  let graph: { nodes: GraphNode[]; edges: GraphEdge[] };
+
+  if (focus) {
+    const file = db.getFileByPath(resolve(projectDir, focus));
+    if (file) {
+      graph = db.getSubgraph([file.id], maxHops);
+    } else {
+      graph = { nodes: [], edges: [] };
+    }
+  } else {
+    graph = db.getGraph();
+  }
+
+  if (graph.nodes.length === 0) {
+    return "No files indexed or no dependencies found.";
+  }
+
+  // Auto-switch to directory view if too many nodes
+  const effectiveZoom = graph.nodes.length > maxNodes ? "directory" : zoom;
+
+  if (effectiveZoom === "directory") {
+    return generateDirectoryMap(graph, projectDir);
+  }
+
+  return generateFileMap(graph, projectDir);
+}
+
+
+function generateFileMap(
+  graph: { nodes: GraphNode[]; edges: GraphEdge[] },
+  projectDir: string
+): string {
+  // Build adjacency maps
+  const dependsOn = new Map<number, string[]>();
+  const dependedOnBy = new Map<number, string[]>();
+  const idToRel = new Map<number, string>();
+
+  for (const node of graph.nodes) {
+    const relPath = relative(projectDir, node.path);
+    idToRel.set(node.id, relPath);
+    dependsOn.set(node.id, []);
+    dependedOnBy.set(node.id, []);
+  }
+
+  for (const edge of graph.edges) {
+    const fromRel = idToRel.get(edge.fromId);
+    const toRel = idToRel.get(edge.toId);
+    if (fromRel && toRel) {
+      dependsOn.get(edge.fromId)!.push(toRel);
+      dependedOnBy.get(edge.toId)!.push(fromRel);
+    }
+  }
+
+  // Identify entry points (no importers)
+  const entryPoints: GraphNode[] = [];
+  const otherNodes: GraphNode[] = [];
+
+  for (const node of graph.nodes) {
+    if (dependedOnBy.get(node.id)!.length === 0) {
+      entryPoints.push(node);
+    } else {
+      otherNodes.push(node);
+    }
+  }
+
+  const lines: string[] = [];
+  lines.push(`## Project Map (file-level, ${graph.nodes.length} files)\n`);
+
+  function formatNode(node: GraphNode) {
+    const relPath = idToRel.get(node.id)!;
+    lines.push(`  ${relPath}`);
+
+    if (node.exports.length > 0) {
+      const exps = node.exports
+        .slice(0, 8)
+        .map((e) => `${e.name} (${e.type})`)
+        .join(", ");
+      const suffix = node.exports.length > 8 ? `, +${node.exports.length - 8} more` : "";
+      lines.push(`    exports: ${exps}${suffix}`);
+    }
+
+    const deps = dependsOn.get(node.id)!;
+    if (deps.length > 0) {
+      lines.push(`    depends_on: ${deps.join(", ")}`);
+    }
+
+    const importers = dependedOnBy.get(node.id)!;
+    if (importers.length > 0) {
+      lines.push(`    depended_on_by: ${importers.join(", ")}`);
+    }
+  }
+
+  if (entryPoints.length > 0) {
+    lines.push(`### Entry Points (no importers)`);
+    for (const node of entryPoints) {
+      formatNode(node);
+    }
+    lines.push("");
+  }
+
+  if (otherNodes.length > 0) {
+    lines.push(`### Files`);
+    for (const node of otherNodes) {
+      formatNode(node);
+    }
+  }
+
+  return lines.join("\n");
+}
+
+function generateDirectoryMap(
+  graph: { nodes: GraphNode[]; edges: GraphEdge[] },
+  projectDir: string
+): string {
+  // Group nodes by directory
+  const dirFiles = new Map<string, string[]>();
+  const nodeToDir = new Map<number, string>();
+
+  for (const node of graph.nodes) {
+    const relPath = relative(projectDir, node.path);
+    const dir = dirname(relPath) || ".";
+    nodeToDir.set(node.id, dir);
+    if (!dirFiles.has(dir)) dirFiles.set(dir, []);
+    dirFiles.get(dir)!.push(basename(relPath));
+  }
+
+  // Directory-level edges (deduplicated with count)
+  const dirEdgeCounts = new Map<string, number>();
+  for (const edge of graph.edges) {
+    const fromDir = nodeToDir.get(edge.fromId)!;
+    const toDir = nodeToDir.get(edge.toId)!;
+    if (fromDir !== toDir) {
+      const key = `${fromDir} -> ${toDir}`;
+      dirEdgeCounts.set(key, (dirEdgeCounts.get(key) || 0) + 1);
+    }
+  }
+
+  const lines: string[] = [];
+  lines.push(`## Project Map (directory-level, ${dirFiles.size} directories)\n`);
+
+  lines.push("### Directories");
+  for (const [dir, files] of dirFiles) {
+    lines.push(`  ${dir}/ (${files.length} files)`);
+    lines.push(`    files: ${files.join(", ")}`);
+  }
+
+  if (dirEdgeCounts.size > 0) {
+    lines.push("");
+    lines.push("### Dependencies");
+    for (const [edge, count] of dirEdgeCounts) {
+      lines.push(`  ${edge} (${count} import${count !== 1 ? "s" : ""})`);
+    }
+  }
+
+  return lines.join("\n");
+}