@dreb/semantic-search 1.18.0

package/.claude-plugin/plugin.json

@@ -0,0 +1,17 @@
+{
+  "name": "semantic-search",
+  "description": "Semantic codebase search — natural language queries over code and docs using embeddings, tree-sitter parsing, and POEM multi-signal ranking",
+  "version": "1.17.1",
+  "author": {
+    "name": "Drew Brereton"
+  },
+  "repository": "https://github.com/aebrer/dreb",
+  "license": "MIT",
+  "keywords": [
+    "semantic-search",
+    "codebase",
+    "embeddings",
+    "mcp",
+    "poem"
+  ]
+}

package/.mcp.json

@@ -0,0 +1,8 @@
+{
+  "mcpServers": {
+    "semantic-search": {
+      "command": "node",
+      "args": ["${CLAUDE_PLUGIN_ROOT}/bin/server.js"]
+    }
+  }
+}

package/README.md

@@ -0,0 +1,97 @@
+# @dreb/semantic-search
+
+Semantic codebase search engine with embedding-based ranking and an MCP server. Extracts and indexes code using tree-sitter for AST-aware chunking and a transformer embedding model ([all-MiniLM-L6-v2](https://huggingface.co/Xenova/all-MiniLM-L6-v2)), then ranks results using 6-signal fusion via POEM.
+
+## Requirements
+
+- **Node.js 22+** — uses the built-in `node:sqlite` module
+
+## Installation
+
+```bash
+npm install @dreb/semantic-search
+```
+
+## Claude Code Plugin
+
+The package ships as a Claude Code plugin. Install via a marketplace that includes it as an npm source, or register the MCP server directly:
+
+```bash
+claude mcp add --transport stdio semantic-search -- npx @dreb/semantic-search semantic-search-mcp
+```
+
+For local development:
+
+```bash
+claude --plugin-dir /path/to/packages/semantic-search
+```
+
+## MCP Server
+
+The package exposes a `search` tool over the Model Context Protocol (stdio transport). The tool accepts:
+
+| Parameter    | Required | Description                                      |
+| ------------ | -------- | ------------------------------------------------ |
+| `query`      | yes      | Natural language, identifier, or path query       |
+| `projectDir` | yes      | Absolute path to the project directory to search  |
+| `path`       | no       | Restrict search to files under this path          |
+| `limit`      | no       | Maximum results to return (default: 20)           |
+| `rebuild`    | no       | Force a clean index rebuild (default: false)      |
+
+Start the server standalone:
+
+```bash
+npx @dreb/semantic-search semantic-search-mcp
+```
+
+## How Ranking Works
+
+Results are ranked by fusing 6 independent signals using **POEM** (Pareto-Optimal Embedded Modeling) weights that vary per query type:
+
+| Signal                | Description                                                    |
+| --------------------- | -------------------------------------------------------------- |
+| **BM25**              | Keyword matching via FTS5 full-text search                     |
+| **Cosine similarity** | Embedding-based semantic similarity using all-MiniLM-L6-v2    |
+| **Path match**        | Query terms appearing in the file path                         |
+| **Symbol match**      | Query terms matching function, class, or type names            |
+| **Import graph**      | Proximity to high-scoring files in the import/dependency graph |
+| **Git recency**       | Recently modified files ranked higher                          |
+
+Queries are automatically classified as _identifier_, _natural language_, or _path_ queries, and each type applies different POEM column weights. POEM constructs a Pareto front over all signal dimensions and assigns ranks based on dominance depth — no manual weight tuning required. See [Pareto-Optimal Embedded Modeling](https://iopscience.iop.org/article/10.1088/2632-2153/ab891b) for the theoretical foundation.
+
+## Library API
+
+```typescript
+import { SearchEngine } from "@dreb/semantic-search";
+
+const engine = new SearchEngine("/path/to/project", {
+  indexDir: "/custom/index/path",         // default: <projectRoot>/.search-index
+  globalMemoryDir: "~/.dreb/memory",      // additional directory to index
+  modelCacheDir: "~/.cache/models",       // default: ~/.cache/semantic-search/models
+  visibleDirs: (root) => [`${root}/.special`], // extra dirs (bypasses .gitignore)
+});
+
+// First call builds the index (10-60s); subsequent calls are fast
+const results = await engine.search("where is auth handled", {
+  limit: 20,
+  pathFilter: "src/",
+  onProgress: (phase, current, total) => console.log(`${phase}: ${current}/${total}`),
+});
+
+const stats = engine.getStats();    // { files, chunks } | null
+await engine.resetIndex();          // delete index, next search rebuilds
+await engine.close();               // dispose resources
+SearchEngine.isAvailable();         // check for node:sqlite
+```
+
+## What Gets Indexed
+
+- **Code** — tree-sitter AST chunks (functions, classes, methods, interfaces, etc.). TypeScript, JavaScript, Python, Go, Rust, Java, C, C++.
+- **Text** — Markdown (by heading), YAML/TOML (by key), JSON, plaintext (by paragraph).
+- **Extra directories** — via `globalMemoryDir` or `visibleDirs`, scanned even if gitignored.
+
+The index is stored in `.search-index/search.db` at the project root (add `.search-index/` to `.gitignore`).
+
+## License
+
+MIT

package/bin/server.js

@@ -0,0 +1,14 @@
+#!/usr/bin/env node
+
+import { resolve } from "node:path";
+import { startServer } from "../dist/mcp-server.js";
+
+// Project directory: CLI arg if provided, otherwise CWD.
+// Claude Code launches MCP servers with CWD set to the project root,
+// so no argument is needed for typical usage.
+const projectDir = resolve(process.argv[2] || ".");
+
+startServer(projectDir).catch((err) => {
+	console.error("Failed to start MCP server:", err);
+	process.exit(1);
+});

package/dist/chunker.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Chunking coordinator for the semantic search subsystem.
+ *
+ * Dispatches to the tree-sitter AST chunker for code files and the
+ * text chunker for non-code files (markdown, YAML, JSON, etc.).
+ */
+import type { Chunk, FileType } from "./types.js";
+/**
+ * Chunk a file's content into semantically meaningful pieces.
+ *
+ * For code files, uses tree-sitter to parse the AST and extract functions,
+ * classes, methods, etc. For text files, uses format-specific splitting rules.
+ *
+ * If tree-sitter parsing fails for a code file, falls back to plaintext chunking.
+ *
+ * @param content - Raw file content
+ * @param filePath - Relative file path (stored in chunk metadata)
+ * @param fileType - Detected file type
+ */
+export declare function chunkFile(content: string, filePath: string, fileType: FileType): Promise<Chunk[]>;
+//# sourceMappingURL=chunker.d.ts.map

package/dist/chunker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAE,QAAQ,EAAoC,MAAM,YAAY,CAAC;AAsBpF;;;;;;;;;;;GAWG;AACH,wBAAsB,SAAS,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAYvG","sourcesContent":["/**\n * Chunking coordinator for the semantic search subsystem.\n *\n * Dispatches to the tree-sitter AST chunker for code files and the\n * text chunker for non-code files (markdown, YAML, JSON, etc.).\n */\n\nimport { chunkTextFile } from \"./text-chunker.js\";\nimport { chunkWithTreeSitter, initTreeSitter } from \"./tree-sitter-chunker.js\";\nimport type { Chunk, FileType, TextFileType, TreeSitterLanguage } from \"./types.js\";\n\n// ============================================================================\n// Language Sets\n// ============================================================================\n\nconst TREE_SITTER_LANGUAGES: Set<string> = new Set([\n\t\"typescript\",\n\t\"tsx\",\n\t\"javascript\",\n\t\"python\",\n\t\"go\",\n\t\"rust\",\n\t\"java\",\n\t\"c\",\n\t\"cpp\",\n]);\n\n// ============================================================================\n// Public API\n// ============================================================================\n\n/**\n * Chunk a file's content into semantically meaningful pieces.\n *\n * For code files, uses tree-sitter to parse the AST and extract functions,\n * classes, methods, etc. For text files, uses format-specific splitting rules.\n *\n * If tree-sitter parsing fails for a code file, falls back to plaintext chunking.\n *\n * @param content - Raw file content\n * @param filePath - Relative file path (stored in chunk metadata)\n * @param fileType - Detected file type\n */\nexport async function chunkFile(content: string, filePath: string, fileType: FileType): Promise<Chunk[]> {\n\tif (TREE_SITTER_LANGUAGES.has(fileType)) {\n\t\ttry {\n\t\t\tawait initTreeSitter();\n\t\t\treturn await chunkWithTreeSitter(content, filePath, fileType as TreeSitterLanguage);\n\t\t} catch {\n\t\t\t// Tree-sitter failed — fall back to plaintext chunking\n\t\t\treturn chunkTextFile(content, filePath, \"plaintext\");\n\t\t}\n\t}\n\n\treturn chunkTextFile(content, filePath, fileType as TextFileType);\n}\n"]}

package/dist/chunker.js

@@ -0,0 +1,51 @@
+/**
+ * Chunking coordinator for the semantic search subsystem.
+ *
+ * Dispatches to the tree-sitter AST chunker for code files and the
+ * text chunker for non-code files (markdown, YAML, JSON, etc.).
+ */
+import { chunkTextFile } from "./text-chunker.js";
+import { chunkWithTreeSitter, initTreeSitter } from "./tree-sitter-chunker.js";
+// ============================================================================
+// Language Sets
+// ============================================================================
+const TREE_SITTER_LANGUAGES = new Set([
+    "typescript",
+    "tsx",
+    "javascript",
+    "python",
+    "go",
+    "rust",
+    "java",
+    "c",
+    "cpp",
+]);
+// ============================================================================
+// Public API
+// ============================================================================
+/**
+ * Chunk a file's content into semantically meaningful pieces.
+ *
+ * For code files, uses tree-sitter to parse the AST and extract functions,
+ * classes, methods, etc. For text files, uses format-specific splitting rules.
+ *
+ * If tree-sitter parsing fails for a code file, falls back to plaintext chunking.
+ *
+ * @param content - Raw file content
+ * @param filePath - Relative file path (stored in chunk metadata)
+ * @param fileType - Detected file type
+ */
+export async function chunkFile(content, filePath, fileType) {
+    if (TREE_SITTER_LANGUAGES.has(fileType)) {
+        try {
+            await initTreeSitter();
+            return await chunkWithTreeSitter(content, filePath, fileType);
+        }
+        catch {
+            // Tree-sitter failed — fall back to plaintext chunking
+            return chunkTextFile(content, filePath, "plaintext");
+        }
+    }
+    return chunkTextFile(content, filePath, fileType);
+}
+//# sourceMappingURL=chunker.js.map

package/dist/chunker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../src/chunker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAG/E,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E,MAAM,qBAAqB,GAAgB,IAAI,GAAG,CAAC;IAClD,YAAY;IACZ,KAAK;IACL,YAAY;IACZ,QAAQ;IACR,IAAI;IACJ,MAAM;IACN,MAAM;IACN,GAAG;IACH,KAAK;CACL,CAAC,CAAC;AAEH,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,QAAgB,EAAE,QAAkB,EAAoB;IACxG,IAAI,qBAAqB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC;YACJ,MAAM,cAAc,EAAE,CAAC;YACvB,OAAO,MAAM,mBAAmB,CAAC,OAAO,EAAE,QAAQ,EAAE,QAA8B,CAAC,CAAC;QACrF,CAAC;QAAC,MAAM,CAAC;YACR,yDAAuD;YACvD,OAAO,aAAa,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC;QACtD,CAAC;IACF,CAAC;IAED,OAAO,aAAa,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAwB,CAAC,CAAC;AAAA,CAClE","sourcesContent":["/**\n * Chunking coordinator for the semantic search subsystem.\n *\n * Dispatches to the tree-sitter AST chunker for code files and the\n * text chunker for non-code files (markdown, YAML, JSON, etc.).\n */\n\nimport { chunkTextFile } from \"./text-chunker.js\";\nimport { chunkWithTreeSitter, initTreeSitter } from \"./tree-sitter-chunker.js\";\nimport type { Chunk, FileType, TextFileType, TreeSitterLanguage } from \"./types.js\";\n\n// ============================================================================\n// Language Sets\n// ============================================================================\n\nconst TREE_SITTER_LANGUAGES: Set<string> = new Set([\n\t\"typescript\",\n\t\"tsx\",\n\t\"javascript\",\n\t\"python\",\n\t\"go\",\n\t\"rust\",\n\t\"java\",\n\t\"c\",\n\t\"cpp\",\n]);\n\n// ============================================================================\n// Public API\n// ============================================================================\n\n/**\n * Chunk a file's content into semantically meaningful pieces.\n *\n * For code files, uses tree-sitter to parse the AST and extract functions,\n * classes, methods, etc. For text files, uses format-specific splitting rules.\n *\n * If tree-sitter parsing fails for a code file, falls back to plaintext chunking.\n *\n * @param content - Raw file content\n * @param filePath - Relative file path (stored in chunk metadata)\n * @param fileType - Detected file type\n */\nexport async function chunkFile(content: string, filePath: string, fileType: FileType): Promise<Chunk[]> {\n\tif (TREE_SITTER_LANGUAGES.has(fileType)) {\n\t\ttry {\n\t\t\tawait initTreeSitter();\n\t\t\treturn await chunkWithTreeSitter(content, filePath, fileType as TreeSitterLanguage);\n\t\t} catch {\n\t\t\t// Tree-sitter failed — fall back to plaintext chunking\n\t\t\treturn chunkTextFile(content, filePath, \"plaintext\");\n\t\t}\n\t}\n\n\treturn chunkTextFile(content, filePath, fileType as TextFileType);\n}\n"]}

package/dist/db.d.ts

@@ -0,0 +1,89 @@
+/**
+ * SQLite database abstraction for the search index.
+ *
+ * Uses `node:sqlite` (built-in Node 22+). Feature-gated — callers must check
+ * availability via `isSqliteAvailable()` before constructing a SearchDatabase.
+ */
+import type { ChunkKind, FileType, IndexedFile, StoredChunk } from "./types.js";
+/** Check whether `node:sqlite` is available in this Node.js runtime. */
+export declare function isSqliteAvailable(): boolean;
+/** Wrapper around `node:sqlite` DatabaseSync for the search index. */
+export declare class SearchDatabase {
+    private db;
+    constructor(dbPath: string);
+    /** Create or migrate the database schema. */
+    private initSchema;
+    /** Insert or update a file record. Returns the file ID. */
+    upsertFile(filePath: string, mtime: number, fileType: FileType): number;
+    /** Get a file by path. */
+    getFile(filePath: string): IndexedFile | null;
+    /** Get all indexed files. */
+    getAllFiles(): IndexedFile[];
+    /** Delete a file and all its chunks/embeddings/symbols (cascading). */
+    deleteFile(fileId: number): void;
+    /** Insert a chunk. Returns the chunk ID. */
+    insertChunk(fileId: number, filePath: string, startLine: number, endLine: number, kind: ChunkKind, name: string | null, content: string, fileType: FileType): number;
+    /** Delete all chunks for a file. */
+    deleteChunksForFile(fileId: number): void;
+    /** Get all chunks. */
+    getAllChunks(): StoredChunk[];
+    /** Get chunks by file ID. */
+    getChunksByFileId(fileId: number): StoredChunk[];
+    /** Get a chunk by ID. */
+    getChunk(chunkId: number): StoredChunk | null;
+    /** Get multiple chunks by IDs. Batches queries to avoid exceeding SQLite's bind variable limit. */
+    getChunksById(chunkIds: number[]): StoredChunk[];
+    private rowToChunk;
+    /** Store an embedding vector for a chunk. */
+    upsertEmbedding(chunkId: number, modelName: string, vector: Float32Array): void;
+    /** Batch insert embeddings. Uses a transaction for performance. */
+    batchUpsertEmbeddings(items: Array<{
+        chunkId: number;
+        modelName: string;
+        vector: Float32Array;
+    }>): void;
+    /** Get the embedding for a chunk. */
+    getEmbedding(chunkId: number, modelName: string): Float32Array | null;
+    /** Get all embeddings for a model. Returns map of chunkId → vector. */
+    getAllEmbeddings(modelName: string): Map<number, Float32Array>;
+    /** Get chunk IDs that have no embedding for a given model. */
+    getChunkIdsWithoutEmbedding(modelName: string): number[];
+    /** Rebuild the FTS5 index (use after bulk operations). */
+    rebuildFts(): void;
+    /**
+     * Search via FTS5 with BM25 ranking.
+     * Returns chunk IDs with their BM25 scores (negated so higher = better).
+     */
+    ftsSearch(query: string, limit: number): Array<{
+        chunkId: number;
+        score: number;
+    }>;
+    /** Record an import edge. */
+    insertImport(sourceFileId: number, targetFilePath: string): void;
+    /** Delete all imports for a source file. */
+    deleteImportsForFile(sourceFileId: number): void;
+    /** Get files imported by a given source file. */
+    getImportsFrom(sourceFileId: number): string[];
+    /** Get file IDs that import a given target path. */
+    getImportersOf(targetFilePath: string): number[];
+    /** Get all import edges. */
+    getAllImports(): Array<{
+        sourceFileId: number;
+        targetFilePath: string;
+    }>;
+    /** Insert a symbol. */
+    insertSymbol(chunkId: number, name: string, kind: string): void;
+    /** Delete symbols for a chunk. */
+    deleteSymbolsForChunk(chunkId: number): void;
+    /** Get all symbols. Returns map of chunkId → symbol names. */
+    getAllSymbols(): Map<number, string[]>;
+    /** Run a function inside a transaction. */
+    transaction<T>(fn: () => T): T;
+    /** Get the total number of chunks. */
+    getChunkCount(): number;
+    /** Get the total number of files. */
+    getFileCount(): number;
+    /** Close the database connection. */
+    close(): void;
+}
+//# sourceMappingURL=db.d.ts.map

package/dist/db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"db.d.ts","sourceRoot":"","sources":["../src/db.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAUhF,wEAAwE;AACxE,wBAAgB,iBAAiB,IAAI,OAAO,CAS3C;AAYD,sEAAsE;AACtE,qBAAa,cAAc;IAC1B,OAAO,CAAC,EAAE,CAAM;IAEhB,YAAY,MAAM,EAAE,MAAM,EAQzB;IAED,6CAA6C;IAC7C,OAAO,CAAC,UAAU;IA6GlB,2DAA2D;IAC3D,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAUtE;IAED,0BAA0B;IAC1B,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAM5C;IAED,6BAA6B;IAC7B,WAAW,IAAI,WAAW,EAAE,CAQ3B;IAED,uEAAuE;IACvE,UAAU,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAE/B;IAMD,4CAA4C;IAC5C,WAAW,CACV,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,SAAS,EACf,IAAI,EAAE,MAAM,GAAG,IAAI,EACnB,OAAO,EAAE,MAAM,EACf,QAAQ,EAAE,QAAQ,GAChB,MAAM,CAOR;IAED,oCAAoC;IACpC,mBAAmB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAExC;IAED,sBAAsB;IACtB,YAAY,IAAI,WAAW,EAAE,CAK5B;IAED,6BAA6B;IAC7B,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,EAAE,CAO/C;IAED,yBAAyB;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,GAAG,IAAI,CAQ5C;IAED,mGAAmG;IACnG,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,WAAW,EAAE,CAoB/C;IAED,OAAO,CAAC,UAAU;IAkBlB,6CAA6C;IAC7C,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,GAAG,IAAI,CAK9E;IAED,mEAAmE;IACnE,qBAAqB,CAAC,KAAK,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,YAAY,CAAA;KAAE,CAAC,GAAG,IAAI,CAQtG;IAED,qCAAqC;IACrC,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAMpE;IAED,yEAAuE;IACvE,gBAAgB,CAAC,SAAS,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,CAO7D;IAED,8DAA8D;IAC9D,2BAA2B,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,CASvD;IAMD,0DAA0D;IAC1D,UAAU,IAAI,IAAI,CAEjB;IAED;;;OAGG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAgBjF;IAMD,6BAA6B;IAC7B,YAAY,CAAC,YAAY,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,CAI/D;IAED,4CAA4C;IAC5C,oBAAoB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAE/C;IAED,iDAAiD;IACjD,cAAc,CAAC,YAAY,EAAE,MAAM,GAAG,MAAM,EAAE,CAG7C;IAED,oDAAoD;IACpD,cAAc,CAAC,cAAc,EAAE,MAAM,GAAG,MAAM,EAAE,CAG/C;IAED,4BAA4B;IAC5B,aAAa,IAAI,KAAK,CAAC;QAAE,YAAY,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAA;KAAE,CAAC,CAGvE;IAMD,uBAAuB;IACvB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAE9D;IAED,kCAAkC;IAClC,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAE3C;IAED,gEAA8D;IAC9D,aAAa,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CASrC;IAMD,2CAA2C;IAC3C,WAAW,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAU7B;IAED,sCAAsC;IACtC,aAAa,IAAI,MAAM,CAGtB;IAED,qCAAqC;IACrC,YAAY,IAAI,MAAM,CAGrB;IAED,qCAAqC;IACrC,KAAK,IAAI,IAAI,CAEZ;CACD","sourcesContent":["/**\n * SQLite database abstraction for the search index.\n *\n * Uses `node:sqlite` (built-in Node 22+). Feature-gated — callers must check\n * availability via `isSqliteAvailable()` before constructing a SearchDatabase.\n */\n\nimport { createRequire } from \"node:module\";\nimport type { ChunkKind, FileType, IndexedFile, StoredChunk } from \"./types.js\";\nimport { unpackVector } from \"./vector-store.js\";\n\n// ============================================================================\n// Availability Check\n// ============================================================================\n\nconst require = createRequire(import.meta.url);\nlet _sqliteAvailable: boolean | null = null;\n\n/** Check whether `node:sqlite` is available in this Node.js runtime. */\nexport function isSqliteAvailable(): boolean {\n\tif (_sqliteAvailable !== null) return _sqliteAvailable;\n\ttry {\n\t\trequire(\"node:sqlite\");\n\t\t_sqliteAvailable = true;\n\t} catch {\n\t\t_sqliteAvailable = false;\n\t}\n\treturn _sqliteAvailable;\n}\n\n// ============================================================================\n// Schema Version\n// ============================================================================\n\nconst SCHEMA_VERSION = 1;\n\n// ============================================================================\n// Database\n// ============================================================================\n\n/** Wrapper around `node:sqlite` DatabaseSync for the search index. */\nexport class SearchDatabase {\n\tprivate db: any; // DatabaseSync from node:sqlite\n\n\tconstructor(dbPath: string) {\n\t\t// Import synchronously — caller must have verified availability\n\t\tconst { DatabaseSync } = require(\"node:sqlite\");\n\t\tthis.db = new DatabaseSync(dbPath);\n\t\tthis.db.exec(\"PRAGMA journal_mode=WAL\");\n\t\tthis.db.exec(\"PRAGMA synchronous=NORMAL\");\n\t\tthis.db.exec(\"PRAGMA foreign_keys=ON\");\n\t\tthis.initSchema();\n\t}\n\n\t/** Create or migrate the database schema. */\n\tprivate initSchema(): void {\n\t\t// Check schema version\n\t\tthis.db.exec(\"CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)\");\n\t\tconst versionRow = this.db.prepare(\"SELECT value FROM meta WHERE key = 'schema_version'\").get();\n\t\tconst currentVersion = versionRow ? Number.parseInt(versionRow.value, 10) : 0;\n\n\t\tif (currentVersion >= SCHEMA_VERSION) return;\n\n\t\t// Files table\n\t\tthis.db.exec(`\n\t\t\tCREATE TABLE IF NOT EXISTS files (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tfile_path TEXT NOT NULL UNIQUE,\n\t\t\t\tmtime REAL NOT NULL,\n\t\t\t\tfile_type TEXT NOT NULL\n\t\t\t)\n\t\t`);\n\n\t\t// Chunks table\n\t\tthis.db.exec(`\n\t\t\tCREATE TABLE IF NOT EXISTS chunks (\n\t\t\t\tid INTEGER PRIMARY KEY AUTOINCREMENT,\n\t\t\t\tfile_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,\n\t\t\t\tfile_path TEXT NOT NULL,\n\t\t\t\tstart_line INTEGER NOT NULL,\n\t\t\t\tend_line INTEGER NOT NULL,\n\t\t\t\tkind TEXT NOT NULL,\n\t\t\t\tname TEXT,\n\t\t\t\tcontent TEXT NOT NULL,\n\t\t\t\tfile_type TEXT NOT NULL\n\t\t\t)\n\t\t`);\n\t\tthis.db.exec(\"CREATE INDEX IF NOT EXISTS idx_chunks_file_id ON chunks(file_id)\");\n\t\tthis.db.exec(\"CREATE INDEX IF NOT EXISTS idx_chunks_file_path ON chunks(file_path)\");\n\n\t\t// FTS5 virtual table (content-synced with chunks)\n\t\tthis.db.exec(`\n\t\t\tCREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(\n\t\t\t\tcontent,\n\t\t\t\tname,\n\t\t\t\tfile_path,\n\t\t\t\tcontent='chunks',\n\t\t\t\tcontent_rowid='id',\n\t\t\t\ttokenize='porter unicode61'\n\t\t\t)\n\t\t`);\n\n\t\t// FTS triggers for incremental updates\n\t\tthis.db.exec(`\n\t\t\tCREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN\n\t\t\t\tINSERT INTO chunks_fts(rowid, content, name, file_path)\n\t\t\t\tVALUES (new.id, new.content, new.name, new.file_path);\n\t\t\tEND\n\t\t`);\n\t\tthis.db.exec(`\n\t\t\tCREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN\n\t\t\t\tINSERT INTO chunks_fts(chunks_fts, rowid, content, name, file_path)\n\t\t\t\tVALUES ('delete', old.id, old.content, old.name, old.file_path);\n\t\t\tEND\n\t\t`);\n\t\tthis.db.exec(`\n\t\t\tCREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN\n\t\t\t\tINSERT INTO chunks_fts(chunks_fts, rowid, content, name, file_path)\n\t\t\t\tVALUES ('delete', old.id, old.content, old.name, old.file_path);\n\t\t\t\tINSERT INTO chunks_fts(rowid, content, name, file_path)\n\t\t\t\tVALUES (new.id, new.content, new.name, new.file_path);\n\t\t\tEND\n\t\t`);\n\n\t\t// Embeddings table (keyed by model name for multi-model support)\n\t\tthis.db.exec(`\n\t\t\tCREATE TABLE IF NOT EXISTS embeddings (\n\t\t\t\tchunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,\n\t\t\t\tmodel_name TEXT NOT NULL,\n\t\t\t\tvector BLOB NOT NULL,\n\t\t\t\tPRIMARY KEY (chunk_id, model_name)\n\t\t\t)\n\t\t`);\n\n\t\t// Imports table (import graph edges between files)\n\t\tthis.db.exec(`\n\t\t\tCREATE TABLE IF NOT EXISTS imports (\n\t\t\t\tsource_file_id INTEGER NOT NULL REFERENCES files(id) ON DELETE CASCADE,\n\t\t\t\ttarget_file_path TEXT NOT NULL,\n\t\t\t\tPRIMARY KEY (source_file_id, target_file_path)\n\t\t\t)\n\t\t`);\n\t\tthis.db.exec(\"CREATE INDEX IF NOT EXISTS idx_imports_target ON imports(target_file_path)\");\n\n\t\t// Symbols table (symbol names extracted from chunks)\n\t\tthis.db.exec(`\n\t\t\tCREATE TABLE IF NOT EXISTS symbols (\n\t\t\t\tchunk_id INTEGER NOT NULL REFERENCES chunks(id) ON DELETE CASCADE,\n\t\t\t\tname TEXT NOT NULL,\n\t\t\t\tkind TEXT NOT NULL\n\t\t\t)\n\t\t`);\n\t\tthis.db.exec(\"CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name)\");\n\n\t\t// Set schema version\n\t\tthis.db\n\t\t\t.prepare(\"INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)\")\n\t\t\t.run(String(SCHEMA_VERSION));\n\t}\n\n\t// ========================================================================\n\t// Files\n\t// ========================================================================\n\n\t/** Insert or update a file record. Returns the file ID. */\n\tupsertFile(filePath: string, mtime: number, fileType: FileType): number {\n\t\tconst existing = this.db.prepare(\"SELECT id FROM files WHERE file_path = ?\").get(filePath);\n\t\tif (existing) {\n\t\t\tthis.db.prepare(\"UPDATE files SET mtime = ?, file_type = ? WHERE id = ?\").run(mtime, fileType, existing.id);\n\t\t\treturn existing.id;\n\t\t}\n\t\tconst result = this.db\n\t\t\t.prepare(\"INSERT INTO files (file_path, mtime, file_type) VALUES (?, ?, ?)\")\n\t\t\t.run(filePath, mtime, fileType);\n\t\treturn Number(result.lastInsertRowid);\n\t}\n\n\t/** Get a file by path. */\n\tgetFile(filePath: string): IndexedFile | null {\n\t\tconst row = this.db\n\t\t\t.prepare(\"SELECT id, file_path, mtime, file_type FROM files WHERE file_path = ?\")\n\t\t\t.get(filePath);\n\t\tif (!row) return null;\n\t\treturn { id: row.id, filePath: row.file_path, mtime: row.mtime, fileType: row.file_type as FileType };\n\t}\n\n\t/** Get all indexed files. */\n\tgetAllFiles(): IndexedFile[] {\n\t\tconst rows = this.db.prepare(\"SELECT id, file_path, mtime, file_type FROM files\").all();\n\t\treturn rows.map((r: any) => ({\n\t\t\tid: r.id,\n\t\t\tfilePath: r.file_path,\n\t\t\tmtime: r.mtime,\n\t\t\tfileType: r.file_type as FileType,\n\t\t}));\n\t}\n\n\t/** Delete a file and all its chunks/embeddings/symbols (cascading). */\n\tdeleteFile(fileId: number): void {\n\t\tthis.db.prepare(\"DELETE FROM files WHERE id = ?\").run(fileId);\n\t}\n\n\t// ========================================================================\n\t// Chunks\n\t// ========================================================================\n\n\t/** Insert a chunk. Returns the chunk ID. */\n\tinsertChunk(\n\t\tfileId: number,\n\t\tfilePath: string,\n\t\tstartLine: number,\n\t\tendLine: number,\n\t\tkind: ChunkKind,\n\t\tname: string | null,\n\t\tcontent: string,\n\t\tfileType: FileType,\n\t): number {\n\t\tconst result = this.db\n\t\t\t.prepare(\n\t\t\t\t\"INSERT INTO chunks (file_id, file_path, start_line, end_line, kind, name, content, file_type) VALUES (?, ?, ?, ?, ?, ?, ?, ?)\",\n\t\t\t)\n\t\t\t.run(fileId, filePath, startLine, endLine, kind, name, content, fileType);\n\t\treturn Number(result.lastInsertRowid);\n\t}\n\n\t/** Delete all chunks for a file. */\n\tdeleteChunksForFile(fileId: number): void {\n\t\tthis.db.prepare(\"DELETE FROM chunks WHERE file_id = ?\").run(fileId);\n\t}\n\n\t/** Get all chunks. */\n\tgetAllChunks(): StoredChunk[] {\n\t\tconst rows = this.db\n\t\t\t.prepare(\"SELECT id, file_id, file_path, start_line, end_line, kind, name, content, file_type FROM chunks\")\n\t\t\t.all();\n\t\treturn rows.map((r: any) => this.rowToChunk(r));\n\t}\n\n\t/** Get chunks by file ID. */\n\tgetChunksByFileId(fileId: number): StoredChunk[] {\n\t\tconst rows = this.db\n\t\t\t.prepare(\n\t\t\t\t\"SELECT id, file_id, file_path, start_line, end_line, kind, name, content, file_type FROM chunks WHERE file_id = ?\",\n\t\t\t)\n\t\t\t.all(fileId);\n\t\treturn rows.map((r: any) => this.rowToChunk(r));\n\t}\n\n\t/** Get a chunk by ID. */\n\tgetChunk(chunkId: number): StoredChunk | null {\n\t\tconst row = this.db\n\t\t\t.prepare(\n\t\t\t\t\"SELECT id, file_id, file_path, start_line, end_line, kind, name, content, file_type FROM chunks WHERE id = ?\",\n\t\t\t)\n\t\t\t.get(chunkId);\n\t\tif (!row) return null;\n\t\treturn this.rowToChunk(row);\n\t}\n\n\t/** Get multiple chunks by IDs. Batches queries to avoid exceeding SQLite's bind variable limit. */\n\tgetChunksById(chunkIds: number[]): StoredChunk[] {\n\t\tif (chunkIds.length === 0) return [];\n\n\t\tconst BATCH_SIZE = 500;\n\t\tconst results: StoredChunk[] = [];\n\n\t\tfor (let i = 0; i < chunkIds.length; i += BATCH_SIZE) {\n\t\t\tconst batch = chunkIds.slice(i, i + BATCH_SIZE);\n\t\t\tconst placeholders = batch.map(() => \"?\").join(\",\");\n\t\t\tconst rows = this.db\n\t\t\t\t.prepare(\n\t\t\t\t\t`SELECT id, file_id, file_path, start_line, end_line, kind, name, content, file_type FROM chunks WHERE id IN (${placeholders})`,\n\t\t\t\t)\n\t\t\t\t.all(...batch);\n\t\t\tfor (const r of rows) {\n\t\t\t\tresults.push(this.rowToChunk(r));\n\t\t\t}\n\t\t}\n\n\t\treturn results;\n\t}\n\n\tprivate rowToChunk(r: any): StoredChunk {\n\t\treturn {\n\t\t\tid: r.id,\n\t\t\tfileId: r.file_id,\n\t\t\tfilePath: r.file_path,\n\t\t\tstartLine: r.start_line,\n\t\t\tendLine: r.end_line,\n\t\t\tkind: r.kind as ChunkKind,\n\t\t\tname: r.name,\n\t\t\tcontent: r.content,\n\t\t\tfileType: r.file_type as FileType,\n\t\t};\n\t}\n\n\t// ========================================================================\n\t// Embeddings\n\t// ========================================================================\n\n\t/** Store an embedding vector for a chunk. */\n\tupsertEmbedding(chunkId: number, modelName: string, vector: Float32Array): void {\n\t\tconst blob = Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);\n\t\tthis.db\n\t\t\t.prepare(\"INSERT OR REPLACE INTO embeddings (chunk_id, model_name, vector) VALUES (?, ?, ?)\")\n\t\t\t.run(chunkId, modelName, blob);\n\t}\n\n\t/** Batch insert embeddings. Uses a transaction for performance. */\n\tbatchUpsertEmbeddings(items: Array<{ chunkId: number; modelName: string; vector: Float32Array }>): void {\n\t\tconst stmt = this.db.prepare(\"INSERT OR REPLACE INTO embeddings (chunk_id, model_name, vector) VALUES (?, ?, ?)\");\n\t\tthis.transaction(() => {\n\t\t\tfor (const item of items) {\n\t\t\t\tconst blob = Buffer.from(item.vector.buffer, item.vector.byteOffset, item.vector.byteLength);\n\t\t\t\tstmt.run(item.chunkId, item.modelName, blob);\n\t\t\t}\n\t\t});\n\t}\n\n\t/** Get the embedding for a chunk. */\n\tgetEmbedding(chunkId: number, modelName: string): Float32Array | null {\n\t\tconst row = this.db\n\t\t\t.prepare(\"SELECT vector FROM embeddings WHERE chunk_id = ? AND model_name = ?\")\n\t\t\t.get(chunkId, modelName);\n\t\tif (!row) return null;\n\t\treturn unpackVector(row.vector);\n\t}\n\n\t/** Get all embeddings for a model. Returns map of chunkId → vector. */\n\tgetAllEmbeddings(modelName: string): Map<number, Float32Array> {\n\t\tconst rows = this.db.prepare(\"SELECT chunk_id, vector FROM embeddings WHERE model_name = ?\").all(modelName);\n\t\tconst map = new Map<number, Float32Array>();\n\t\tfor (const row of rows) {\n\t\t\tmap.set(row.chunk_id, unpackVector(row.vector));\n\t\t}\n\t\treturn map;\n\t}\n\n\t/** Get chunk IDs that have no embedding for a given model. */\n\tgetChunkIdsWithoutEmbedding(modelName: string): number[] {\n\t\tconst rows = this.db\n\t\t\t.prepare(\n\t\t\t\t`SELECT c.id FROM chunks c\n\t\t\t\t LEFT JOIN embeddings e ON c.id = e.chunk_id AND e.model_name = ?\n\t\t\t\t WHERE e.chunk_id IS NULL`,\n\t\t\t)\n\t\t\t.all(modelName);\n\t\treturn rows.map((r: any) => r.id);\n\t}\n\n\t// ========================================================================\n\t// FTS5\n\t// ========================================================================\n\n\t/** Rebuild the FTS5 index (use after bulk operations). */\n\trebuildFts(): void {\n\t\tthis.db.exec(\"INSERT INTO chunks_fts(chunks_fts) VALUES ('rebuild')\");\n\t}\n\n\t/**\n\t * Search via FTS5 with BM25 ranking.\n\t * Returns chunk IDs with their BM25 scores (negated so higher = better).\n\t */\n\tftsSearch(query: string, limit: number): Array<{ chunkId: number; score: number }> {\n\t\ttry {\n\t\t\tconst rows = this.db\n\t\t\t\t.prepare(\n\t\t\t\t\t`SELECT chunks_fts.rowid as chunk_id, -bm25(chunks_fts, 1.0, 10.0, 5.0) as score\n\t\t\t\t\t FROM chunks_fts\n\t\t\t\t\t WHERE chunks_fts MATCH ?\n\t\t\t\t\t ORDER BY score DESC\n\t\t\t\t\t LIMIT ?`,\n\t\t\t\t)\n\t\t\t\t.all(query, limit);\n\t\t\treturn rows.map((r: any) => ({ chunkId: r.chunk_id, score: r.score }));\n\t\t} catch {\n\t\t\t// FTS5 MATCH can fail on malformed queries\n\t\t\treturn [];\n\t\t}\n\t}\n\n\t// ========================================================================\n\t// Imports\n\t// ========================================================================\n\n\t/** Record an import edge. */\n\tinsertImport(sourceFileId: number, targetFilePath: string): void {\n\t\tthis.db\n\t\t\t.prepare(\"INSERT OR IGNORE INTO imports (source_file_id, target_file_path) VALUES (?, ?)\")\n\t\t\t.run(sourceFileId, targetFilePath);\n\t}\n\n\t/** Delete all imports for a source file. */\n\tdeleteImportsForFile(sourceFileId: number): void {\n\t\tthis.db.prepare(\"DELETE FROM imports WHERE source_file_id = ?\").run(sourceFileId);\n\t}\n\n\t/** Get files imported by a given source file. */\n\tgetImportsFrom(sourceFileId: number): string[] {\n\t\tconst rows = this.db.prepare(\"SELECT target_file_path FROM imports WHERE source_file_id = ?\").all(sourceFileId);\n\t\treturn rows.map((r: any) => r.target_file_path);\n\t}\n\n\t/** Get file IDs that import a given target path. */\n\tgetImportersOf(targetFilePath: string): number[] {\n\t\tconst rows = this.db.prepare(\"SELECT source_file_id FROM imports WHERE target_file_path = ?\").all(targetFilePath);\n\t\treturn rows.map((r: any) => r.source_file_id);\n\t}\n\n\t/** Get all import edges. */\n\tgetAllImports(): Array<{ sourceFileId: number; targetFilePath: string }> {\n\t\tconst rows = this.db.prepare(\"SELECT source_file_id, target_file_path FROM imports\").all();\n\t\treturn rows.map((r: any) => ({ sourceFileId: r.source_file_id, targetFilePath: r.target_file_path }));\n\t}\n\n\t// ========================================================================\n\t// Symbols\n\t// ========================================================================\n\n\t/** Insert a symbol. */\n\tinsertSymbol(chunkId: number, name: string, kind: string): void {\n\t\tthis.db.prepare(\"INSERT INTO symbols (chunk_id, name, kind) VALUES (?, ?, ?)\").run(chunkId, name, kind);\n\t}\n\n\t/** Delete symbols for a chunk. */\n\tdeleteSymbolsForChunk(chunkId: number): void {\n\t\tthis.db.prepare(\"DELETE FROM symbols WHERE chunk_id = ?\").run(chunkId);\n\t}\n\n\t/** Get all symbols. Returns map of chunkId → symbol names. */\n\tgetAllSymbols(): Map<number, string[]> {\n\t\tconst rows = this.db.prepare(\"SELECT chunk_id, name FROM symbols\").all();\n\t\tconst map = new Map<number, string[]>();\n\t\tfor (const row of rows) {\n\t\t\tconst existing = map.get(row.chunk_id);\n\t\t\tif (existing) existing.push(row.name);\n\t\t\telse map.set(row.chunk_id, [row.name]);\n\t\t}\n\t\treturn map;\n\t}\n\n\t// ========================================================================\n\t// Transaction Helpers\n\t// ========================================================================\n\n\t/** Run a function inside a transaction. */\n\ttransaction<T>(fn: () => T): T {\n\t\tthis.db.exec(\"BEGIN\");\n\t\ttry {\n\t\t\tconst result = fn();\n\t\t\tthis.db.exec(\"COMMIT\");\n\t\t\treturn result;\n\t\t} catch (err) {\n\t\t\tthis.db.exec(\"ROLLBACK\");\n\t\t\tthrow err;\n\t\t}\n\t}\n\n\t/** Get the total number of chunks. */\n\tgetChunkCount(): number {\n\t\tconst row = this.db.prepare(\"SELECT COUNT(*) as count FROM chunks\").get();\n\t\treturn row.count;\n\t}\n\n\t/** Get the total number of files. */\n\tgetFileCount(): number {\n\t\tconst row = this.db.prepare(\"SELECT COUNT(*) as count FROM files\").get();\n\t\treturn row.count;\n\t}\n\n\t/** Close the database connection. */\n\tclose(): void {\n\t\tthis.db.close();\n\t}\n}\n"]}