@dreb/semantic-search 1.18.0

package/dist/vector-store.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Vector operations for semantic search.
+ *
+ * Pure JS implementations — no native dependencies or SQLite UDFs needed.
+ * Vectors are computed and compared in JS, stored as BLOBs in SQLite.
+ */
+/**
+ * Compute cosine similarity between two normalized vectors.
+ *
+ * For normalized vectors, cosine similarity is simply the dot product:
+ *   cos(a, b) = Σ a[i] * b[i]
+ *
+ * Returns a value in [-1, 1] where 1 = identical, 0 = orthogonal, -1 = opposite.
+ */
+export declare function cosineSimilarity(a: Float32Array, b: Float32Array): number;
+/**
+ * Pack a Float32Array into a Buffer for SQLite BLOB storage.
+ *
+ * Creates a copy to ensure the buffer isn't shared with other typed arrays.
+ */
+export declare function packVector(vector: Float32Array): Buffer;
+/**
+ * Unpack a BLOB (Uint8Array from node:sqlite) back to a Float32Array.
+ *
+ * The returned array shares the underlying buffer with the input for
+ * zero-copy performance. Callers should not mutate the input after calling.
+ */
+export declare function unpackVector(blob: Uint8Array): Float32Array;
+/**
+ * Find the top-K most similar vectors from a set.
+ *
+ * Computes cosine similarity between the query vector and every candidate,
+ * then returns the K highest-scoring results sorted by descending score.
+ *
+ * Uses a simple full scan — suitable for the index sizes we expect in a
+ * single-project codebase (typically <100K chunks). For millions of vectors,
+ * an approximate nearest neighbor index (HNSW, IVF) would be needed.
+ */
+export declare function topKSimilar(query: Float32Array, vectors: Map<number, Float32Array>, k: number): Array<{
+    id: number;
+    score: number;
+}>;
+//# sourceMappingURL=vector-store.d.ts.map

package/dist/vector-store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH;;;;;;;GAOG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,YAAY,GAAG,MAAM,CAOzE;AAMD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,YAAY,GAAG,MAAM,CAEvD;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,UAAU,GAAG,YAAY,CAE3D;AAMD;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAC1B,KAAK,EAAE,YAAY,EACnB,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC,EAClC,CAAC,EAAE,MAAM,GACP,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAgBtC","sourcesContent":["/**\n * Vector operations for semantic search.\n *\n * Pure JS implementations — no native dependencies or SQLite UDFs needed.\n * Vectors are computed and compared in JS, stored as BLOBs in SQLite.\n */\n\n// ============================================================================\n// Similarity\n// ============================================================================\n\n/**\n * Compute cosine similarity between two normalized vectors.\n *\n * For normalized vectors, cosine similarity is simply the dot product:\n *   cos(a, b) = Σ a[i] * b[i]\n *\n * Returns a value in [-1, 1] where 1 = identical, 0 = orthogonal, -1 = opposite.\n */\nexport function cosineSimilarity(a: Float32Array, b: Float32Array): number {\n\tconst len = a.length;\n\tlet dot = 0;\n\tfor (let i = 0; i < len; i++) {\n\t\tdot += a[i] * b[i];\n\t}\n\treturn dot;\n}\n\n// ============================================================================\n// Serialization\n// ============================================================================\n\n/**\n * Pack a Float32Array into a Buffer for SQLite BLOB storage.\n *\n * Creates a copy to ensure the buffer isn't shared with other typed arrays.\n */\nexport function packVector(vector: Float32Array): Buffer {\n\treturn Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);\n}\n\n/**\n * Unpack a BLOB (Uint8Array from node:sqlite) back to a Float32Array.\n *\n * The returned array shares the underlying buffer with the input for\n * zero-copy performance. Callers should not mutate the input after calling.\n */\nexport function unpackVector(blob: Uint8Array): Float32Array {\n\treturn new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);\n}\n\n// ============================================================================\n// Top-K Search\n// ============================================================================\n\n/**\n * Find the top-K most similar vectors from a set.\n *\n * Computes cosine similarity between the query vector and every candidate,\n * then returns the K highest-scoring results sorted by descending score.\n *\n * Uses a simple full scan — suitable for the index sizes we expect in a\n * single-project codebase (typically <100K chunks). For millions of vectors,\n * an approximate nearest neighbor index (HNSW, IVF) would be needed.\n */\nexport function topKSimilar(\n\tquery: Float32Array,\n\tvectors: Map<number, Float32Array>,\n\tk: number,\n): Array<{ id: number; score: number }> {\n\tif (k <= 0 || vectors.size === 0) return [];\n\n\t// For small k relative to n, a min-heap would be more efficient.\n\t// For typical codebase sizes (<100K vectors) the difference is negligible,\n\t// and a sorted array is simpler and correct.\n\tconst scored: Array<{ id: number; score: number }> = [];\n\n\tfor (const [id, vector] of vectors) {\n\t\tscored.push({ id, score: cosineSimilarity(query, vector) });\n\t}\n\n\t// Partial sort: only need top-k, but full sort is fine for expected sizes\n\tscored.sort((a, b) => b.score - a.score);\n\n\treturn scored.slice(0, k);\n}\n"]}

package/dist/vector-store.js

@@ -0,0 +1,73 @@
+/**
+ * Vector operations for semantic search.
+ *
+ * Pure JS implementations — no native dependencies or SQLite UDFs needed.
+ * Vectors are computed and compared in JS, stored as BLOBs in SQLite.
+ */
+// ============================================================================
+// Similarity
+// ============================================================================
+/**
+ * Compute cosine similarity between two normalized vectors.
+ *
+ * For normalized vectors, cosine similarity is simply the dot product:
+ *   cos(a, b) = Σ a[i] * b[i]
+ *
+ * Returns a value in [-1, 1] where 1 = identical, 0 = orthogonal, -1 = opposite.
+ */
+export function cosineSimilarity(a, b) {
+    const len = a.length;
+    let dot = 0;
+    for (let i = 0; i < len; i++) {
+        dot += a[i] * b[i];
+    }
+    return dot;
+}
+// ============================================================================
+// Serialization
+// ============================================================================
+/**
+ * Pack a Float32Array into a Buffer for SQLite BLOB storage.
+ *
+ * Creates a copy to ensure the buffer isn't shared with other typed arrays.
+ */
+export function packVector(vector) {
+    return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+}
+/**
+ * Unpack a BLOB (Uint8Array from node:sqlite) back to a Float32Array.
+ *
+ * The returned array shares the underlying buffer with the input for
+ * zero-copy performance. Callers should not mutate the input after calling.
+ */
+export function unpackVector(blob) {
+    return new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);
+}
+// ============================================================================
+// Top-K Search
+// ============================================================================
+/**
+ * Find the top-K most similar vectors from a set.
+ *
+ * Computes cosine similarity between the query vector and every candidate,
+ * then returns the K highest-scoring results sorted by descending score.
+ *
+ * Uses a simple full scan — suitable for the index sizes we expect in a
+ * single-project codebase (typically <100K chunks). For millions of vectors,
+ * an approximate nearest neighbor index (HNSW, IVF) would be needed.
+ */
+export function topKSimilar(query, vectors, k) {
+    if (k <= 0 || vectors.size === 0)
+        return [];
+    // For small k relative to n, a min-heap would be more efficient.
+    // For typical codebase sizes (<100K vectors) the difference is negligible,
+    // and a sorted array is simpler and correct.
+    const scored = [];
+    for (const [id, vector] of vectors) {
+        scored.push({ id, score: cosineSimilarity(query, vector) });
+    }
+    // Partial sort: only need top-k, but full sort is fine for expected sizes
+    scored.sort((a, b) => b.score - a.score);
+    return scored.slice(0, k);
+}
+//# sourceMappingURL=vector-store.js.map

package/dist/vector-store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vector-store.js","sourceRoot":"","sources":["../src/vector-store.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,CAAe,EAAE,CAAe,EAAU;IAC1E,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC;IACrB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,GAAG,CAAC;AAAA,CACX;AAED,+EAA+E;AAC/E,gBAAgB;AAChB,+EAA+E;AAE/E;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,MAAoB,EAAU;IACxD,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;AAAA,CACxE;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,IAAgB,EAAgB;IAC5D,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;AAAA,CAC3E;AAED,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E;;;;;;;;;GASG;AACH,MAAM,UAAU,WAAW,CAC1B,KAAmB,EACnB,OAAkC,EAClC,CAAS,EAC8B;IACvC,IAAI,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE5C,iEAAiE;IACjE,2EAA2E;IAC3E,6CAA6C;IAC7C,MAAM,MAAM,GAAyC,EAAE,CAAC;IAExD,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC;IAED,0EAA0E;IAC1E,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAAA,CAC1B","sourcesContent":["/**\n * Vector operations for semantic search.\n *\n * Pure JS implementations — no native dependencies or SQLite UDFs needed.\n * Vectors are computed and compared in JS, stored as BLOBs in SQLite.\n */\n\n// ============================================================================\n// Similarity\n// ============================================================================\n\n/**\n * Compute cosine similarity between two normalized vectors.\n *\n * For normalized vectors, cosine similarity is simply the dot product:\n *   cos(a, b) = Σ a[i] * b[i]\n *\n * Returns a value in [-1, 1] where 1 = identical, 0 = orthogonal, -1 = opposite.\n */\nexport function cosineSimilarity(a: Float32Array, b: Float32Array): number {\n\tconst len = a.length;\n\tlet dot = 0;\n\tfor (let i = 0; i < len; i++) {\n\t\tdot += a[i] * b[i];\n\t}\n\treturn dot;\n}\n\n// ============================================================================\n// Serialization\n// ============================================================================\n\n/**\n * Pack a Float32Array into a Buffer for SQLite BLOB storage.\n *\n * Creates a copy to ensure the buffer isn't shared with other typed arrays.\n */\nexport function packVector(vector: Float32Array): Buffer {\n\treturn Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);\n}\n\n/**\n * Unpack a BLOB (Uint8Array from node:sqlite) back to a Float32Array.\n *\n * The returned array shares the underlying buffer with the input for\n * zero-copy performance. Callers should not mutate the input after calling.\n */\nexport function unpackVector(blob: Uint8Array): Float32Array {\n\treturn new Float32Array(blob.buffer, blob.byteOffset, blob.byteLength / 4);\n}\n\n// ============================================================================\n// Top-K Search\n// ============================================================================\n\n/**\n * Find the top-K most similar vectors from a set.\n *\n * Computes cosine similarity between the query vector and every candidate,\n * then returns the K highest-scoring results sorted by descending score.\n *\n * Uses a simple full scan — suitable for the index sizes we expect in a\n * single-project codebase (typically <100K chunks). For millions of vectors,\n * an approximate nearest neighbor index (HNSW, IVF) would be needed.\n */\nexport function topKSimilar(\n\tquery: Float32Array,\n\tvectors: Map<number, Float32Array>,\n\tk: number,\n): Array<{ id: number; score: number }> {\n\tif (k <= 0 || vectors.size === 0) return [];\n\n\t// For small k relative to n, a min-heap would be more efficient.\n\t// For typical codebase sizes (<100K vectors) the difference is negligible,\n\t// and a sorted array is simpler and correct.\n\tconst scored: Array<{ id: number; score: number }> = [];\n\n\tfor (const [id, vector] of vectors) {\n\t\tscored.push({ id, score: cosineSimilarity(query, vector) });\n\t}\n\n\t// Partial sort: only need top-k, but full sort is fine for expected sizes\n\tscored.sort((a, b) => b.score - a.score);\n\n\treturn scored.slice(0, k);\n}\n"]}

package/package.json

@@ -0,0 +1,71 @@
+{
+	"name": "@dreb/semantic-search",
+	"version": "1.18.0",
+	"description": "Semantic codebase search engine with embedding-based ranking and MCP server",
+	"publishConfig": {
+		"access": "public"
+	},
+	"type": "module",
+	"main": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js"
+		}
+	},
+	"bin": {
+		"semantic-search-mcp": "./bin/server.js"
+	},
+	"files": [
+		"dist",
+		"bin",
+		".claude-plugin",
+		".mcp.json",
+		"skills",
+		"README.md"
+	],
+	"scripts": {
+		"clean": "shx rm -rf dist",
+		"build": "tsgo -p tsconfig.build.json",
+		"dev": "tsgo -p tsconfig.build.json --watch --preserveWatchOutput",
+		"test": "vitest --run",
+		"prepublishOnly": "npm run clean && npm run build"
+	},
+	"dependencies": {
+		"@huggingface/transformers": "^4.0.1",
+		"@modelcontextprotocol/sdk": "^1.29.0",
+		"ignore": "^7.0.5",
+		"tree-sitter-c": "^0.24.1",
+		"tree-sitter-cpp": "^0.23.4",
+		"tree-sitter-go": "^0.25.0",
+		"tree-sitter-java": "^0.23.5",
+		"tree-sitter-javascript": "^0.25.0",
+		"tree-sitter-python": "^0.25.0",
+		"tree-sitter-rust": "^0.24.0",
+		"tree-sitter-typescript": "^0.23.2",
+		"web-tree-sitter": "^0.26.8"
+	},
+	"devDependencies": {
+		"@types/node": "^24.3.0",
+		"shx": "^0.4.0",
+		"typescript": "^5.9.2",
+		"vitest": "^3.2.4"
+	},
+	"keywords": [
+		"semantic-search",
+		"codebase",
+		"embeddings",
+		"mcp"
+	],
+	"author": "Drew Brereton",
+	"license": "MIT",
+	"repository": {
+		"type": "git",
+		"url": "https://github.com/aebrer/dreb.git",
+		"directory": "packages/semantic-search"
+	},
+	"engines": {
+		"node": ">=22.0.0"
+	}
+}

package/skills/search/SKILL.md

@@ -0,0 +1,56 @@
+# Semantic Codebase Search
+
+Use `search` as your **default exploration tool** for understanding code, finding implementations, and answering questions about the codebase. Use `grep` when you already know the exact text or pattern you're looking for.
+
+## Indexing
+
+The first query builds the index — this may take 10–60 seconds depending on project size. Subsequent queries are fast because the index is incrementally updated (only changed files are re-processed).
+
+## Query Types
+
+The search tool supports three kinds of queries, automatically classified:
+
+- **Identifier queries** — e.g. `AuthMiddleware`, `handleRequest`, `SearchEngine` — finds definitions, usages, and related code for a specific symbol
+- **Natural language queries** — e.g. `where is rate limiting handled`, `how does authentication work` — semantic search across code and documentation
+- **Path queries** — e.g. `src/auth/`, `packages/ai` — finds code within a directory structure
+
+## Parameters
+
+| Parameter    | Required | Description                                                                 |
+| ------------ | -------- | --------------------------------------------------------------------------- |
+| `query`      | Yes      | Search query — natural language, identifier, or path                        |
+| `projectDir` | Yes      | Absolute path to the project directory. Set this to your current working directory |
+| `path`       | No       | Restrict search to files under this subdirectory (relative to project root) |
+| `limit`      | No       | Maximum number of results to return (default: 20)                           |
+| `rebuild`    | No       | Force a clean index rebuild — use when files have changed significantly     |
+
+## Ranking
+
+Results are ranked using 6 signals fused via **POEM** (Pareto-Optimal Embedded Modeling):
+
+1. **BM25** — keyword matching via FTS5 full-text search
+2. **Cosine similarity** — embedding-based semantic similarity (all-MiniLM-L6-v2)
+3. **Path match** — query terms appearing in the file path
+4. **Symbol match** — query terms matching function/class/type names
+5. **Import graph proximity** — files imported by or importing high-scoring files
+6. **Git recency** — recently modified files ranked higher
+
+The weight given to each signal varies by query type. Identifier queries emphasize symbol match and BM25; natural language queries emphasize cosine similarity; path queries emphasize path match.
+
+## Results
+
+Each result includes:
+
+- **File path** and **line range** (start–end)
+- **Chunk kind** (function, class, method, interface, heading_section, etc.) and **name**
+- **Metric scores** for each of the 6 signals
+- **Content preview** of the matching code or text
+
+## Tips
+
+- Start broad, then narrow with `path` if you get too many results from different areas
+- Use `limit` to get more results when exploring a broad topic (e.g. `limit: 50`)
+- Use `rebuild: true` after major refactors, branch switches, or large file changes
+- Identifier queries work best for finding where something is defined or used
+- Natural language queries work best for understanding how a feature or concept is implemented
+- Combine search with `grep` for a powerful workflow: search to find the right files, then grep for exact patterns within them