@lojban/semantic-search-mcp 1.0.2 → 1.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lojban/semantic-search-mcp",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Local-first MCP server for semantic search using transformers.js and SQLite",
   "type": "module",
   "scripts": {

package/src/embeddings.ts

@@ -24,23 +24,30 @@ export async function getEmbedding(text: string): Promise<Float32Array> {
   return new Float32Array(output.data as ArrayLike<number>);
 }
 
+const EMBEDDING_DIM = 384; // all-MiniLM-L6-v2
+
 /**
- * Generate embeddings for multiple texts (batched for efficiency)
+ * Generate embeddings for multiple texts (batched for efficiency).
+ * Passes arrays to the pipeline so the model runs one forward pass per batch.
  */
 export async function getBatchEmbeddings(texts: string[]): Promise<Float32Array[]> {
+  if (texts.length === 0) return [];
+
   const ext = await getExtractor();
   const results: Float32Array[] = [];
-  
-  // Process in batches of 32 for memory efficiency
-  const batchSize = 32;
+
+  // Process in batches for memory; each batch is one model forward pass
+  const batchSize = 64;
   for (let i = 0; i < texts.length; i += batchSize) {
     const batch = texts.slice(i, i + batchSize);
-    for (const text of batch) {
-      const output = await ext(text, { pooling: 'mean', normalize: true });
-      results.push(new Float32Array(output.data as ArrayLike<number>));
+    const output = await ext(batch, { pooling: 'mean', normalize: true });
+    const data = output.data as Float32Array;
+    const dim = EMBEDDING_DIM;
+    for (let j = 0; j < batch.length; j++) {
+      results.push(new Float32Array(data.slice(j * dim, (j + 1) * dim)));
     }
   }
-  
+
   return results;
 }
 

package/src/index.ts

@@ -109,13 +109,23 @@ async function main() {
           const lines = await scanDirectories(directories);
           console.error(`Found ${lines.length} lines to index`);
 
-          const batchSize = 50;
+          const batchSize = 128;
           let indexed = 0;
 
+          // Pipeline: compute embeddings for next batch while writing current batch to DB
+          let embedPromise: Promise<Float32Array[]> =
+            lines.length > 0
+              ? getBatchEmbeddings(lines.slice(0, batchSize).map((l) => l.content))
+              : Promise.resolve([]);
+
           for (let i = 0; i < lines.length; i += batchSize) {
             const batch = lines.slice(i, i + batchSize);
-            const texts = batch.map(l => l.content);
-            const embeddings = await getBatchEmbeddings(texts);
+            const embeddings = await embedPromise;
+
+            if (i + batchSize < lines.length) {
+              const nextTexts = lines.slice(i + batchSize, i + batchSize * 2).map((l) => l.content);
+              embedPromise = getBatchEmbeddings(nextTexts);
+            }
 
             const batchData = batch.map((line, idx) => ({
               filePath: line.filePath,

package/src/scanner.ts

@@ -67,15 +67,9 @@ export async function scanDirectory(dirPath: string): Promise<FileLine[]> {
 }
 
 /**
- * Scan multiple directories
+ * Scan multiple directories in parallel
  */
 export async function scanDirectories(dirPaths: string[]): Promise<FileLine[]> {
-  const allLines: FileLine[] = [];
-  
-  for (const dirPath of dirPaths) {
-    const lines = await scanDirectory(dirPath);
-    allLines.push(...lines);
-  }
-  
-  return allLines;
+  const results = await Promise.all(dirPaths.map((dirPath) => scanDirectory(dirPath)));
+  return results.flat();
 }

package/src/storage.ts

@@ -63,18 +63,20 @@ export class VectorStorage {
     const sel = await this.db.prepare('SELECT id FROM lines WHERE file_path = ? AND line_number = ?');
     const row = sel.get([filePath, lineNumber]) as { id: number } | undefined;
     if (row == null) throw new Error('Failed to get line id');
-    const id = Math.trunc(Number(row.id));
+    const idInt = BigInt(row.id); // vec0 requires INTEGER primary key; BigInt binds as integer in better-sqlite3
 
-    (await this.db.prepare('DELETE FROM vec_lines WHERE line_id = ?')).run([id]);
-    (await this.db.prepare('INSERT INTO vec_lines (line_id, embedding) VALUES (?, ?)')).run([id, embedding.buffer]);
+    (await this.db.prepare('DELETE FROM vec_lines WHERE line_id = ?')).run([idInt]);
+    (await this.db.prepare('INSERT INTO vec_lines (line_id, embedding) VALUES (?, ?)')).run([idInt, embedding.buffer]);
   }
 
   /**
-   * Batch insert lines for efficiency
+   * Batch insert lines for efficiency (single transaction)
    */
   async upsertLinesBatch(
     lines: Array<{ filePath: string; lineNumber: number; content: string; embedding: Float32Array }>
   ): Promise<void> {
+    if (lines.length === 0) return;
+
     const insertLine = await this.db.prepare(
       `INSERT INTO lines (file_path, line_number, content)
        VALUES (?, ?, ?)
@@ -84,12 +86,19 @@ export class VectorStorage {
     const deleteVec = await this.db.prepare('DELETE FROM vec_lines WHERE line_id = ?');
     const insertVec = await this.db.prepare('INSERT INTO vec_lines (line_id, embedding) VALUES (?, ?)');
 
-    for (const item of lines) {
-      insertLine.run([item.filePath, item.lineNumber, item.content]);
-      const row = selId.get([item.filePath, item.lineNumber]) as { id: number };
-      const id = Math.trunc(Number(row.id));
-      deleteVec.run([id]);
-      insertVec.run([id, item.embedding.buffer]);
+    this.db.exec('BEGIN');
+    try {
+      for (const item of lines) {
+        insertLine.run([item.filePath, item.lineNumber, item.content]);
+        const row = selId.get([item.filePath, item.lineNumber]) as { id: number };
+        const idInt = BigInt(row.id);
+        deleteVec.run([idInt]);
+        insertVec.run([idInt, item.embedding.buffer]);
+      }
+      this.db.exec('COMMIT');
+    } catch (e) {
+      this.db.exec('ROLLBACK');
+      throw e;
     }
   }