@zuvia-software-solutions/code-mapper 2.0.2 → 2.1.1

package/dist/cli/analyze.js

@@ -189,7 +189,7 @@ export const analyzeCommand = async (inputPath, options) => {
     updateBar(60, 'Loading into database...');
     // Reset the database (delete and recreate)
     const t0Db = Date.now();
-    const db = resetDb(dbPath);
+    let db = resetDb(dbPath);
     let dbMsgCount = 0;
     const dbResult = loadGraphToDb(db, pipelineResult.graph, pipelineResult.repoPath, (msg) => {
         dbMsgCount++;
@@ -229,14 +229,61 @@ export const analyzeCommand = async (inputPath, options) => {
         embeddingSkipped = false;
     }
     if (!embeddingSkipped) {
-        updateBar(90, 'Loading embedding model...');
+        updateBar(90, 'Generating embeddings...');
         const t0Emb = Date.now();
-        const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
-        await runEmbeddingPipeline(db, (progress) => {
-            const scaled = 90 + Math.round((progress.percent / 100) * 8);
-            const label = progress.phase === 'loading-model' ? 'Loading embedding model...' : `Embedding ${progress.nodesProcessed || 0}/${progress.totalNodes || '?'}`;
-            updateBar(scaled, label);
-        }, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined);
+        // Close DB so Python can write to it
+        closeDb(dbPath);
+        // Run Python embedder in batch mode — reads from SQLite, embeds, writes back.
+        // Zero IPC overhead: ~3x faster than Node↔Python JSON streaming.
+        const { execFile } = await import('child_process');
+        const { fileURLToPath } = await import('url');
+        const mlxScript = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..', 'models', 'mlx-embedder.py');
+        await new Promise((resolve, reject) => {
+            const proc = execFile('python3', [mlxScript, 'batch', dbPath, '--dims', '256', '--max-tokens', '2048'], {
+                maxBuffer: 10 * 1024 * 1024,
+                timeout: 600_000, // 10 min max for huge codebases
+            }, (err, _stdout, stderr) => {
+                if (err) {
+                    console.error(stderr || '');
+                    reject(new Error(`Embedding failed: ${err.message}`));
+                }
+                else {
+                    resolve();
+                }
+            });
+            // Stream progress from Python's JSON lines on stdout
+            let lineBuf = '';
+            proc.stdout?.on('data', (chunk) => {
+                lineBuf += chunk.toString();
+                const lines = lineBuf.split('\n');
+                lineBuf = lines.pop() || '';
+                for (const line of lines) {
+                    if (!line.trim())
+                        continue;
+                    try {
+                        const msg = JSON.parse(line);
+                        if (msg.phase === 'loaded') {
+                            updateBar(91, `Model loaded (${msg.load_ms}ms)`);
+                        }
+                        else if (msg.phase === 'queried') {
+                            updateBar(92, `Found ${msg.nodes} embeddable nodes`);
+                        }
+                        else if (msg.phase === 'prepared') {
+                            updateBar(93, `${msg.to_embed} to embed, ${msg.skipped} cached`);
+                        }
+                        else if (msg.phase === 'embedded') {
+                            updateBar(96, `Embedded ${msg.count} nodes (${(msg.ms / 1000).toFixed(1)}s)`);
+                        }
+                        else if (msg.phase === 'done') {
+                            updateBar(98, `Embeddings complete (${msg.embedded} new, ${msg.skipped} cached)`);
+                        }
+                    }
+                    catch { }
+                }
+            });
+        });
+        // Reopen DB after Python is done
+        db = openDb(dbPath);
         embeddingTime = ((Date.now() - t0Emb) / 1000).toFixed(1);
     }
     // Phase 5: Finalize (98-100%)

package/dist/core/embeddings/embedder.js

@@ -159,24 +159,34 @@ export const embedBatch = async (texts) => {
         return [];
     if (!ready)
         await initEmbedder();
-    // Send all texts to Python in one call — Python does optimal length-tiered
-    // batching internally for Metal GPU. No need to double-batch at the Node level.
-    console.error(`Code Mapper: embedBatch sending ${texts.length} texts to MLX...`);
+    // Batch at Node level to keep stdin/stdout JSON messages manageable.
+    // Python does internal length-tiered batching within each chunk.
+    // 500 texts per chunk balances IPC overhead vs pipe buffer limits.
+    const CHUNK_SIZE = 500;
+    const allResults = [];
+    const totalChunks = Math.ceil(texts.length / CHUNK_SIZE);
     const t0 = Date.now();
-    const result = await sendAndReceive({
-        texts,
-        task: 'nl2code',
-        type: 'passage',
-        dims: DEFAULT_EMBEDDING_CONFIG.dimensions,
-    });
-    if (result.error)
-        throw new Error(`Batch embedding failed: ${result.error}`);
-    if (!result.embeddings || !Array.isArray(result.embeddings)) {
-        throw new Error(`Batch embedding returned invalid response: ${JSON.stringify(result).slice(0, 200)}`);
+    console.error(`Code Mapper: embedBatch ${texts.length} texts in ${totalChunks} chunk(s)...`);
+    for (let i = 0; i < texts.length; i += CHUNK_SIZE) {
+        const chunk = texts.slice(i, i + CHUNK_SIZE);
+        const result = await sendAndReceive({
+            texts: chunk,
+            task: 'nl2code',
+            type: 'passage',
+            dims: DEFAULT_EMBEDDING_CONFIG.dimensions,
+        });
+        if (result.error)
+            throw new Error(`Batch embedding failed: ${result.error}`);
+        if (!result.embeddings || !Array.isArray(result.embeddings)) {
+            throw new Error(`Batch embedding returned invalid response: ${JSON.stringify(result).slice(0, 200)}`);
+        }
+        for (const e of result.embeddings) {
+            allResults.push(new Float32Array(e));
+        }
     }
     const elapsed = Date.now() - t0;
-    console.error(`Code Mapper: embedBatch complete — ${result.embeddings.length} embeddings in ${elapsed}ms (${result.ms ?? '?'}ms inference)`);
-    return result.embeddings.map((e) => new Float32Array(e));
+    console.error(`Code Mapper: embedBatch complete — ${allResults.length} embeddings in ${elapsed}ms`);
+    return allResults;
 };
 /**
  * Embed a query text for semantic search (cached, uses "query" prompt type)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zuvia-software-solutions/code-mapper",
-  "version": "2.0.2",
+  "version": "2.1.1",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",