gitnexus 1.6.2-rc.1 → 1.6.2-rc.3

package/dist/core/embeddings/embedding-pipeline.js

@@ -67,8 +67,8 @@ const queryEmbeddableNodes = async (executeQuery) => {
  * that occurs when UPDATEing nodes with large content fields
  */
 const batchInsertEmbeddings = async (executeWithReusedStatement, updates) => {
-    // INSERT into separate embedding table - much more memory efficient!
-    const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`;
+    // MERGE instead of CREATE — idempotent, handles concurrent analyzes and partial prior runs
+    const cypher = `MERGE (e:CodeEmbedding {nodeId: $nodeId}) SET e.embedding = $embedding`;
     const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding }));
     await executeWithReusedStatement(cypher, paramsList);
 };

package/dist/core/lbug/csv-generator.js

@@ -246,14 +246,17 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
         Interface: interfaceWriter,
         CodeElement: codeElemWriter,
     };
-    const seenFileIds = new Set();
+    // Deduplicate all node types — the pipeline can produce duplicate IDs across
+    // all symbol types (Class, Method, Function, etc.), not just File nodes.
+    // A single Set covering every label prevents PK violations on COPY.
+    const seenNodeIds = new Set();
     // --- SINGLE PASS over all nodes ---
     for (const node of graph.iterNodes()) {
+        if (seenNodeIds.has(node.id))
+            continue;
+        seenNodeIds.add(node.id);
         switch (node.label) {
             case 'File': {
-                if (seenFileIds.has(node.id))
-                    break;
-                seenFileIds.add(node.id);
                 const content = await extractContent(node, contentCache);
                 await fileWriter.addRow([
                     escapeCSVField(node.id),

package/dist/core/lbug/lbug-adapter.js

@@ -1,6 +1,8 @@
 import fs from 'fs/promises';
 import { createReadStream, createWriteStream } from 'fs';
 import { createInterface } from 'readline';
+import { once } from 'events';
+import { finished } from 'stream/promises';
 import path from 'path';
 import lbug from '@ladybugdb/core';
 import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
@@ -25,100 +27,82 @@ export const splitRelCsvByLabelPair = async (csvPath, csvDir, validTables, getNo
     const pairWriteStreams = new Map();
     let skippedRels = 0;
     let totalValidRels = 0;
-    await new Promise((resolve, reject) => {
-        const inputStream = createReadStream(csvPath, 'utf-8');
-        const rl = createInterface({
-            input: inputStream,
-            crlfDelay: Infinity,
-        });
-        // Track which streams are already waiting for drain to prevent
-        // listener accumulation. rl.pause() is not synchronous — buffered
-        // line events continue firing after pause(), and without this guard
-        // each line targeting the same pairKey would add another drain listener.
-        const waitingForDrain = new Set();
-        let settled = false;
-        const cleanup = (err) => {
-            if (settled)
-                return;
-            settled = true;
-            try {
-                rl.close();
-            }
-            catch { }
-            try {
-                inputStream.destroy();
-            }
-            catch { }
-            for (const ws of pairWriteStreams.values()) {
-                try {
-                    ws.destroy();
-                }
-                catch { }
-            }
-            reject(err);
-        };
+    const inputStream = createReadStream(csvPath, 'utf-8');
+    const rl = createInterface({ input: inputStream, crlfDelay: Infinity });
+    // If any pair WriteStream errors (disk full, EMFILE, etc.) or the input
+    // stream fails, we need to abort the pending `once(ws, 'drain')` await.
+    // An AbortController gives us one signal to cancel all pending waits
+    // without a custom state machine.
+    const abortOnError = new AbortController();
+    let streamError = null;
+    const markStreamError = (err) => {
+        streamError ??= err;
+        abortOnError.abort(err);
+    };
+    try {
+        // `for await (const line of rl)` replaces the old manual
+        // on('line')/pause()/resume()/waitingForDrain state machine: readline's
+        // async iterator naturally serializes line delivery with our awaits, so
+        // at most one ws can be in backpressure at a time and we just await its
+        // 'drain' event.
         let isFirst = true;
-        rl.on('line', (line) => {
+        for await (const line of rl) {
+            if (streamError)
+                throw streamError;
             if (isFirst) {
                 relHeader = line;
                 isFirst = false;
-                return;
+                continue;
             }
             if (!line.trim())
-                return;
+                continue;
             const match = line.match(/"([^"]*)","([^"]*)"/);
             if (!match) {
                 skippedRels++;
-                return;
+                continue;
             }
             const fromLabel = getNodeLabel(match[1]);
             const toLabel = getNodeLabel(match[2]);
             if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
                 skippedRels++;
-                return;
+                continue;
             }
             const pairKey = `${fromLabel}|${toLabel}`;
             let ws = pairWriteStreams.get(pairKey);
             if (!ws) {
                 const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
                 ws = wsFactory(pairCsvPath);
-                // If any per-pair WriteStream errors (disk full, EMFILE, etc.),
-                // tear down everything and reject the Promise. Without this handler,
-                // a stream error while rl is paused waiting for drain would cause
-                // the drain callback to never fire and the Promise to hang forever.
-                ws.on('error', cleanup);
-                ws.write(relHeader + '\n');
+                ws.on('error', markStreamError);
                 pairWriteStreams.set(pairKey, ws);
                 relsByPairMeta.set(pairKey, { csvPath: pairCsvPath, rows: 0 });
+                if (!ws.write(relHeader + '\n')) {
+                    await once(ws, 'drain', { signal: abortOnError.signal });
+                }
+            }
+            if (!ws.write(line + '\n')) {
+                await once(ws, 'drain', { signal: abortOnError.signal });
             }
-            const ok = ws.write(line + '\n');
             relsByPairMeta.get(pairKey).rows++;
             totalValidRels++;
-            // Handle backpressure: pause reading when the write buffer is full,
-            // resume when the stream drains. Prevents unbounded memory growth
-            // on repos with millions of relationships.
-            // Guard with waitingForDrain to ensure only one drain listener is
-            // registered per stream at a time — rl.pause() doesn't stop buffered
-            // line events immediately. Only resume when ALL streams have drained
-            // to avoid writing into still-full streams.
-            if (!ok && !waitingForDrain.has(pairKey)) {
-                waitingForDrain.add(pairKey);
-                rl.pause();
-                ws.once('drain', () => {
-                    waitingForDrain.delete(pairKey);
-                    if (waitingForDrain.size === 0)
-                        rl.resume();
-                });
-            }
-        });
-        rl.on('close', () => {
-            if (!settled) {
-                settled = true;
-                resolve();
-            }
-        });
-        rl.on('error', cleanup);
-    });
+        }
+        if (streamError)
+            throw streamError;
+    }
+    catch (err) {
+        // Tear down everything so no fd is left dangling. If the abort was caused
+        // by a stream error, rethrow that error (more actionable than AbortError).
+        for (const ws of pairWriteStreams.values())
+            ws.destroy();
+        inputStream.destroy();
+        throw streamError ?? err;
+    }
+    finally {
+        // Readline 'close' fires before the underlying fs.ReadStream releases its
+        // fd — on Windows that race caused ENOTEMPTY on the parent dir.
+        // stream/promises.finished is the stdlib "wait until this stream is fully
+        // closed" primitive and handles both success and error paths.
+        await finished(inputStream).catch(() => { });
+    }
     return { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels };
 };
 let db = null;
@@ -332,15 +316,13 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
     }
     // Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
     const { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels } = await splitRelCsvByLabelPair(csvResult.relCsvPath, csvDir, validTables, getNodeLabel);
-    // Close all per-pair write streams before COPY
-    await Promise.all(Array.from(pairWriteStreams.values()).map((ws) => new Promise((resolve, reject) => {
-        const onError = (err) => reject(err);
-        ws.on('error', onError);
-        ws.end(() => {
-            ws.removeListener('error', onError);
-            resolve();
-        });
-    })));
+    // Close all per-pair write streams before COPY. `stream/promises.finished`
+    // resolves on the stream's 'finish' event and rejects on 'error' — replaces
+    // a hand-rolled promisification with the stdlib primitive.
+    await Promise.all(Array.from(pairWriteStreams.values()).map(async (ws) => {
+        ws.end();
+        await finished(ws);
+    }));
     const insertedRels = totalValidRels;
     const warnings = [];
     if (insertedRels > 0) {

package/dist/core/run-analyze.js

@@ -149,7 +149,7 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
                     const batch = cachedEmbeddings.slice(i, i + EMBED_BATCH);
                     const paramsList = batch.map((e) => ({ nodeId: e.nodeId, embedding: e.embedding }));
                     try {
-                        await executeWithReusedStatement(`CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`, paramsList);
+                        await executeWithReusedStatement(`MERGE (e:CodeEmbedding {nodeId: $nodeId}) SET e.embedding = $embedding`, paramsList);
                     }
                     catch {
                         /* some may fail if node was removed, that's fine */

package/dist/server/api.js

@@ -1277,6 +1277,26 @@ export const createServer = async (port, host = '127.0.0.1') => {
                     const lbugPath = path.join(entry.storagePath, 'lbug');
                     await withLbugDb(lbugPath, async () => {
                         const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
+                        // Skip nodes that already have embeddings — Kuzu forbids SET on vector-indexed properties.
+                        let skipNodeIds;
+                        try {
+                            const rows = await executeQuery('MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId');
+                            if (rows && rows.length > 0) {
+                                skipNodeIds = new Set(rows.map((r) => r.nodeId ?? r[0]).filter(Boolean));
+                                console.log(`[embed] ${skipNodeIds.size} nodes already embedded — skipping in incremental run`);
+                            }
+                        }
+                        catch (err) {
+                            // Swallow only "table does not exist" — let real connection errors propagate.
+                            // Log so ops can see this path fire if Kuzu ever changes error wording.
+                            const msg = err?.message ?? '';
+                            if (msg.includes('does not exist') || msg.includes('not found')) {
+                                console.log(`[embed] CodeEmbedding table not yet present — full embedding run (${msg})`);
+                            }
+                            else {
+                                throw err;
+                            }
+                        }
                         await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
                             embedJobManager.updateJob(job.id, {
                                 progress: {
@@ -1293,7 +1313,8 @@ export const createServer = async (port, host = '127.0.0.1') => {
                                                     : `${p.phase} (${p.percent}%)`,
                                 },
                             });
-                        });
+                        }, {}, // config: use defaults (runEmbeddingPipeline signature: executeQuery, executeWithReusedStatement, onProgress, config, skipNodeIds)
+                        skipNodeIds);
                     });
                     clearTimeout(embedTimeout);
                     releaseRepoLock(repoLockPath);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.2-rc.1",
+  "version": "1.6.2-rc.3",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",