gitnexus 1.6.0 → 1.6.2-rc.1

package/dist/core/lbug/lbug-adapter.js

@@ -1,10 +1,126 @@
 import fs from 'fs/promises';
-import { createReadStream } from 'fs';
+import { createReadStream, createWriteStream } from 'fs';
 import { createInterface } from 'readline';
 import path from 'path';
 import lbug from '@ladybugdb/core';
 import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
 import { streamAllCSVsToDisk } from './csv-generator.js';
+/**
+ * Split a relationship CSV into per-label-pair files on disk.
+ *
+ * Streams the CSV line-by-line, routing each relationship to a file named
+ * `rel_{fromLabel}_{toLabel}.csv`. Handles backpressure correctly: only one
+ * drain listener per stream at a time, and readline resumes only when ALL
+ * backpressured streams have drained.
+ *
+ * @param csvPath       Path to the combined relationship CSV
+ * @param csvDir        Directory to write per-pair CSV files
+ * @param validTables   Set of valid node table names
+ * @param getNodeLabel  Function to extract the label from a node ID
+ * @param wsFactory     Optional WriteStream factory (defaults to fs.createWriteStream)
+ */
+export const splitRelCsvByLabelPair = async (csvPath, csvDir, validTables, getNodeLabel, wsFactory = (p) => createWriteStream(p, 'utf-8')) => {
+    let relHeader = '';
+    const relsByPairMeta = new Map();
+    const pairWriteStreams = new Map();
+    let skippedRels = 0;
+    let totalValidRels = 0;
+    await new Promise((resolve, reject) => {
+        const inputStream = createReadStream(csvPath, 'utf-8');
+        const rl = createInterface({
+            input: inputStream,
+            crlfDelay: Infinity,
+        });
+        // Track which streams are already waiting for drain to prevent
+        // listener accumulation. rl.pause() is not synchronous — buffered
+        // line events continue firing after pause(), and without this guard
+        // each line targeting the same pairKey would add another drain listener.
+        const waitingForDrain = new Set();
+        let settled = false;
+        const cleanup = (err) => {
+            if (settled)
+                return;
+            settled = true;
+            try {
+                rl.close();
+            }
+            catch { }
+            try {
+                inputStream.destroy();
+            }
+            catch { }
+            for (const ws of pairWriteStreams.values()) {
+                try {
+                    ws.destroy();
+                }
+                catch { }
+            }
+            reject(err);
+        };
+        let isFirst = true;
+        rl.on('line', (line) => {
+            if (isFirst) {
+                relHeader = line;
+                isFirst = false;
+                return;
+            }
+            if (!line.trim())
+                return;
+            const match = line.match(/"([^"]*)","([^"]*)"/);
+            if (!match) {
+                skippedRels++;
+                return;
+            }
+            const fromLabel = getNodeLabel(match[1]);
+            const toLabel = getNodeLabel(match[2]);
+            if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
+                skippedRels++;
+                return;
+            }
+            const pairKey = `${fromLabel}|${toLabel}`;
+            let ws = pairWriteStreams.get(pairKey);
+            if (!ws) {
+                const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
+                ws = wsFactory(pairCsvPath);
+                // If any per-pair WriteStream errors (disk full, EMFILE, etc.),
+                // tear down everything and reject the Promise. Without this handler,
+                // a stream error while rl is paused waiting for drain would cause
+                // the drain callback to never fire and the Promise to hang forever.
+                ws.on('error', cleanup);
+                ws.write(relHeader + '\n');
+                pairWriteStreams.set(pairKey, ws);
+                relsByPairMeta.set(pairKey, { csvPath: pairCsvPath, rows: 0 });
+            }
+            const ok = ws.write(line + '\n');
+            relsByPairMeta.get(pairKey).rows++;
+            totalValidRels++;
+            // Handle backpressure: pause reading when the write buffer is full,
+            // resume when the stream drains. Prevents unbounded memory growth
+            // on repos with millions of relationships.
+            // Guard with waitingForDrain to ensure only one drain listener is
+            // registered per stream at a time — rl.pause() doesn't stop buffered
+            // line events immediately. Only resume when ALL streams have drained
+            // to avoid writing into still-full streams.
+            if (!ok && !waitingForDrain.has(pairKey)) {
+                waitingForDrain.add(pairKey);
+                rl.pause();
+                ws.once('drain', () => {
+                    waitingForDrain.delete(pairKey);
+                    if (waitingForDrain.size === 0)
+                        rl.resume();
+                });
+            }
+        });
+        rl.on('close', () => {
+            if (!settled) {
+                settled = true;
+                resolve();
+            }
+        });
+        rl.on('error', cleanup);
+    });
+    return { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels };
+};
 let db = null;
 let conn = null;
 let currentDbPath = null;
@@ -215,64 +331,30 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
         }
     }
     // Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
-    // Stream-read the relation CSV line by line to avoid exceeding V8 max string length
-    let relHeader = '';
-    const relsByPair = new Map();
-    let skippedRels = 0;
-    let totalValidRels = 0;
-    await new Promise((resolve, reject) => {
-        const rl = createInterface({
-            input: createReadStream(csvResult.relCsvPath, 'utf-8'),
-            crlfDelay: Infinity,
-        });
-        let isFirst = true;
-        rl.on('line', (line) => {
-            if (isFirst) {
-                relHeader = line;
-                isFirst = false;
-                return;
-            }
-            if (!line.trim())
-                return;
-            const match = line.match(/"([^"]*)","([^"]*)"/);
-            if (!match) {
-                skippedRels++;
-                return;
-            }
-            const fromLabel = getNodeLabel(match[1]);
-            const toLabel = getNodeLabel(match[2]);
-            if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
-                skippedRels++;
-                return;
-            }
-            const pairKey = `${fromLabel}|${toLabel}`;
-            let list = relsByPair.get(pairKey);
-            if (!list) {
-                list = [];
-                relsByPair.set(pairKey, list);
-            }
-            list.push(line);
-            totalValidRels++;
+    const { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels } = await splitRelCsvByLabelPair(csvResult.relCsvPath, csvDir, validTables, getNodeLabel);
+    // Close all per-pair write streams before COPY
+    await Promise.all(Array.from(pairWriteStreams.values()).map((ws) => new Promise((resolve, reject) => {
+        const onError = (err) => reject(err);
+        ws.on('error', onError);
+        ws.end(() => {
+            ws.removeListener('error', onError);
+            resolve();
         });
-        rl.on('close', resolve);
-        rl.on('error', reject);
-    });
+    })));
     const insertedRels = totalValidRels;
     const warnings = [];
     if (insertedRels > 0) {
-        log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`);
+        log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPairMeta.size} types`);
         let pairIdx = 0;
         let failedPairEdges = 0;
-        const failedPairLines = [];
-        for (const [pairKey, lines] of relsByPair) {
+        const failedPairCsvPaths = new Set();
+        for (const [pairKey, { csvPath: pairCsvPath, rows }] of relsByPairMeta) {
             pairIdx++;
             const [fromLabel, toLabel] = pairKey.split('|');
-            const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
-            await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8');
             const normalizedPath = normalizeCopyPath(pairCsvPath);
             const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
-            if (pairIdx % 5 === 0 || lines.length > 1000) {
-                log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`);
+            if (pairIdx % 5 === 0 || rows > 1000) {
+                log(`Loading edges: ${pairIdx}/${relsByPairMeta.size} types (${fromLabel} -> ${toLabel})`);
             }
             try {
                 await conn.query(copyQuery);
@@ -284,19 +366,42 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
                 }
                 catch (retryErr) {
                     const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
-                    warnings.push(`${fromLabel}->${toLabel} (${lines.length} edges): ${retryMsg.slice(0, 80)}`);
-                    failedPairEdges += lines.length;
-                    failedPairLines.push(...lines);
+                    warnings.push(`${fromLabel}->${toLabel} (${rows} edges): ${retryMsg.slice(0, 80)}`);
+                    failedPairEdges += rows;
+                    failedPairCsvPaths.add(pairCsvPath);
                 }
             }
-            try {
-                await fs.unlink(pairCsvPath);
+            // Only delete if not in failedPairCsvPaths (needed for fallback)
+            if (!failedPairCsvPaths.has(pairCsvPath)) {
+                try {
+                    await fs.unlink(pairCsvPath);
+                }
+                catch { }
             }
-            catch { }
         }
-        if (failedPairLines.length > 0) {
+        if (failedPairCsvPaths.size > 0) {
             log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`);
-            await fallbackRelationshipInserts([relHeader, ...failedPairLines], validTables, getNodeLabel);
+            // Read failed pair files and merge for fallback inserts
+            const allLines = [relHeader];
+            for (const failedPath of failedPairCsvPaths) {
+                try {
+                    const content = await fs.readFile(failedPath, 'utf-8');
+                    const lines = content.split('\n');
+                    // Skip header line (first) and empty lines
+                    for (let i = 1; i < lines.length; i++) {
+                        if (lines[i].trim())
+                            allLines.push(lines[i]);
+                    }
+                }
+                catch { }
+                try {
+                    await fs.unlink(failedPath);
+                }
+                catch { }
+            }
+            if (allLines.length > 1) {
+                await fallbackRelationshipInserts(allLines, validTables, getNodeLabel);
+            }
         }
     }
     // Cleanup all CSVs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.0",
+  "version": "1.6.2-rc.1",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",
@@ -59,7 +59,6 @@
     "commander": "^12.0.0",
     "cors": "^2.8.5",
     "express": "^4.19.2",
-    "gitnexus-shared": "file:../gitnexus-shared",
     "glob": "^11.0.0",
     "graphology": "^0.25.4",
     "graphology-indices": "^0.17.0",
@@ -85,8 +84,9 @@
     "uuid": "^13.0.0"
   },
   "optionalDependencies": {
-    "tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4",
+    "tree-sitter-dart": "git+https://github.com/UserNobody14/tree-sitter-dart.git#80e23c07b64494f7e21090bb3450223ef0b192f4",
     "tree-sitter-kotlin": "^0.3.8",
+    "tree-sitter-proto": "file:./vendor/tree-sitter-proto",
     "tree-sitter-swift": "^0.6.0"
   },
   "devDependencies": {

package/vendor/tree-sitter-proto/binding.gyp

@@ -0,0 +1,30 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_proto_binding",
+      "dependencies": [
+        "<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
+      ],
+      "include_dirs": [
+        "src",
+      ],
+      "sources": [
+        "bindings/node/binding.cc",
+        "src/parser.c",
+        # NOTE: if your language has an external scanner, add it here.
+      ],
+      "conditions": [
+        ["OS!='win'", {
+          "cflags_c": [
+            "-std=c11",
+          ],
+        }, { # OS == "win"
+          "cflags_c": [
+            "/std:c11",
+            "/utf-8",
+          ],
+        }],
+      ],
+    }
+  ]
+}

package/vendor/tree-sitter-proto/bindings/node/binding.cc

@@ -0,0 +1,20 @@
+#include <napi.h>
+
+typedef struct TSLanguage TSLanguage;
+
+extern "C" TSLanguage *tree_sitter_proto();
+
+// "tree-sitter", "language" hashed with BLAKE2
+const napi_type_tag LANGUAGE_TYPE_TAG = {
+    0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
+};
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+    exports["name"] = Napi::String::New(env, "proto");
+    auto language = Napi::External<TSLanguage>::New(env, tree_sitter_proto());
+    language.TypeTag(&LANGUAGE_TYPE_TAG);
+    exports["language"] = language;
+    return exports;
+}
+
+NODE_API_MODULE(tree_sitter_proto_binding, Init)

package/vendor/tree-sitter-proto/bindings/node/index.d.ts

@@ -0,0 +1,28 @@
+type BaseNode = {
+  type: string;
+  named: boolean;
+};
+
+type ChildNode = {
+  multiple: boolean;
+  required: boolean;
+  types: BaseNode[];
+};
+
+type NodeInfo =
+  | (BaseNode & {
+      subtypes: BaseNode[];
+    })
+  | (BaseNode & {
+      fields: { [name: string]: ChildNode };
+      children: ChildNode[];
+    });
+
+type Language = {
+  name: string;
+  language: unknown;
+  nodeTypeInfo: NodeInfo[];
+};
+
+declare const language: Language;
+export = language;

package/vendor/tree-sitter-proto/bindings/node/index.js

@@ -0,0 +1,7 @@
+const root = require("path").join(__dirname, "..", "..");
+
+module.exports = require("node-gyp-build")(root);
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

package/vendor/tree-sitter-proto/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "tree-sitter-proto",
+  "version": "0.4.1",
+  "description": "tree-sitter grammar for protobuf — ABI 14 build from coder3101/tree-sitter-proto latest grammar.js, compatible with tree-sitter 0.25",
+  "repository": "https://github.com/coder3101/tree-sitter-proto",
+  "license": "MIT",
+  "main": "bindings/node",
+  "scripts": {
+    "install": "node-gyp-build"
+  },
+  "peerDependencies": {
+    "tree-sitter": ">=0.21.0"
+  },
+  "dependencies": {
+    "node-addon-api": "^8.0.0",
+    "node-gyp-build": "^4.8.0"
+  }
+}