gitnexus 1.6.0 → 1.6.1

package/dist/core/lbug/lbug-adapter.js

@@ -1,5 +1,5 @@
 import fs from 'fs/promises';
-import { createReadStream } from 'fs';
+import { createReadStream, createWriteStream } from 'fs';
 import { createInterface } from 'readline';
 import path from 'path';
 import lbug from '@ladybugdb/core';
@@ -215,9 +215,12 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
         }
     }
     // Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
-    // Stream-read the relation CSV line by line to avoid exceeding V8 max string length
+    // Stream-read the relation CSV line by line and write directly to per-pair
+    // temp files on disk. This avoids accumulating potentially millions of CSV
+    // lines in memory which could exceed V8 Map or array limits on large repos.
     let relHeader = '';
-    const relsByPair = new Map();
+    const relsByPairMeta = new Map();
+    const pairWriteStreams = new Map();
     let skippedRels = 0;
     let totalValidRels = 0;
     await new Promise((resolve, reject) => {
@@ -246,33 +249,49 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
                 return;
             }
             const pairKey = `${fromLabel}|${toLabel}`;
-            let list = relsByPair.get(pairKey);
-            if (!list) {
-                list = [];
-                relsByPair.set(pairKey, list);
+            let ws = pairWriteStreams.get(pairKey);
+            if (!ws) {
+                const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
+                ws = createWriteStream(pairCsvPath, 'utf-8');
+                ws.write(relHeader + '\n');
+                pairWriteStreams.set(pairKey, ws);
+                relsByPairMeta.set(pairKey, { csvPath: pairCsvPath, rows: 0 });
             }
-            list.push(line);
+            const ok = ws.write(line + '\n');
+            relsByPairMeta.get(pairKey).rows++;
             totalValidRels++;
+            // Handle backpressure: pause reading when the write buffer is full,
+            // resume when the stream drains. Prevents unbounded memory growth
+            // on repos with millions of relationships.
+            if (!ok) {
+                rl.pause();
+                ws.once('drain', () => rl.resume());
+            }
         });
         rl.on('close', resolve);
-        rl.on('error', reject);
+        rl.on('error', (err) => {
+            // Destroy all open write streams to avoid resource leaks
+            for (const ws of pairWriteStreams.values())
+                ws.destroy();
+            reject(err);
+        });
     });
+    // Close all per-pair write streams before COPY
+    await Promise.all(Array.from(pairWriteStreams.values()).map((ws) => new Promise((resolve, reject) => ws.end((err) => (err ? reject(err) : resolve())))));
     const insertedRels = totalValidRels;
     const warnings = [];
     if (insertedRels > 0) {
-        log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`);
+        log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPairMeta.size} types`);
         let pairIdx = 0;
         let failedPairEdges = 0;
-        const failedPairLines = [];
-        for (const [pairKey, lines] of relsByPair) {
+        const failedPairCsvPaths = new Set();
+        for (const [pairKey, { csvPath: pairCsvPath, rows }] of relsByPairMeta) {
             pairIdx++;
             const [fromLabel, toLabel] = pairKey.split('|');
-            const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
-            await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8');
             const normalizedPath = normalizeCopyPath(pairCsvPath);
             const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
-            if (pairIdx % 5 === 0 || lines.length > 1000) {
-                log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`);
+            if (pairIdx % 5 === 0 || rows > 1000) {
+                log(`Loading edges: ${pairIdx}/${relsByPairMeta.size} types (${fromLabel} -> ${toLabel})`);
             }
             try {
                 await conn.query(copyQuery);
@@ -284,19 +303,42 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
                 }
                 catch (retryErr) {
                     const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
-                    warnings.push(`${fromLabel}->${toLabel} (${lines.length} edges): ${retryMsg.slice(0, 80)}`);
-                    failedPairEdges += lines.length;
-                    failedPairLines.push(...lines);
+                    warnings.push(`${fromLabel}->${toLabel} (${rows} edges): ${retryMsg.slice(0, 80)}`);
+                    failedPairEdges += rows;
+                    failedPairCsvPaths.add(pairCsvPath);
                 }
             }
-            try {
-                await fs.unlink(pairCsvPath);
+            // Only delete if not in failedPairCsvPaths (needed for fallback)
+            if (!failedPairCsvPaths.has(pairCsvPath)) {
+                try {
+                    await fs.unlink(pairCsvPath);
+                }
+                catch { }
             }
-            catch { }
         }
-        if (failedPairLines.length > 0) {
+        if (failedPairCsvPaths.size > 0) {
             log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`);
-            await fallbackRelationshipInserts([relHeader, ...failedPairLines], validTables, getNodeLabel);
+            // Read failed pair files and merge for fallback inserts
+            const allLines = [relHeader];
+            for (const failedPath of failedPairCsvPaths) {
+                try {
+                    const content = await fs.readFile(failedPath, 'utf-8');
+                    const lines = content.split('\n');
+                    // Skip header line (first) and empty lines
+                    for (let i = 1; i < lines.length; i++) {
+                        if (lines[i].trim())
+                            allLines.push(lines[i]);
+                    }
+                }
+                catch { }
+                try {
+                    await fs.unlink(failedPath);
+                }
+                catch { }
+            }
+            if (allLines.length > 1) {
+                await fallbackRelationshipInserts(allLines, validTables, getNodeLabel);
+            }
         }
     }
     // Cleanup all CSVs

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "gitnexus",
-  "version": "1.6.0",
+  "version": "1.6.1",
   "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
   "author": "Abhigyan Patwari",
   "license": "PolyForm-Noncommercial-1.0.0",
@@ -59,7 +59,6 @@
     "commander": "^12.0.0",
     "cors": "^2.8.5",
     "express": "^4.19.2",
-    "gitnexus-shared": "file:../gitnexus-shared",
     "glob": "^11.0.0",
     "graphology": "^0.25.4",
     "graphology-indices": "^0.17.0",
@@ -85,8 +84,9 @@
     "uuid": "^13.0.0"
   },
   "optionalDependencies": {
-    "tree-sitter-dart": "github:UserNobody14/tree-sitter-dart#80e23c07b64494f7e21090bb3450223ef0b192f4",
+    "tree-sitter-dart": "https://github.com/UserNobody14/tree-sitter-dart/archive/80e23c07b64494f7e21090bb3450223ef0b192f4.tar.gz",
     "tree-sitter-kotlin": "^0.3.8",
+    "tree-sitter-proto": "file:./vendor/tree-sitter-proto",
     "tree-sitter-swift": "^0.6.0"
   },
   "devDependencies": {

package/vendor/tree-sitter-proto/binding.gyp

@@ -0,0 +1,30 @@
+{
+  "targets": [
+    {
+      "target_name": "tree_sitter_proto_binding",
+      "dependencies": [
+        "<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
+      ],
+      "include_dirs": [
+        "src",
+      ],
+      "sources": [
+        "bindings/node/binding.cc",
+        "src/parser.c",
+        # NOTE: if your language has an external scanner, add it here.
+      ],
+      "conditions": [
+        ["OS!='win'", {
+          "cflags_c": [
+            "-std=c11",
+          ],
+        }, { # OS == "win"
+          "cflags_c": [
+            "/std:c11",
+            "/utf-8",
+          ],
+        }],
+      ],
+    }
+  ]
+}

package/vendor/tree-sitter-proto/bindings/node/binding.cc

@@ -0,0 +1,20 @@
+#include <napi.h>
+
+typedef struct TSLanguage TSLanguage;
+
+extern "C" TSLanguage *tree_sitter_proto();
+
+// "tree-sitter", "language" hashed with BLAKE2
+const napi_type_tag LANGUAGE_TYPE_TAG = {
+    0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
+};
+
+Napi::Object Init(Napi::Env env, Napi::Object exports) {
+    exports["name"] = Napi::String::New(env, "proto");
+    auto language = Napi::External<TSLanguage>::New(env, tree_sitter_proto());
+    language.TypeTag(&LANGUAGE_TYPE_TAG);
+    exports["language"] = language;
+    return exports;
+}
+
+NODE_API_MODULE(tree_sitter_proto_binding, Init)

package/vendor/tree-sitter-proto/bindings/node/index.d.ts

@@ -0,0 +1,28 @@
+type BaseNode = {
+  type: string;
+  named: boolean;
+};
+
+type ChildNode = {
+  multiple: boolean;
+  required: boolean;
+  types: BaseNode[];
+};
+
+type NodeInfo =
+  | (BaseNode & {
+      subtypes: BaseNode[];
+    })
+  | (BaseNode & {
+      fields: { [name: string]: ChildNode };
+      children: ChildNode[];
+    });
+
+type Language = {
+  name: string;
+  language: unknown;
+  nodeTypeInfo: NodeInfo[];
+};
+
+declare const language: Language;
+export = language;

package/vendor/tree-sitter-proto/bindings/node/index.js

@@ -0,0 +1,7 @@
+const root = require("path").join(__dirname, "..", "..");
+
+module.exports = require("node-gyp-build")(root);
+
+try {
+  module.exports.nodeTypeInfo = require("../../src/node-types.json");
+} catch (_) {}

package/vendor/tree-sitter-proto/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "tree-sitter-proto",
+  "version": "0.4.1",
+  "description": "tree-sitter grammar for protobuf — ABI 14 build from coder3101/tree-sitter-proto latest grammar.js, compatible with tree-sitter 0.25",
+  "repository": "https://github.com/coder3101/tree-sitter-proto",
+  "license": "MIT",
+  "main": "bindings/node",
+  "scripts": {
+    "install": "node-gyp-build"
+  },
+  "peerDependencies": {
+    "tree-sitter": ">=0.21.0"
+  },
+  "dependencies": {
+    "node-addon-api": "^8.0.0",
+    "node-gyp-build": "^4.8.0"
+  }
+}