gitnexus 1.5.3 → 1.6.1

package/dist/core/lbug/lbug-adapter.d.ts

@@ -49,6 +49,7 @@ export declare const batchInsertNodesToLbug: (nodes: Array<{
     failed: number;
 }>;
 export declare const executeQuery: (cypher: string) => Promise<any[]>;
+export declare const streamQuery: (cypher: string, onRow: (row: any) => void | Promise<void>) => Promise<number>;
 /**
  * Execute a single parameterized query (prepare/execute pattern).
  * Prevents Cypher injection by binding values as parameters.
@@ -88,6 +89,11 @@ export declare const getEmbeddingTableName: () => string;
  * Safe to call multiple times — tracks loaded state via module-level ftsLoaded.
  */
 export declare const loadFTSExtension: () => Promise<void>;
+/**
+ * Load the VECTOR extension (required before using QUERY_VECTOR_INDEX).
+ * Safe to call multiple times -- tracks loaded state via module-level vectorExtensionLoaded.
+ */
+export declare const loadVectorExtension: () => Promise<void>;
 /**
  * Create a full-text search index on a table
  * @param tableName - The node table name (e.g., 'File', 'CodeSymbol')

package/dist/core/lbug/lbug-adapter.js

@@ -1,5 +1,5 @@
 import fs from 'fs/promises';
-import { createReadStream } from 'fs';
+import { createReadStream, createWriteStream } from 'fs';
 import { createInterface } from 'readline';
 import path from 'path';
 import lbug from '@ladybugdb/core';
@@ -9,6 +9,7 @@ let db = null;
 let conn = null;
 let currentDbPath = null;
 let ftsLoaded = false;
+let vectorExtensionLoaded = false;
 /** Expose the current Database for pool adapter reuse in tests. */
 export const getDatabase = () => db;
 // Global session lock for operations that touch module-level lbug globals.
@@ -91,6 +92,7 @@ export const withLbugDb = async (dbPath, operation) => {
                 db = null;
                 currentDbPath = null;
                 ftsLoaded = false;
+                vectorExtensionLoaded = false;
             });
             // Sleep outside the lock — no need to block others while waiting
             await new Promise((resolve) => setTimeout(resolve, DB_LOCK_RETRY_DELAY_MS * attempt));
@@ -124,6 +126,7 @@ const doInitLbug = async (dbPath) => {
         db = null;
         currentDbPath = null;
         ftsLoaded = false;
+        vectorExtensionLoaded = false;
     }
     // LadybugDB stores the database as a single file (not a directory).
     // If the path already exists, it must be a valid LadybugDB database file.
@@ -167,6 +170,8 @@ const doInitLbug = async (dbPath) => {
             }
         }
     }
+    // Load VECTOR extension for semantic search support
+    await loadVectorExtension();
     currentDbPath = dbPath;
     return { db, conn };
 };
@@ -210,9 +215,12 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
         }
     }
     // Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
-    // Stream-read the relation CSV line by line to avoid exceeding V8 max string length
+    // Stream-read the relation CSV line by line and write directly to per-pair
+    // temp files on disk. This avoids accumulating potentially millions of CSV
+    // lines in memory which could exceed V8 Map or array limits on large repos.
     let relHeader = '';
-    const relsByPair = new Map();
+    const relsByPairMeta = new Map();
+    const pairWriteStreams = new Map();
     let skippedRels = 0;
     let totalValidRels = 0;
     await new Promise((resolve, reject) => {
@@ -241,33 +249,49 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
                 return;
             }
             const pairKey = `${fromLabel}|${toLabel}`;
-            let list = relsByPair.get(pairKey);
-            if (!list) {
-                list = [];
-                relsByPair.set(pairKey, list);
+            let ws = pairWriteStreams.get(pairKey);
+            if (!ws) {
+                const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
+                ws = createWriteStream(pairCsvPath, 'utf-8');
+                ws.write(relHeader + '\n');
+                pairWriteStreams.set(pairKey, ws);
+                relsByPairMeta.set(pairKey, { csvPath: pairCsvPath, rows: 0 });
             }
-            list.push(line);
+            const ok = ws.write(line + '\n');
+            relsByPairMeta.get(pairKey).rows++;
             totalValidRels++;
+            // Handle backpressure: pause reading when the write buffer is full,
+            // resume when the stream drains. Prevents unbounded memory growth
+            // on repos with millions of relationships.
+            if (!ok) {
+                rl.pause();
+                ws.once('drain', () => rl.resume());
+            }
         });
         rl.on('close', resolve);
-        rl.on('error', reject);
+        rl.on('error', (err) => {
+            // Destroy all open write streams to avoid resource leaks
+            for (const ws of pairWriteStreams.values())
+                ws.destroy();
+            reject(err);
+        });
     });
+    // Close all per-pair write streams before COPY
+    await Promise.all(Array.from(pairWriteStreams.values()).map((ws) => new Promise((resolve, reject) => ws.end((err) => (err ? reject(err) : resolve())))));
     const insertedRels = totalValidRels;
     const warnings = [];
     if (insertedRels > 0) {
-        log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPair.size} types`);
+        log(`Loading edges: ${insertedRels.toLocaleString()} across ${relsByPairMeta.size} types`);
         let pairIdx = 0;
         let failedPairEdges = 0;
-        const failedPairLines = [];
-        for (const [pairKey, lines] of relsByPair) {
+        const failedPairCsvPaths = new Set();
+        for (const [pairKey, { csvPath: pairCsvPath, rows }] of relsByPairMeta) {
             pairIdx++;
             const [fromLabel, toLabel] = pairKey.split('|');
-            const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
-            await fs.writeFile(pairCsvPath, relHeader + '\n' + lines.join('\n'), 'utf-8');
             const normalizedPath = normalizeCopyPath(pairCsvPath);
             const copyQuery = `COPY ${REL_TABLE_NAME} FROM "${normalizedPath}" (from="${fromLabel}", to="${toLabel}", HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
-            if (pairIdx % 5 === 0 || lines.length > 1000) {
-                log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`);
+            if (pairIdx % 5 === 0 || rows > 1000) {
+                log(`Loading edges: ${pairIdx}/${relsByPairMeta.size} types (${fromLabel} -> ${toLabel})`);
             }
             try {
                 await conn.query(copyQuery);
@@ -279,19 +303,42 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
                 }
                 catch (retryErr) {
                     const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
-                    warnings.push(`${fromLabel}->${toLabel} (${lines.length} edges): ${retryMsg.slice(0, 80)}`);
-                    failedPairEdges += lines.length;
-                    failedPairLines.push(...lines);
+                    warnings.push(`${fromLabel}->${toLabel} (${rows} edges): ${retryMsg.slice(0, 80)}`);
+                    failedPairEdges += rows;
+                    failedPairCsvPaths.add(pairCsvPath);
                 }
             }
-            try {
-                await fs.unlink(pairCsvPath);
+            // Only delete if not in failedPairCsvPaths (needed for fallback)
+            if (!failedPairCsvPaths.has(pairCsvPath)) {
+                try {
+                    await fs.unlink(pairCsvPath);
+                }
+                catch { }
             }
-            catch { }
         }
-        if (failedPairLines.length > 0) {
+        if (failedPairCsvPaths.size > 0) {
             log(`Inserting ${failedPairEdges} edges individually (missing schema pairs)`);
-            await fallbackRelationshipInserts([relHeader, ...failedPairLines], validTables, getNodeLabel);
+            // Read failed pair files and merge for fallback inserts
+            const allLines = [relHeader];
+            for (const failedPath of failedPairCsvPaths) {
+                try {
+                    const content = await fs.readFile(failedPath, 'utf-8');
+                    const lines = content.split('\n');
+                    // Skip header line (first) and empty lines
+                    for (let i = 1; i < lines.length; i++) {
+                        if (lines[i].trim())
+                            allLines.push(lines[i]);
+                    }
+                }
+                catch { }
+                try {
+                    await fs.unlink(failedPath);
+                }
+                catch { }
+            }
+            if (allLines.length > 1) {
+                await fallbackRelationshipInserts(allLines, validTables, getNodeLabel);
+            }
         }
     }
     // Cleanup all CSVs
@@ -589,6 +636,30 @@ export const executeQuery = async (cypher) => {
     const rows = await result.getAll();
     return rows;
 };
+export const streamQuery = async (cypher, onRow) => {
+    if (!conn) {
+        throw new Error('LadybugDB not initialized. Call initLbug first.');
+    }
+    const queryResult = await conn.query(cypher);
+    const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
+    let rowCount = 0;
+    try {
+        while (await result.hasNext()) {
+            const row = await result.getNext();
+            await onRow(row);
+            rowCount++;
+        }
+        return rowCount;
+    }
+    finally {
+        try {
+            await result.close();
+        }
+        catch {
+            // Best-effort cleanup only.
+        }
+    }
+};
 /**
  * Execute a single parameterized query (prepare/execute pattern).
  * Prevents Cypher injection by binding values as parameters.
@@ -715,6 +786,7 @@ export const closeLbug = async () => {
     }
     currentDbPath = null;
     ftsLoaded = false;
+    vectorExtensionLoaded = false;
 };
 export const isLbugReady = () => conn !== null && db !== null;
 /**
@@ -803,19 +875,54 @@ export const loadFTSExtension = async () => {
         throw new Error('LadybugDB not initialized. Call initLbug first.');
     }
     try {
-        await conn.query('INSTALL fts');
+        // Try loading locally first (no network required)
         await conn.query('LOAD EXTENSION fts');
         ftsLoaded = true;
     }
+    catch {
+        // Fall back to install + load (requires network)
+        try {
+            await conn.query('INSTALL fts');
+            await conn.query('LOAD EXTENSION fts');
+            ftsLoaded = true;
+        }
+        catch (err) {
+            const msg = err?.message || '';
+            if (msg.includes('already loaded') ||
+                msg.includes('already installed') ||
+                msg.includes('already exists')) {
+                ftsLoaded = true;
+            }
+            else {
+                console.error('GitNexus: FTS extension load failed:', msg);
+            }
+        }
+    }
+};
+/**
+ * Load the VECTOR extension (required before using QUERY_VECTOR_INDEX).
+ * Safe to call multiple times -- tracks loaded state via module-level vectorExtensionLoaded.
+ */
+export const loadVectorExtension = async () => {
+    if (vectorExtensionLoaded)
+        return;
+    if (!conn) {
+        throw new Error('LadybugDB not initialized. Call initLbug first.');
+    }
+    try {
+        await conn.query('INSTALL VECTOR');
+        await conn.query('LOAD EXTENSION VECTOR');
+        vectorExtensionLoaded = true;
+    }
     catch (err) {
         const msg = err?.message || '';
         if (msg.includes('already loaded') ||
             msg.includes('already installed') ||
             msg.includes('already exists')) {
-            ftsLoaded = true;
+            vectorExtensionLoaded = true;
         }
         else {
-            console.error('GitNexus: FTS extension load failed:', msg);
+            console.error('GitNexus: VECTOR extension load failed:', msg);
         }
     }
 };

package/dist/core/lbug/pool-adapter.d.ts

@@ -0,0 +1,76 @@
+/**
+ * LadybugDB connection pool (core). Used by MCP, sync, search, wiki, etc.
+ *
+ * LadybugDB Adapter (Connection Pool)
+ *
+ * Manages a pool of LadybugDB databases keyed by repoId, each with
+ * multiple Connection objects for safe concurrent query execution.
+ *
+ * LadybugDB Connections are NOT thread-safe — a single Connection
+ * segfaults if concurrent .query() calls hit it simultaneously.
+ * This adapter provides a checkout/return connection pool so each
+ * concurrent query gets its own Connection from the same Database.
+ *
+ * @see https://docs.ladybugdb.com/concurrency — multiple Connections
+ * from the same Database is the officially supported concurrency pattern.
+ */
+import lbug from '@ladybugdb/core';
+/** Saved real stdout/stderr write — used to silence native module output without race conditions */
+export declare const realStdoutWrite: any;
+export declare const realStderrWrite: any;
+/**
+ * Touch a repo to reset its idle timeout.
+ * Call this during long-running operations to prevent the connection from being closed.
+ */
+export declare const touchRepo: (repoId: string) => void;
+/**
+ * Silence stdout by replacing process.stdout.write with a no-op.
+ * Uses a reference counter so nested silence/restore pairs are safe.
+ * Exported so other modules (e.g. embedder) use the same mechanism instead
+ * of independently patching stdout, which causes restore-order conflicts.
+ */
+export declare function silenceStdout(): void;
+export declare function restoreStdout(): void;
+/**
+ * Initialize (or reuse) a Database + connection pool for a specific repo.
+ * Retries on lock errors (e.g., when `gitnexus analyze` is running).
+ *
+ * Concurrent calls for the same repoId are deduplicated — the second caller
+ * awaits the first's in-progress init rather than starting a redundant one.
+ */
+export declare const initLbug: (repoId: string, dbPath: string) => Promise<void>;
+/**
+ * Initialize a pool entry from a pre-existing Database object.
+ *
+ * Used in tests to avoid the writable→close→read-only cycle that crashes
+ * on macOS due to N-API destructor segfaults.  The pool adapter reuses
+ * the core adapter's writable Database instead of opening a new read-only one.
+ *
+ * The Database is registered in the shared dbCache so closeOne() decrements
+ * the refCount correctly.  If the Database is already cached (e.g. another
+ * repoId already injected it), the existing entry is reused.
+ */
+export declare function initLbugWithDb(repoId: string, existingDb: lbug.Database, dbPath: string): Promise<void>;
+export declare const executeQuery: (repoId: string, cypher: string) => Promise<any[]>;
+/**
+ * Execute a parameterized query on a specific repo's connection pool.
+ * Uses prepare/execute pattern to prevent Cypher injection.
+ */
+export declare const executeParameterized: (repoId: string, cypher: string, params: Record<string, any>) => Promise<any[]>;
+/**
+ * Close one or all repo pools.
+ * If repoId is provided, close only that repo's connections.
+ * If omitted, close all repos.
+ */
+export declare const closeLbug: (repoId?: string) => Promise<void>;
+/**
+ * Check if a specific repo's pool is active
+ */
+export declare const isLbugReady: (repoId: string) => boolean;
+/** Regex to detect write operations in user-supplied Cypher queries.
+ * Note: CALL is NOT blocked — it's used for read-only FTS (CALL QUERY_FTS_INDEX)
+ * and vector search (CALL QUERY_VECTOR_INDEX). The database is opened in
+ * read-only mode as defense-in-depth against write procedures. */
+export declare const CYPHER_WRITE_RE: RegExp;
+/** Check if a Cypher query contains write operations */
+export declare function isWriteQuery(query: string): boolean;