gitnexus 1.6.2-rc.2 → 1.6.2-rc.21

package/dist/core/lbug/lbug-adapter.js

@@ -1,9 +1,11 @@
 import fs from 'fs/promises';
 import { createReadStream, createWriteStream } from 'fs';
 import { createInterface } from 'readline';
+import { once } from 'events';
+import { finished } from 'stream/promises';
 import path from 'path';
 import lbug from '@ladybugdb/core';
-import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
+import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, STALE_HASH_SENTINEL, } from './schema.js';
 import { streamAllCSVsToDisk } from './csv-generator.js';
 /**
  * Split a relationship CSV into per-label-pair files on disk.
@@ -25,100 +27,82 @@ export const splitRelCsvByLabelPair = async (csvPath, csvDir, validTables, getNo
     const pairWriteStreams = new Map();
     let skippedRels = 0;
     let totalValidRels = 0;
-    await new Promise((resolve, reject) => {
-        const inputStream = createReadStream(csvPath, 'utf-8');
-        const rl = createInterface({
-            input: inputStream,
-            crlfDelay: Infinity,
-        });
-        // Track which streams are already waiting for drain to prevent
-        // listener accumulation. rl.pause() is not synchronous — buffered
-        // line events continue firing after pause(), and without this guard
-        // each line targeting the same pairKey would add another drain listener.
-        const waitingForDrain = new Set();
-        let settled = false;
-        const cleanup = (err) => {
-            if (settled)
-                return;
-            settled = true;
-            try {
-                rl.close();
-            }
-            catch { }
-            try {
-                inputStream.destroy();
-            }
-            catch { }
-            for (const ws of pairWriteStreams.values()) {
-                try {
-                    ws.destroy();
-                }
-                catch { }
-            }
-            reject(err);
-        };
+    const inputStream = createReadStream(csvPath, 'utf-8');
+    const rl = createInterface({ input: inputStream, crlfDelay: Infinity });
+    // If any pair WriteStream errors (disk full, EMFILE, etc.) or the input
+    // stream fails, we need to abort the pending `once(ws, 'drain')` await.
+    // An AbortController gives us one signal to cancel all pending waits
+    // without a custom state machine.
+    const abortOnError = new AbortController();
+    let streamError = null;
+    const markStreamError = (err) => {
+        streamError ??= err;
+        abortOnError.abort(err);
+    };
+    try {
+        // `for await (const line of rl)` replaces the old manual
+        // on('line')/pause()/resume()/waitingForDrain state machine: readline's
+        // async iterator naturally serializes line delivery with our awaits, so
+        // at most one ws can be in backpressure at a time and we just await its
+        // 'drain' event.
         let isFirst = true;
-        rl.on('line', (line) => {
+        for await (const line of rl) {
+            if (streamError)
+                throw streamError;
             if (isFirst) {
                 relHeader = line;
                 isFirst = false;
-                return;
+                continue;
             }
             if (!line.trim())
-                return;
+                continue;
             const match = line.match(/"([^"]*)","([^"]*)"/);
             if (!match) {
                 skippedRels++;
-                return;
+                continue;
             }
             const fromLabel = getNodeLabel(match[1]);
             const toLabel = getNodeLabel(match[2]);
             if (!validTables.has(fromLabel) || !validTables.has(toLabel)) {
                 skippedRels++;
-                return;
+                continue;
             }
             const pairKey = `${fromLabel}|${toLabel}`;
             let ws = pairWriteStreams.get(pairKey);
             if (!ws) {
                 const pairCsvPath = path.join(csvDir, `rel_${fromLabel}_${toLabel}.csv`);
                 ws = wsFactory(pairCsvPath);
-                // If any per-pair WriteStream errors (disk full, EMFILE, etc.),
-                // tear down everything and reject the Promise. Without this handler,
-                // a stream error while rl is paused waiting for drain would cause
-                // the drain callback to never fire and the Promise to hang forever.
-                ws.on('error', cleanup);
-                ws.write(relHeader + '\n');
+                ws.on('error', markStreamError);
                 pairWriteStreams.set(pairKey, ws);
                 relsByPairMeta.set(pairKey, { csvPath: pairCsvPath, rows: 0 });
+                if (!ws.write(relHeader + '\n')) {
+                    await once(ws, 'drain', { signal: abortOnError.signal });
+                }
+            }
+            if (!ws.write(line + '\n')) {
+                await once(ws, 'drain', { signal: abortOnError.signal });
             }
-            const ok = ws.write(line + '\n');
             relsByPairMeta.get(pairKey).rows++;
             totalValidRels++;
-            // Handle backpressure: pause reading when the write buffer is full,
-            // resume when the stream drains. Prevents unbounded memory growth
-            // on repos with millions of relationships.
-            // Guard with waitingForDrain to ensure only one drain listener is
-            // registered per stream at a time — rl.pause() doesn't stop buffered
-            // line events immediately. Only resume when ALL streams have drained
-            // to avoid writing into still-full streams.
-            if (!ok && !waitingForDrain.has(pairKey)) {
-                waitingForDrain.add(pairKey);
-                rl.pause();
-                ws.once('drain', () => {
-                    waitingForDrain.delete(pairKey);
-                    if (waitingForDrain.size === 0)
-                        rl.resume();
-                });
-            }
-        });
-        rl.on('close', () => {
-            if (!settled) {
-                settled = true;
-                resolve();
-            }
-        });
-        rl.on('error', cleanup);
-    });
+        }
+        if (streamError)
+            throw streamError;
+    }
+    catch (err) {
+        // Tear down everything so no fd is left dangling. If the abort was caused
+        // by a stream error, rethrow that error (more actionable than AbortError).
+        for (const ws of pairWriteStreams.values())
+            ws.destroy();
+        inputStream.destroy();
+        throw streamError ?? err;
+    }
+    finally {
+        // Readline 'close' fires before the underlying fs.ReadStream releases its
+        // fd — on Windows that race caused ENOTEMPTY on the parent dir.
+        // stream/promises.finished is the stdlib "wait until this stream is fully
+        // closed" primitive and handles both success and error paths.
+        await finished(inputStream).catch(() => { });
+    }
     return { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels };
 };
 let db = null;
@@ -126,6 +110,14 @@ let conn = null;
 let currentDbPath = null;
 let ftsLoaded = false;
 let vectorExtensionLoaded = false;
+/**
+ * Check if an error indicates a missing column or table (schema-level problem)
+ * rather than a transient/connection error. Used for legacy DB fallback logic.
+ */
+const isMissingColumnOrTableError = (msg) => msg.includes('does not exist') ||
+    // Kuzu-specific: "(table|column|property) ... not found" — narrow enough to avoid
+    // matching transient errors like "connection not found" or "key not found".
+    /(table|column|property).*not found/i.test(msg);
 /** Expose the current Database for pool adapter reuse in tests. */
 export const getDatabase = () => db;
 // Global session lock for operations that touch module-level lbug globals.
@@ -332,15 +324,13 @@ export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress)
     }
     // Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
     const { relHeader, relsByPairMeta, pairWriteStreams, skippedRels, totalValidRels } = await splitRelCsvByLabelPair(csvResult.relCsvPath, csvDir, validTables, getNodeLabel);
-    // Close all per-pair write streams before COPY
-    await Promise.all(Array.from(pairWriteStreams.values()).map((ws) => new Promise((resolve, reject) => {
-        const onError = (err) => reject(err);
-        ws.on('error', onError);
-        ws.end(() => {
-            ws.removeListener('error', onError);
-            resolve();
-        });
-    })));
+    // Close all per-pair write streams before COPY. `stream/promises.finished`
+    // resolves on the stream's 'finish' event and rejects on 'error' — replaces
+    // a hand-rolled promisification with the stdlib primitive.
+    await Promise.all(Array.from(pairWriteStreams.values()).map(async (ws) => {
+        ws.end();
+        await finished(ws);
+    }));
     const insertedRels = totalValidRels;
     const warnings = [];
     if (insertedRels > 0) {
@@ -801,6 +791,8 @@ export const getLbugStats = async () => {
  * Load cached embeddings from LadybugDB before a rebuild.
  * Returns all embedding vectors so they can be re-inserted after the graph is reloaded,
  * avoiding expensive re-embedding of unchanged nodes.
+ *
+ * Detects old schema (no chunkIndex column) and returns empty cache to trigger rebuild.
  */
 export const loadCachedEmbeddings = async () => {
     if (!conn) {
@@ -809,20 +801,51 @@ export const loadCachedEmbeddings = async () => {
     const embeddingNodeIds = new Set();
     const embeddings = [];
     try {
-        const rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.embedding AS embedding`);
+        // Schema migration detection: query with new columns to verify schema version.
+        // Old schema only had (nodeId, embedding); new schema adds (id, chunkIndex, startLine, endLine, contentHash).
+        // If the query fails (column missing), we return empty cache to force a full rebuild.
+        try {
+            const check = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex LIMIT 1`);
+            const checkResult = Array.isArray(check) ? check[0] : check;
+            await checkResult.getAll();
+        }
+        catch {
+            return { embeddingNodeIds: new Set(), embeddings: [] };
+        }
+        // Try to read contentHash alongside chunk columns
+        let rows;
+        let hasContentHash = true;
+        try {
+            rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding, e.contentHash AS contentHash`);
+        }
+        catch (err) {
+            // Fallback for legacy DBs without contentHash column
+            const msg = err?.message ?? '';
+            if (isMissingColumnOrTableError(msg)) {
+                hasContentHash = false;
+                rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding`);
+            }
+            else {
+                throw err;
+            }
+        }
         const result = Array.isArray(rows) ? rows[0] : rows;
         for (const row of await result.getAll()) {
             const nodeId = String(row.nodeId ?? row[0] ?? '');
             if (!nodeId)
                 continue;
             embeddingNodeIds.add(nodeId);
-            const embedding = row.embedding ?? row[1];
+            const embedding = row.embedding ?? row[4];
             if (embedding) {
                 embeddings.push({
                     nodeId,
+                    chunkIndex: Number(row.chunkIndex ?? row[1] ?? 0),
+                    startLine: Number(row.startLine ?? row[2] ?? 0),
+                    endLine: Number(row.endLine ?? row[3] ?? 0),
                     embedding: Array.isArray(embedding)
                         ? embedding.map(Number)
                         : Array.from(embedding).map(Number),
+                    contentHash: hasContentHash ? (row.contentHash ?? row[5] ?? undefined) : undefined,
                 });
             }
         }
@@ -832,6 +855,68 @@ export const loadCachedEmbeddings = async () => {
     }
     return { embeddingNodeIds, embeddings };
 };
+/**
+ * Fetch existing embedding hashes from CodeEmbedding table for incremental embedding.
+ * Returns a Map<nodeId, contentHash> suitable for passing to `runEmbeddingPipeline`.
+ * Handles legacy DBs without the `contentHash` column (all rows treated as stale with empty hash).
+ * Returns undefined if the CodeEmbedding table does not exist.
+ *
+ * @param execQuery - Cypher query executor (typically pool-adapter's `executeQuery`)
+ */
+export const fetchExistingEmbeddingHashes = async (execQuery) => {
+    try {
+        const rows = await execQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.contentHash AS contentHash`);
+        if (!rows || rows.length === 0)
+            return undefined;
+        const map = new Map();
+        for (const r of rows) {
+            const nodeId = r.nodeId ?? r[0];
+            const chunkIndex = r.chunkIndex ?? r[1];
+            const startLine = r.startLine ?? r[2];
+            const endLine = r.endLine ?? r[3];
+            const hash = r.contentHash ?? r[4] ?? STALE_HASH_SENTINEL;
+            if (nodeId) {
+                const hasChunkMetadata = chunkIndex !== undefined &&
+                    chunkIndex !== null &&
+                    startLine !== undefined &&
+                    startLine !== null &&
+                    endLine !== undefined &&
+                    endLine !== null;
+                // Empty/null contentHash or missing chunk metadata means legacy row — treat as stale.
+                map.set(nodeId, hasChunkMetadata && hash ? hash : STALE_HASH_SENTINEL);
+            }
+        }
+        return map;
+    }
+    catch (err) {
+        const msg = err?.message ?? '';
+        if (isMissingColumnOrTableError(msg)) {
+            // Legacy rows missing chunk-aware columns — treat every row as stale.
+            try {
+                const rows = await execQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId`);
+                if (!rows || rows.length === 0)
+                    return undefined;
+                const map = new Map();
+                for (const r of rows) {
+                    const nodeId = r.nodeId ?? r[0];
+                    if (nodeId)
+                        map.set(nodeId, STALE_HASH_SENTINEL);
+                }
+                console.log(`[embed] ${map.size} nodes in legacy DB (missing chunk-aware columns) — all treated as stale`);
+                return map;
+            }
+            catch (fallbackErr) {
+                const fallbackMsg = fallbackErr?.message ?? '';
+                if (isMissingColumnOrTableError(fallbackMsg)) {
+                    console.log(`[embed] CodeEmbedding table not yet present — full embedding run (${fallbackMsg})`);
+                    return undefined;
+                }
+                throw fallbackErr;
+            }
+        }
+        throw err;
+    }
+};
 export const closeLbug = async () => {
     if (conn) {
         try {

package/dist/core/lbug/schema.d.ts

@@ -31,6 +31,7 @@ export declare const IMPL_SCHEMA: string;
 export declare const TYPE_ALIAS_SCHEMA: string;
 export declare const CONST_SCHEMA: string;
 export declare const STATIC_SCHEMA: string;
+export declare const VARIABLE_SCHEMA: string;
 export declare const PROPERTY_SCHEMA: string;
 export declare const RECORD_SCHEMA: string;
 export declare const DELEGATE_SCHEMA: string;
@@ -41,8 +42,15 @@ export declare const MODULE_SCHEMA: string;
 export declare const ROUTE_SCHEMA = "\nCREATE NODE TABLE Route (\n  id STRING,\n  name STRING,\n  filePath STRING,\n  responseKeys STRING[],\n  errorKeys STRING[],\n  middleware STRING[],\n  PRIMARY KEY (id)\n)";
 export declare const TOOL_SCHEMA = "\nCREATE NODE TABLE Tool (\n  id STRING,\n  name STRING,\n  filePath STRING,\n  description STRING,\n  PRIMARY KEY (id)\n)";
 export declare const SECTION_SCHEMA = "\nCREATE NODE TABLE Section (\n  id STRING,\n  name STRING,\n  filePath STRING,\n  startLine INT64,\n  endLine INT64,\n  level INT64,\n  content STRING,\n  description STRING,\n  PRIMARY KEY (id)\n)";
-export declare const RELATION_SCHEMA = "\nCREATE REL TABLE CodeRelation (\n  FROM File TO File,\n  FROM File TO Folder,\n  FROM File TO Function,\n  FROM File TO Class,\n  FROM File TO Interface,\n  FROM File TO Method,\n  FROM File TO CodeElement,\n  FROM File TO `Struct`,\n  FROM File TO `Enum`,\n  FROM File TO `Macro`,\n  FROM File TO `Typedef`,\n  FROM File TO `Union`,\n  FROM File TO `Namespace`,\n  FROM File TO `Trait`,\n  FROM File TO `Impl`,\n  FROM File TO `TypeAlias`,\n  FROM File TO `Const`,\n  FROM File TO `Static`,\n  FROM File TO `Property`,\n  FROM File TO `Record`,\n  FROM File TO `Delegate`,\n  FROM File TO `Annotation`,\n  FROM File TO `Constructor`,\n  FROM File TO `Template`,\n  FROM File TO `Module`,\n  FROM File TO Section,\n  FROM Folder TO Folder,\n  FROM Folder TO File,\n  FROM Function TO Function,\n  FROM Function TO Method,\n  FROM Function TO Class,\n  FROM Function TO Community,\n  FROM Function TO `Macro`,\n  FROM Function TO `Struct`,\n  FROM Function TO `Template`,\n  FROM Function TO `Enum`,\n  FROM Function TO `Namespace`,\n  FROM Function TO `TypeAlias`,\n  FROM Function TO `Module`,\n  FROM Function TO `Impl`,\n  FROM Function TO Interface,\n  FROM Function TO `Constructor`,\n  FROM Function TO `Const`,\n  FROM Function TO `Typedef`,\n  FROM Function TO `Union`,\n  FROM Function TO `Property`,\n  FROM Function TO CodeElement,\n  FROM Class TO Method,\n  FROM Class TO Function,\n  FROM Class TO Class,\n  FROM Class TO Interface,\n  FROM Class TO Community,\n  FROM Class TO `Template`,\n  FROM Class TO `TypeAlias`,\n  FROM Class TO `Struct`,\n  FROM Class TO `Enum`,\n  FROM Class TO `Annotation`,\n  FROM Class TO `Constructor`,\n  FROM Class TO `Trait`,\n  FROM Class TO `Macro`,\n  FROM Class TO `Impl`,\n  FROM Class TO `Union`,\n  FROM Class TO `Namespace`,\n  FROM Class TO `Typedef`,\n  FROM Class TO `Property`,\n  FROM Method TO Function,\n  FROM Method TO Method,\n  FROM Method TO Class,\n  FROM Method TO Community,\n  FROM Method TO `Template`,\n  FROM Method TO `Struct`,\n  FROM Method TO `TypeAlias`,\n  FROM Method TO `Enum`,\n  FROM Method TO `Macro`,\n  FROM Method TO `Namespace`,\n  FROM Method TO `Module`,\n  FROM Method TO `Impl`,\n  FROM Method TO Interface,\n  FROM Method TO `Constructor`,\n  FROM Method TO `Property`,\n  FROM Method TO CodeElement,\n  FROM `Template` TO `Template`,\n  FROM `Template` TO Function,\n  FROM `Template` TO Method,\n  FROM `Template` TO Class,\n  FROM `Template` TO `Struct`,\n  FROM `Template` TO `TypeAlias`,\n  FROM `Template` TO `Enum`,\n  FROM `Template` TO `Macro`,\n  FROM `Template` TO Interface,\n  FROM `Template` TO `Constructor`,\n  FROM `Module` TO `Module`,\n  FROM Section TO Section,\n  FROM Section TO File,\n  FROM File TO Route,\n  FROM Function TO Route,\n  FROM Method TO Route,\n  FROM File TO Tool,\n  FROM Function TO Tool,\n  FROM Method TO Tool,\n  FROM CodeElement TO Community,\n  FROM Interface TO Community,\n  FROM Interface TO Function,\n  FROM Interface TO Method,\n  FROM Interface TO Class,\n  FROM Interface TO Interface,\n  FROM Interface TO `TypeAlias`,\n  FROM Interface TO `Struct`,\n  FROM Interface TO `Constructor`,\n  FROM Interface TO `Property`,\n  FROM `Struct` TO Community,\n  FROM `Struct` TO `Trait`,\n  FROM `Struct` TO `Struct`,\n  FROM `Struct` TO Class,\n  FROM `Struct` TO `Enum`,\n  FROM `Struct` TO Function,\n  FROM `Struct` TO Method,\n  FROM `Struct` TO Interface,\n  FROM `Struct` TO `Constructor`,\n  FROM `Struct` TO `Property`,\n  FROM `Enum` TO `Enum`,\n  FROM `Enum` TO Community,\n  FROM `Enum` TO Class,\n  FROM `Enum` TO Interface,\n  FROM `Macro` TO Community,\n  FROM `Macro` TO Function,\n  FROM `Macro` TO Method,\n  FROM `Module` TO Function,\n  FROM `Module` TO Method,\n  FROM `Typedef` TO Community,\n  FROM `Union` TO Community,\n  FROM `Namespace` TO Community,\n  FROM `Namespace` TO `Struct`,\n  FROM `Trait` TO Method,\n  FROM `Trait` TO `Constructor`,\n  FROM `Trait` TO `Property`,\n  FROM `Trait` TO Community,\n  FROM `Impl` TO Method,\n  FROM `Impl` TO `Constructor`,\n  FROM `Impl` TO `Property`,\n  FROM `Impl` TO Community,\n  FROM `Impl` TO `Trait`,\n  FROM `Impl` TO `Struct`,\n  FROM `Impl` TO `Impl`,\n  FROM `TypeAlias` TO Community,\n  FROM `TypeAlias` TO `Trait`,\n  FROM `TypeAlias` TO Class,\n  FROM `Const` TO Community,\n  FROM `Static` TO Community,\n  FROM `Property` TO Community,\n  FROM `Record` TO Method,\n  FROM `Record` TO `Constructor`,\n  FROM `Record` TO `Property`,\n  FROM `Record` TO Community,\n  FROM `Delegate` TO Community,\n  FROM `Annotation` TO Community,\n  FROM `Constructor` TO Community,\n  FROM `Constructor` TO Interface,\n  FROM `Constructor` TO Class,\n  FROM `Constructor` TO Method,\n  FROM `Constructor` TO Function,\n  FROM `Constructor` TO `Constructor`,\n  FROM `Constructor` TO `Struct`,\n  FROM `Constructor` TO `Macro`,\n  FROM `Constructor` TO `Template`,\n  FROM `Constructor` TO `TypeAlias`,\n  FROM `Constructor` TO `Enum`,\n  FROM `Constructor` TO `Annotation`,\n  FROM `Constructor` TO `Impl`,\n  FROM `Constructor` TO `Namespace`,\n  FROM `Constructor` TO `Module`,\n  FROM `Constructor` TO `Property`,\n  FROM `Constructor` TO `Typedef`,\n  FROM `Template` TO Community,\n  FROM `Module` TO Community,\n  FROM Function TO Process,\n  FROM Method TO Process,\n  FROM Class TO Process,\n  FROM Interface TO Process,\n  FROM `Struct` TO Process,\n  FROM `Constructor` TO Process,\n  FROM `Module` TO Process,\n  FROM `Macro` TO Process,\n  FROM `Impl` TO Process,\n  FROM `Typedef` TO Process,\n  FROM `TypeAlias` TO Process,\n  FROM `Enum` TO Process,\n  FROM `Union` TO Process,\n  FROM `Namespace` TO Process,\n  FROM `Trait` TO Process,\n  FROM `Const` TO Process,\n  FROM `Static` TO Process,\n  FROM `Property` TO Process,\n  FROM `Record` TO Process,\n  FROM `Delegate` TO Process,\n  FROM `Annotation` TO Process,\n  FROM `Template` TO Process,\n  FROM CodeElement TO Process,\n  FROM Route TO Process,\n  FROM Tool TO Process,\n  type STRING,\n  confidence DOUBLE,\n  reason STRING,\n  step INT32\n)";
+export declare const RELATION_SCHEMA = "\nCREATE REL TABLE CodeRelation (\n  FROM File TO File,\n  FROM File TO Folder,\n  FROM File TO Function,\n  FROM File TO Class,\n  FROM File TO Interface,\n  FROM File TO Method,\n  FROM File TO CodeElement,\n  FROM File TO `Struct`,\n  FROM File TO `Enum`,\n  FROM File TO `Macro`,\n  FROM File TO `Typedef`,\n  FROM File TO `Union`,\n  FROM File TO `Namespace`,\n  FROM File TO `Trait`,\n  FROM File TO `Impl`,\n  FROM File TO `TypeAlias`,\n  FROM File TO `Const`,\n  FROM File TO `Static`,\n  FROM File TO `Variable`,\n  FROM File TO `Property`,\n  FROM File TO `Record`,\n  FROM File TO `Delegate`,\n  FROM File TO `Annotation`,\n  FROM File TO `Constructor`,\n  FROM File TO `Template`,\n  FROM File TO `Module`,\n  FROM File TO Section,\n  FROM Folder TO Folder,\n  FROM Folder TO File,\n  FROM Function TO Function,\n  FROM Function TO Method,\n  FROM Function TO Class,\n  FROM Function TO Community,\n  FROM Function TO `Macro`,\n  FROM Function TO `Struct`,\n  FROM Function TO `Template`,\n  FROM Function TO `Enum`,\n  FROM Function TO `Namespace`,\n  FROM Function TO `TypeAlias`,\n  FROM Function TO `Module`,\n  FROM Function TO `Impl`,\n  FROM Function TO Interface,\n  FROM Function TO `Constructor`,\n  FROM Function TO `Const`,\n  FROM Function TO `Typedef`,\n  FROM Function TO `Union`,\n  FROM Function TO `Property`,\n  FROM Function TO CodeElement,\n  FROM Class TO Method,\n  FROM Class TO Function,\n  FROM Class TO Class,\n  FROM Class TO Interface,\n  FROM Class TO Community,\n  FROM Class TO `Template`,\n  FROM Class TO `TypeAlias`,\n  FROM Class TO `Struct`,\n  FROM Class TO `Enum`,\n  FROM Class TO `Annotation`,\n  FROM Class TO `Constructor`,\n  FROM Class TO `Trait`,\n  FROM Class TO `Macro`,\n  FROM Class TO `Impl`,\n  FROM Class TO `Union`,\n  FROM Class TO `Namespace`,\n  FROM Class TO `Typedef`,\n  FROM Class TO `Property`,\n  FROM Method TO Function,\n  FROM Method TO Method,\n  FROM Method TO Class,\n  FROM Method TO Community,\n  FROM Method TO `Template`,\n  FROM Method TO `Struct`,\n  FROM Method TO `TypeAlias`,\n  FROM Method TO `Enum`,\n  FROM Method TO `Macro`,\n  FROM Method TO `Namespace`,\n  FROM Method TO `Module`,\n  FROM Method TO `Impl`,\n  FROM Method TO Interface,\n  FROM Method TO `Constructor`,\n  FROM Method TO `Property`,\n  FROM Method TO CodeElement,\n  FROM `Template` TO `Template`,\n  FROM `Template` TO Function,\n  FROM `Template` TO Method,\n  FROM `Template` TO Class,\n  FROM `Template` TO `Struct`,\n  FROM `Template` TO `TypeAlias`,\n  FROM `Template` TO `Enum`,\n  FROM `Template` TO `Macro`,\n  FROM `Template` TO Interface,\n  FROM `Template` TO `Constructor`,\n  FROM `Module` TO `Module`,\n  FROM Section TO Section,\n  FROM Section TO File,\n  FROM File TO Route,\n  FROM Function TO Route,\n  FROM Method TO Route,\n  FROM File TO Tool,\n  FROM Function TO Tool,\n  FROM Method TO Tool,\n  FROM CodeElement TO Community,\n  FROM Interface TO Community,\n  FROM Interface TO Function,\n  FROM Interface TO Method,\n  FROM Interface TO Class,\n  FROM Interface TO Interface,\n  FROM Interface TO `TypeAlias`,\n  FROM Interface TO `Struct`,\n  FROM Interface TO `Constructor`,\n  FROM Interface TO `Property`,\n  FROM `Struct` TO Community,\n  FROM `Struct` TO `Trait`,\n  FROM `Struct` TO `Struct`,\n  FROM `Struct` TO Class,\n  FROM `Struct` TO `Enum`,\n  FROM `Struct` TO Function,\n  FROM `Struct` TO Method,\n  FROM `Struct` TO Interface,\n  FROM `Struct` TO `Constructor`,\n  FROM `Struct` TO `Property`,\n  FROM `Enum` TO `Enum`,\n  FROM `Enum` TO Community,\n  FROM `Enum` TO Class,\n  FROM `Enum` TO Interface,\n  FROM `Macro` TO Community,\n  FROM `Macro` TO Function,\n  FROM `Macro` TO Method,\n  FROM `Module` TO Function,\n  FROM `Module` TO Method,\n  FROM `Typedef` TO Community,\n  FROM `Union` TO Community,\n  FROM `Namespace` TO Community,\n  FROM `Namespace` TO `Struct`,\n  FROM `Trait` TO Method,\n  FROM `Trait` TO `Constructor`,\n  FROM `Trait` TO `Property`,\n  FROM `Trait` TO Community,\n  FROM `Impl` TO Method,\n  FROM `Impl` TO `Constructor`,\n  FROM `Impl` TO `Property`,\n  FROM `Impl` TO Community,\n  FROM `Impl` TO `Trait`,\n  FROM `Impl` TO `Struct`,\n  FROM `Impl` TO `Impl`,\n  FROM `TypeAlias` TO Community,\n  FROM `TypeAlias` TO `Trait`,\n  FROM `TypeAlias` TO Class,\n  FROM `Const` TO Community,\n  FROM `Static` TO Community,\n  FROM `Variable` TO Community,\n  FROM `Property` TO Community,\n  FROM `Record` TO Method,\n  FROM `Record` TO `Constructor`,\n  FROM `Record` TO `Property`,\n  FROM `Record` TO Community,\n  FROM `Delegate` TO Community,\n  FROM `Annotation` TO Community,\n  FROM `Constructor` TO Community,\n  FROM `Constructor` TO Interface,\n  FROM `Constructor` TO Class,\n  FROM `Constructor` TO Method,\n  FROM `Constructor` TO Function,\n  FROM `Constructor` TO `Constructor`,\n  FROM `Constructor` TO `Struct`,\n  FROM `Constructor` TO `Macro`,\n  FROM `Constructor` TO `Template`,\n  FROM `Constructor` TO `TypeAlias`,\n  FROM `Constructor` TO `Enum`,\n  FROM `Constructor` TO `Annotation`,\n  FROM `Constructor` TO `Impl`,\n  FROM `Constructor` TO `Namespace`,\n  FROM `Constructor` TO `Module`,\n  FROM `Constructor` TO `Property`,\n  FROM `Constructor` TO `Typedef`,\n  FROM `Template` TO Community,\n  FROM `Module` TO Community,\n  FROM Function TO Process,\n  FROM Method TO Process,\n  FROM Class TO Process,\n  FROM Interface TO Process,\n  FROM `Struct` TO Process,\n  FROM `Constructor` TO Process,\n  FROM `Module` TO Process,\n  FROM `Macro` TO Process,\n  FROM `Impl` TO Process,\n  FROM `Typedef` TO Process,\n  FROM `TypeAlias` TO Process,\n  FROM `Enum` TO Process,\n  FROM `Union` TO Process,\n  FROM `Namespace` TO Process,\n  FROM `Trait` TO Process,\n  FROM `Const` TO Process,\n  FROM `Static` TO Process,\n  FROM `Variable` TO Process,\n  FROM `Property` TO Process,\n  FROM `Record` TO Process,\n  FROM `Delegate` TO Process,\n  FROM `Annotation` TO Process,\n  FROM `Template` TO Process,\n  FROM CodeElement TO Process,\n  FROM Route TO Process,\n  FROM Tool TO Process,\n  type STRING,\n  confidence DOUBLE,\n  reason STRING,\n  step INT32\n)";
 export declare const EMBEDDING_DIMS: number;
+/** HNSW vector index name for the CodeEmbedding table. */
+export declare const EMBEDDING_INDEX_NAME = "code_embedding_idx";
+/**
+ * Sentinel value for "no content hash available" — used in legacy DBs and null rows.
+ * Nodes with this hash are always treated as stale and re-embedded.
+ */
+export declare const STALE_HASH_SENTINEL = "";
 export declare const EMBEDDING_SCHEMA: string;
 /**
  * Create vector index for semantic search

package/dist/core/lbug/schema.js

@@ -149,6 +149,7 @@ export const IMPL_SCHEMA = CODE_ELEMENT_BASE('Impl');
 export const TYPE_ALIAS_SCHEMA = CODE_ELEMENT_BASE('TypeAlias');
 export const CONST_SCHEMA = CODE_ELEMENT_BASE('Const');
 export const STATIC_SCHEMA = CODE_ELEMENT_BASE('Static');
+export const VARIABLE_SCHEMA = CODE_ELEMENT_BASE('Variable');
 export const PROPERTY_SCHEMA = CODE_ELEMENT_BASE('Property');
 export const RECORD_SCHEMA = CODE_ELEMENT_BASE('Record');
 export const DELEGATE_SCHEMA = CODE_ELEMENT_BASE('Delegate');
@@ -213,6 +214,7 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
   FROM File TO \`TypeAlias\`,
   FROM File TO \`Const\`,
   FROM File TO \`Static\`,
+  FROM File TO \`Variable\`,
   FROM File TO \`Property\`,
   FROM File TO \`Record\`,
   FROM File TO \`Delegate\`,
@@ -344,6 +346,7 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
   FROM \`TypeAlias\` TO Class,
   FROM \`Const\` TO Community,
   FROM \`Static\` TO Community,
+  FROM \`Variable\` TO Community,
   FROM \`Property\` TO Community,
   FROM \`Record\` TO Method,
   FROM \`Record\` TO \`Constructor\`,
@@ -387,6 +390,7 @@ CREATE REL TABLE ${REL_TABLE_NAME} (
   FROM \`Trait\` TO Process,
   FROM \`Const\` TO Process,
   FROM \`Static\` TO Process,
+  FROM \`Variable\` TO Process,
   FROM \`Property\` TO Process,
   FROM \`Record\` TO Process,
   FROM \`Delegate\` TO Process,
@@ -410,18 +414,30 @@ if (Number.isNaN(_rawDims) || _rawDims <= 0) {
     throw new Error(`GITNEXUS_EMBEDDING_DIMS must be a positive integer, got "${process.env.GITNEXUS_EMBEDDING_DIMS}"`);
 }
 export const EMBEDDING_DIMS = _rawDims;
+/** HNSW vector index name for the CodeEmbedding table. */
+export const EMBEDDING_INDEX_NAME = 'code_embedding_idx';
+/**
+ * Sentinel value for "no content hash available" — used in legacy DBs and null rows.
+ * Nodes with this hash are always treated as stale and re-embedded.
+ */
+export const STALE_HASH_SENTINEL = '';
 export const EMBEDDING_SCHEMA = `
 CREATE NODE TABLE ${EMBEDDING_TABLE_NAME} (
+  id STRING,
   nodeId STRING,
+  chunkIndex INT32,
+  startLine INT64,
+  endLine INT64,
   embedding FLOAT[${EMBEDDING_DIMS}],
-  PRIMARY KEY (nodeId)
+  contentHash STRING,
+  PRIMARY KEY (id)
 )`;
 /**
  * Create vector index for semantic search
  * Uses HNSW (Hierarchical Navigable Small World) algorithm with cosine similarity
  */
 export const CREATE_VECTOR_INDEX_QUERY = `
-CALL CREATE_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', 'code_embedding_idx', 'embedding', metric := 'cosine')
+CALL CREATE_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', 'embedding', metric := 'cosine')
 `;
 // ============================================================================
 // ALL SCHEMA QUERIES IN ORDER
@@ -449,6 +465,7 @@ export const NODE_SCHEMA_QUERIES = [
     TYPE_ALIAS_SCHEMA,
     CONST_SCHEMA,
     STATIC_SCHEMA,
+    VARIABLE_SCHEMA,
     PROPERTY_SCHEMA,
     RECORD_SCHEMA,
     DELEGATE_SCHEMA,

package/dist/core/run-analyze.js

@@ -15,6 +15,8 @@ import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReuse
 import { getStoragePaths, saveMeta, loadMeta, addToGitignore, registerRepo, cleanupOldKuzuFiles, } from '../storage/repo-manager.js';
 import { getCurrentCommit, hasGitDir } from '../storage/git.js';
 import { generateAIContextFiles } from '../cli/ai-context.js';
+import { EMBEDDING_TABLE_NAME } from './lbug/schema.js';
+import { STALE_HASH_SENTINEL } from './lbug/schema.js';
 /** Threshold: auto-skip embeddings for repos with more nodes than this */
 const EMBEDDING_NODE_LIMIT = 50_000;
 export const PHASE_LABELS = {
@@ -144,12 +146,12 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
             }
             else {
                 progress('embeddings', 88, `Restoring ${cachedEmbeddings.length} cached embeddings...`);
+                const { batchInsertEmbeddings: batchInsert } = await import('./embeddings/embedding-pipeline.js');
                 const EMBED_BATCH = 200;
                 for (let i = 0; i < cachedEmbeddings.length; i += EMBED_BATCH) {
                     const batch = cachedEmbeddings.slice(i, i + EMBED_BATCH);
-                    const paramsList = batch.map((e) => ({ nodeId: e.nodeId, embedding: e.embedding }));
                     try {
-                        await executeWithReusedStatement(`MERGE (e:CodeEmbedding {nodeId: $nodeId}) SET e.embedding = $embedding`, paramsList);
+                        await batchInsert(executeWithReusedStatement, batch);
                     }
                     catch {
                         /* some may fail if node was removed, that's fine */
@@ -170,6 +172,17 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
             const httpMode = isHttpMode();
             progress('embeddings', 90, httpMode ? 'Connecting to embedding endpoint...' : 'Loading embedding model...');
             const { runEmbeddingPipeline } = await import('./embeddings/embedding-pipeline.js');
+            // Build a Map<nodeId, contentHash> from cached embeddings for incremental mode
+            let existingEmbeddings;
+            if (cachedEmbeddingNodeIds.size > 0) {
+                existingEmbeddings = new Map();
+                for (const e of cachedEmbeddings) {
+                    existingEmbeddings.set(e.nodeId, e.contentHash ?? STALE_HASH_SENTINEL);
+                }
+            }
+            const { readServerMapping } = await import('./embeddings/server-mapping.js');
+            const projectName = path.basename(repoPath);
+            const serverName = await readServerMapping(projectName);
             await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
                 const scaled = 90 + Math.round((p.percent / 100) * 8);
                 const label = p.phase === 'loading-model'
@@ -178,14 +191,14 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
                         : 'Loading embedding model...'
                     : `Embedding ${p.nodesProcessed || 0}/${p.totalNodes || '?'}`;
                 progress('embeddings', scaled, label);
-            }, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined);
+            }, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined, { repoName: projectName, serverName }, existingEmbeddings);
         }
         // ── Phase 5: Finalize (98–100%) ───────────────────────────────────
         progress('done', 98, 'Saving metadata...');
         // Count embeddings in the index (cached + newly generated)
         let embeddingCount = 0;
         try {
-            const embResult = await executeQuery(`MATCH (e:CodeEmbedding) RETURN count(e) AS cnt`);
+            const embResult = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN count(e) AS cnt`);
             embeddingCount = embResult?.[0]?.cnt ?? 0;
         }
         catch {

package/dist/core/tree-sitter/parser-loader.d.ts

@@ -1,5 +1,8 @@
 import Parser from 'tree-sitter';
 import { SupportedLanguages } from '../../_shared/index.js';
 export declare const isLanguageAvailable: (language: SupportedLanguages) => boolean;
+export declare const resolveLanguageKey: (language: SupportedLanguages, filePath?: string) => string;
+export declare const getLanguageGrammar: (language: SupportedLanguages, filePath?: string) => any;
 export declare const loadParser: () => Promise<Parser>;
 export declare const loadLanguage: (language: SupportedLanguages, filePath?: string) => Promise<void>;
+export declare const createParserForLanguage: (language: SupportedLanguages, filePath?: string) => Promise<Parser>;

package/dist/core/tree-sitter/parser-loader.js

@@ -50,6 +50,17 @@ const languageMap = {
     ...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
 };
 export const isLanguageAvailable = (language) => language in languageMap;
+export const resolveLanguageKey = (language, filePath) => language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
+    ? `${language}:tsx`
+    : language;
+export const getLanguageGrammar = (language, filePath) => {
+    const key = resolveLanguageKey(language, filePath);
+    const lang = languageMap[key];
+    if (!lang) {
+        throw new Error(`Unsupported language: ${language}`);
+    }
+    return lang;
+};
 export const loadParser = async () => {
     if (parser)
         return parser;
@@ -59,12 +70,10 @@ export const loadParser = async () => {
 export const loadLanguage = async (language, filePath) => {
     if (!parser)
         await loadParser();
-    const key = language === SupportedLanguages.TypeScript && filePath?.endsWith('.tsx')
-        ? `${language}:tsx`
-        : language;
-    const lang = languageMap[key];
-    if (!lang) {
-        throw new Error(`Unsupported language: ${language}`);
-    }
-    parser.setLanguage(lang);
+    parser.setLanguage(getLanguageGrammar(language, filePath));
+};
+export const createParserForLanguage = async (language, filePath) => {
+    const freshParser = new Parser();
+    freshParser.setLanguage(getLanguageGrammar(language, filePath));
+    return freshParser;
 };

package/dist/mcp/core/embedder.js

@@ -30,6 +30,11 @@ export const initEmbedder = async () => {
     initPromise = (async () => {
         try {
             env.allowLocalModels = false;
+            // Default cache to user-writable location. transformers.js defaults to
+            // ./node_modules/.cache inside its own install dir, which is unwritable
+            // when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
+            // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
+            env.cacheDir = process.env.HF_HOME ?? `${process.env.HOME}/.cache/huggingface`;
             console.error('GitNexus: Loading embedding model (first search may take a moment)...');
             // Try GPU first (DirectML on Windows, CUDA on Linux), fall back to CPU
             const isWindows = process.platform === 'win32';