gitnexus 1.6.2-rc.1 → 1.6.2-rc.11

package/dist/core/embeddings/embedder.js

@@ -131,6 +131,11 @@ export const initEmbedder = async (onProgress, config = {}, forceDevice) => {
         try {
             // Configure transformers.js environment
             env.allowLocalModels = false;
+            // Default cache to user-writable location. transformers.js defaults to
+            // ./node_modules/.cache inside its own install dir, which is unwritable
+            // when gitnexus is installed globally (e.g. /usr/lib/node_modules/).
+            // Respect HF_HOME if set, otherwise fall back to ~/.cache/huggingface.
+            env.cacheDir = process.env.HF_HOME ?? `${process.env.HOME}/.cache/huggingface`;
             const isDev = process.env.NODE_ENV === 'development';
             if (isDev) {
                 console.log(`🧠 Loading embedding model: ${finalConfig.modelId}`);

package/dist/core/embeddings/embedding-pipeline.d.ts

@@ -8,7 +8,14 @@
  * 4. Update LadybugDB with embeddings
  * 5. Create vector index for semantic search
  */
-import { type EmbeddingProgress, type EmbeddingConfig, type SemanticSearchResult } from './types.js';
+import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult } from './types.js';
+/**
+ * Compute a stable content fingerprint for an embeddable node.
+ * Used to detect when the underlying text has changed so stale vectors
+ * can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for
+ * vector-indexed rows).
+ */
+export declare const contentHashForNode: (node: EmbeddableNode, config?: Partial<EmbeddingConfig>) => string;
 /**
  * Progress callback type
  */
@@ -20,9 +27,11 @@ export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void;
  * @param executeWithReusedStatement - Function to execute with reused prepared statement
  * @param onProgress - Callback for progress updates
  * @param config - Optional configuration override
- * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
+ * @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
+ *        Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
+ *        and re-embedded; nodes not in the map are embedded fresh.
  */
-export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>) => Promise<void>;
+export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, existingEmbeddings?: Map<string, string>) => Promise<void>;
 /**
  * Perform semantic search using the vector index
  *

package/dist/core/embeddings/embedding-pipeline.js

@@ -8,10 +8,23 @@
  * 4. Update LadybugDB with embeddings
  * 5. Create vector index for semantic search
  */
+import { createHash } from 'crypto';
 import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady, } from './embedder.js';
-import { generateBatchEmbeddingTexts } from './text-generator.js';
+import { generateEmbeddingText, generateBatchEmbeddingTexts } from './text-generator.js';
 import { DEFAULT_EMBEDDING_CONFIG, EMBEDDABLE_LABELS, } from './types.js';
+import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, CREATE_VECTOR_INDEX_QUERY, } from '../lbug/schema.js';
+import { loadVectorExtension } from '../lbug/lbug-adapter.js';
 const isDev = process.env.NODE_ENV === 'development';
+/**
+ * Compute a stable content fingerprint for an embeddable node.
+ * Used to detect when the underlying text has changed so stale vectors
+ * can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for
+ * vector-indexed rows).
+ */
+export const contentHashForNode = (node, config = {}) => {
+    const text = generateEmbeddingText(node, config);
+    return createHash('sha1').update(text).digest('hex');
+};
 /**
  * Query all embeddable nodes from LadybugDB
  * Uses table-specific queries (File has different schema than code elements)
@@ -67,34 +80,26 @@ const queryEmbeddableNodes = async (executeQuery) => {
  * that occurs when UPDATEing nodes with large content fields
  */
 const batchInsertEmbeddings = async (executeWithReusedStatement, updates) => {
-    // INSERT into separate embedding table - much more memory efficient!
-    const cypher = `CREATE (e:CodeEmbedding {nodeId: $nodeId, embedding: $embedding})`;
-    const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding }));
+    // MERGE instead of CREATE — idempotent, handles concurrent analyzes and partial prior runs
+    const cypher = `MERGE (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) SET e.embedding = $embedding, e.contentHash = $contentHash`;
+    const paramsList = updates.map((u) => ({
+        nodeId: u.id,
+        embedding: u.embedding,
+        contentHash: u.contentHash,
+    }));
     await executeWithReusedStatement(cypher, paramsList);
 };
 /**
  * Create the vector index for semantic search
- * Now indexes the separate CodeEmbedding table
+ * Now indexes the separate CodeEmbedding table.
+ * Delegates extension loading to lbug-adapter's loadVectorExtension(),
+ * which owns the VECTOR extension lifecycle and state tracking.
  */
-let vectorExtensionLoaded = false;
 const createVectorIndex = async (executeQuery) => {
-    // LadybugDB v0.15+ requires explicit VECTOR extension loading (once per session)
-    if (!vectorExtensionLoaded) {
-        try {
-            await executeQuery('INSTALL VECTOR');
-            await executeQuery('LOAD EXTENSION VECTOR');
-            vectorExtensionLoaded = true;
-        }
-        catch {
-            // Extension may already be loaded — CREATE_VECTOR_INDEX will fail clearly if not
-            vectorExtensionLoaded = true;
-        }
-    }
-    const cypher = `
-    CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')
-  `;
+    // Delegate to the adapter which tracks loaded state and handles DB reconnect resets
+    await loadVectorExtension();
     try {
-        await executeQuery(cypher);
+        await executeQuery(CREATE_VECTOR_INDEX_QUERY);
     }
     catch (error) {
         // Index might already exist
@@ -110,9 +115,11 @@ const createVectorIndex = async (executeQuery) => {
  * @param executeWithReusedStatement - Function to execute with reused prepared statement
  * @param onProgress - Callback for progress updates
  * @param config - Optional configuration override
- * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
+ * @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
+ *        Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
+ *        and re-embedded; nodes not in the map are embedded fresh.
  */
-export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds) => {
+export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, existingEmbeddings) => {
     const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
     try {
         // Phase 1: Load embedding model
@@ -141,12 +148,50 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
         }
         // Phase 2: Query embeddable nodes
         let nodes = await queryEmbeddableNodes(executeQuery);
-        // Incremental mode: filter out nodes that already have embeddings
-        if (skipNodeIds && skipNodeIds.size > 0) {
+        // Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
+        // Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
+        // (avoids double computation).
+        const computedStaleHashes = new Map();
+        if (existingEmbeddings && existingEmbeddings.size > 0) {
             const beforeCount = nodes.length;
-            nodes = nodes.filter((n) => !skipNodeIds.has(n.id));
+            const staleNodeIds = [];
+            nodes = nodes.filter((n) => {
+                const existingHash = existingEmbeddings.get(n.id);
+                if (existingHash === undefined) {
+                    // New node — needs embedding
+                    return true;
+                }
+                const currentHash = contentHashForNode(n, finalConfig);
+                if (currentHash !== existingHash) {
+                    // Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
+                    computedStaleHashes.set(n.id, currentHash);
+                    staleNodeIds.push(n.id);
+                    return true;
+                }
+                // Hash matches — skip (fresh); no need to cache hash for skipped nodes
+                return false;
+            });
+            // DELETE stale embedding rows so they can be re-inserted
+            // (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
+            if (staleNodeIds.length > 0) {
+                if (isDev) {
+                    console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
+                }
+                try {
+                    await executeWithReusedStatement(`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, staleNodeIds.map((nodeId) => ({ nodeId })));
+                }
+                catch (err) {
+                    // "does not exist" = rows already gone — safe to proceed.
+                    // All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT
+                    // for vector-indexed properties) — propagate so the pipeline aborts cleanly.
+                    const msg = err instanceof Error ? err.message : String(err);
+                    if (!msg.includes('does not exist')) {
+                        throw new Error(`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`);
+                    }
+                }
+            }
             if (isDev) {
-                console.log(`📦 Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`);
+                console.log(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`);
             }
         }
         const totalNodes = nodes.length;
@@ -154,6 +199,10 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
             console.log(`📊 Found ${totalNodes} embeddable nodes`);
         }
         if (totalNodes === 0) {
+            // Ensure the vector index exists even when no new nodes need embedding.
+            // A prior crash or first-time incremental run may have left CodeEmbedding
+            // rows without ever reaching index creation.
+            await createVectorIndex(executeQuery);
             onProgress({
                 phase: 'ready',
                 percent: 100,
@@ -186,6 +235,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
             const updates = batch.map((node, i) => ({
                 id: node.id,
                 embedding: embeddingToArray(embeddings[i]),
+                contentHash: computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig),
             }));
             await batchInsertEmbeddings(executeWithReusedStatement, updates);
             processedNodes += batch.length;
@@ -256,7 +306,7 @@ export const semanticSearch = async (executeQuery, query, k = 10, maxDistance =
     const queryVecStr = `[${queryVec.join(',')}]`;
     // Query the vector index on CodeEmbedding to get nodeIds and distances
     const vectorQuery = `
-    CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 
+    CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', 
       CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${k})
     YIELD node AS emb, distance
     WITH emb, distance

package/dist/core/group/extractors/manifest-extractor.js

@@ -68,14 +68,29 @@ export function manifestSymbolUid(repo, contractId) {
 }
 export class ManifestExtractor {
     async extractFromManifest(links, dbExecutors) {
-        const contracts = [];
-        const crossLinks = [];
-        for (const link of links) {
+        const resolveCache = new Map();
+        const resolveOnce = (repo, link) => {
+            const key = `${repo}\u0000${link.type}\u0000${link.contract}`;
+            let pending = resolveCache.get(key);
+            if (!pending) {
+                pending = this.resolveSymbol(repo, link, dbExecutors);
+                resolveCache.set(key, pending);
+            }
+            return pending;
+        };
+        const perLink = await Promise.all(links.map(async (link) => {
             const contractId = this.buildContractId(link.type, link.contract);
             const providerRepo = link.role === 'provider' ? link.from : link.to;
             const consumerRepo = link.role === 'provider' ? link.to : link.from;
-            const providerSymbol = await this.resolveSymbol(providerRepo, link, dbExecutors);
-            const consumerSymbol = await this.resolveSymbol(consumerRepo, link, dbExecutors);
+            const [providerSymbol, consumerSymbol] = await Promise.all([
+                resolveOnce(providerRepo, link),
+                resolveOnce(consumerRepo, link),
+            ]);
+            return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol };
+        }));
+        const contracts = [];
+        const crossLinks = [];
+        for (const { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol, } of perLink) {
             const providerRef = providerSymbol || { filePath: '', name: link.contract };
             const consumerRef = consumerSymbol || { filePath: '', name: link.contract };
             // When the resolver finds a real graph symbol we keep its uid, otherwise

package/dist/core/group/sync.js

@@ -6,6 +6,7 @@ import { readRegistry } from '../../storage/repo-manager.js';
 import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
 import { GrpcExtractor } from './extractors/grpc-extractor.js';
 import { TopicExtractor } from './extractors/topic-extractor.js';
+import { ManifestExtractor } from './extractors/manifest-extractor.js';
 import { runExactMatch } from './matching.js';
 import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
 import { writeContractRegistry } from './storage.js';
@@ -34,10 +35,28 @@ function defaultResolveHandle(allEntries) {
         };
     };
 }
+/**
+ * Dedupe cross-links that point from the same consumer endpoint to the same
+ * provider endpoint for the same contract. Preserves first-seen order so the
+ * caller controls precedence (e.g., pass manifest links first).
+ */
+function dedupeCrossLinks(links) {
+    const seen = new Set();
+    const out = [];
+    for (const link of links) {
+        const key = `${link.from.repo}::${link.from.symbolUid}|${link.to.repo}::${link.to.symbolUid}|${link.type}|${link.contractId}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(link);
+    }
+    return out;
+}
 export async function syncGroup(config, opts) {
     const missingRepos = [];
     const repoSnapshots = {};
     let autoContracts = [];
+    let manifestCrossLinks = [];
     let dbExecutors;
     const eo = opts?.extractorOverride;
     if (eo && eo.length === 0) {
@@ -124,8 +143,37 @@ export async function syncGroup(config, opts) {
             }
         }
     }
+    // Process manifest links declared in group.yaml.
+    // ManifestExtractor is fully implemented but was never wired into this
+    // pipeline — config.links were parsed and validated but silently dropped.
+    // Placed after the DB try/finally: resolveSymbol falls back to synthetic
+    // UIDs when dbExecutors is undefined or a pool is closed, so cross-links
+    // are always generated regardless of whether real DB executors are available.
+    if (config.links.length > 0) {
+        // Warn about dangling links that reference repos not declared in config.repos.
+        // They still generate cross-links via synthetic UIDs (determinism is preserved),
+        // but the operator probably meant something that now silently does nothing useful.
+        const knownRepos = new Set(Object.keys(config.repos));
+        for (const link of config.links) {
+            const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
+            if (dangling.length > 0) {
+                console.warn(`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`);
+            }
+        }
+        const manifestEx = new ManifestExtractor();
+        const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors);
+        autoContracts.push(...manifestResult.contracts);
+        manifestCrossLinks = manifestResult.crossLinks;
+        if (opts?.verbose) {
+            console.log(`  manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`);
+        }
+    }
     const { matched, unmatched } = runExactMatch(autoContracts);
-    const crossLinks = matched;
+    // Dedupe cross-links. Manifest contracts participate in runExactMatch, so a
+    // manifest-declared link can also emit a matchType:'exact' CrossLink with the
+    // same endpoints. Prefer the manifest version — it reflects operator intent
+    // and carries matchType:'manifest' which downstream consumers may rely on.
+    const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]);
     const allContracts = autoContracts;
     const registry = {
         version: 1,

package/dist/core/lbug/csv-generator.js

@@ -246,14 +246,17 @@ export const streamAllCSVsToDisk = async (graph, repoPath, csvDir) => {
         Interface: interfaceWriter,
         CodeElement: codeElemWriter,
     };
-    const seenFileIds = new Set();
+    // Deduplicate all node types — the pipeline can produce duplicate IDs across
+    // all symbol types (Class, Method, Function, etc.), not just File nodes.
+    // A single Set covering every label prevents PK violations on COPY.
+    const seenNodeIds = new Set();
     // --- SINGLE PASS over all nodes ---
     for (const node of graph.iterNodes()) {
+        if (seenNodeIds.has(node.id))
+            continue;
+        seenNodeIds.add(node.id);
         switch (node.label) {
             case 'File': {
-                if (seenFileIds.has(node.id))
-                    break;
-                seenFileIds.add(node.id);
                 const content = await extractContent(node, contentCache);
                 await fileWriter.addRow([
                     escapeCSVField(node.id),

package/dist/core/lbug/lbug-adapter.d.ts

@@ -98,8 +98,18 @@ export declare const loadCachedEmbeddings: () => Promise<{
     embeddings: Array<{
         nodeId: string;
         embedding: number[];
+        contentHash?: string;
     }>;
 }>;
+/**
+ * Fetch existing embedding hashes from CodeEmbedding table for incremental embedding.
+ * Returns a Map<nodeId, contentHash> suitable for passing to `runEmbeddingPipeline`.
+ * Handles legacy DBs without the `contentHash` column (all rows treated as stale with empty hash).
+ * Returns undefined if the CodeEmbedding table does not exist.
+ *
+ * @param execQuery - Cypher query executor (typically pool-adapter's `executeQuery`)
+ */
+export declare const fetchExistingEmbeddingHashes: (execQuery: (cypher: string) => Promise<any[]>) => Promise<Map<string, string> | undefined>;
 export declare const closeLbug: () => Promise<void>;
 export declare const isLbugReady: () => boolean;
 /**