gitnexus 1.6.2-rc.3 → 1.6.2-rc.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,7 +8,14 @@
8
8
  * 4. Update LadybugDB with embeddings
9
9
  * 5. Create vector index for semantic search
10
10
  */
11
- import { type EmbeddingProgress, type EmbeddingConfig, type SemanticSearchResult } from './types.js';
11
+ import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult } from './types.js';
12
+ /**
13
+ * Compute a stable content fingerprint for an embeddable node.
14
+ * Used to detect when the underlying text has changed so stale vectors
15
+ * can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for
16
+ * vector-indexed rows).
17
+ */
18
+ export declare const contentHashForNode: (node: EmbeddableNode, config?: Partial<EmbeddingConfig>) => string;
12
19
  /**
13
20
  * Progress callback type
14
21
  */
@@ -20,9 +27,11 @@ export type EmbeddingProgressCallback = (progress: EmbeddingProgress) => void;
20
27
  * @param executeWithReusedStatement - Function to execute with reused prepared statement
21
28
  * @param onProgress - Callback for progress updates
22
29
  * @param config - Optional configuration override
23
- * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
30
+ * @param existingEmbeddings - Optional map of nodeId contentHash for incremental mode.
31
+ * Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
32
+ * and re-embedded; nodes not in the map are embedded fresh.
24
33
  */
25
- export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>) => Promise<void>;
34
+ export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, existingEmbeddings?: Map<string, string>) => Promise<void>;
26
35
  /**
27
36
  * Perform semantic search using the vector index
28
37
  *
@@ -8,10 +8,23 @@
8
8
  * 4. Update LadybugDB with embeddings
9
9
  * 5. Create vector index for semantic search
10
10
  */
11
+ import { createHash } from 'crypto';
11
12
  import { initEmbedder, embedBatch, embedText, embeddingToArray, isEmbedderReady, } from './embedder.js';
12
- import { generateBatchEmbeddingTexts } from './text-generator.js';
13
+ import { generateEmbeddingText, generateBatchEmbeddingTexts } from './text-generator.js';
13
14
  import { DEFAULT_EMBEDDING_CONFIG, EMBEDDABLE_LABELS, } from './types.js';
15
+ import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, CREATE_VECTOR_INDEX_QUERY, } from '../lbug/schema.js';
16
+ import { loadVectorExtension } from '../lbug/lbug-adapter.js';
14
17
  const isDev = process.env.NODE_ENV === 'development';
18
+ /**
19
+ * Compute a stable content fingerprint for an embeddable node.
20
+ * Used to detect when the underlying text has changed so stale vectors
21
+ * can be replaced (DELETE-then-INSERT, the Kuzu-sanctioned pattern for
22
+ * vector-indexed rows).
23
+ */
24
+ export const contentHashForNode = (node, config = {}) => {
25
+ const text = generateEmbeddingText(node, config);
26
+ return createHash('sha1').update(text).digest('hex');
27
+ };
15
28
  /**
16
29
  * Query all embeddable nodes from LadybugDB
17
30
  * Uses table-specific queries (File has different schema than code elements)
@@ -68,33 +81,25 @@ const queryEmbeddableNodes = async (executeQuery) => {
68
81
  */
69
82
  const batchInsertEmbeddings = async (executeWithReusedStatement, updates) => {
70
83
  // MERGE instead of CREATE — idempotent, handles concurrent analyzes and partial prior runs
71
- const cypher = `MERGE (e:CodeEmbedding {nodeId: $nodeId}) SET e.embedding = $embedding`;
72
- const paramsList = updates.map((u) => ({ nodeId: u.id, embedding: u.embedding }));
84
+ const cypher = `MERGE (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) SET e.embedding = $embedding, e.contentHash = $contentHash`;
85
+ const paramsList = updates.map((u) => ({
86
+ nodeId: u.id,
87
+ embedding: u.embedding,
88
+ contentHash: u.contentHash,
89
+ }));
73
90
  await executeWithReusedStatement(cypher, paramsList);
74
91
  };
75
92
  /**
76
93
  * Create the vector index for semantic search
77
- * Now indexes the separate CodeEmbedding table
94
+ * Now indexes the separate CodeEmbedding table.
95
+ * Delegates extension loading to lbug-adapter's loadVectorExtension(),
96
+ * which owns the VECTOR extension lifecycle and state tracking.
78
97
  */
79
- let vectorExtensionLoaded = false;
80
98
  const createVectorIndex = async (executeQuery) => {
81
- // LadybugDB v0.15+ requires explicit VECTOR extension loading (once per session)
82
- if (!vectorExtensionLoaded) {
83
- try {
84
- await executeQuery('INSTALL VECTOR');
85
- await executeQuery('LOAD EXTENSION VECTOR');
86
- vectorExtensionLoaded = true;
87
- }
88
- catch {
89
- // Extension may already be loaded — CREATE_VECTOR_INDEX will fail clearly if not
90
- vectorExtensionLoaded = true;
91
- }
92
- }
93
- const cypher = `
94
- CALL CREATE_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx', 'embedding', metric := 'cosine')
95
- `;
99
+ // Delegate to the adapter which tracks loaded state and handles DB reconnect resets
100
+ await loadVectorExtension();
96
101
  try {
97
- await executeQuery(cypher);
102
+ await executeQuery(CREATE_VECTOR_INDEX_QUERY);
98
103
  }
99
104
  catch (error) {
100
105
  // Index might already exist
@@ -110,9 +115,11 @@ const createVectorIndex = async (executeQuery) => {
110
115
  * @param executeWithReusedStatement - Function to execute with reused prepared statement
111
116
  * @param onProgress - Callback for progress updates
112
117
  * @param config - Optional configuration override
113
- * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
118
+ * @param existingEmbeddings - Optional map of nodeId contentHash for incremental mode.
119
+ * Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
120
+ * and re-embedded; nodes not in the map are embedded fresh.
114
121
  */
115
- export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds) => {
122
+ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, existingEmbeddings) => {
116
123
  const finalConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
117
124
  try {
118
125
  // Phase 1: Load embedding model
@@ -141,12 +148,50 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
141
148
  }
142
149
  // Phase 2: Query embeddable nodes
143
150
  let nodes = await queryEmbeddableNodes(executeQuery);
144
- // Incremental mode: filter out nodes that already have embeddings
145
- if (skipNodeIds && skipNodeIds.size > 0) {
151
+ // Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
152
+ // Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
153
+ // (avoids double computation).
154
+ const computedStaleHashes = new Map();
155
+ if (existingEmbeddings && existingEmbeddings.size > 0) {
146
156
  const beforeCount = nodes.length;
147
- nodes = nodes.filter((n) => !skipNodeIds.has(n.id));
157
+ const staleNodeIds = [];
158
+ nodes = nodes.filter((n) => {
159
+ const existingHash = existingEmbeddings.get(n.id);
160
+ if (existingHash === undefined) {
161
+ // New node — needs embedding
162
+ return true;
163
+ }
164
+ const currentHash = contentHashForNode(n, finalConfig);
165
+ if (currentHash !== existingHash) {
166
+ // Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
167
+ computedStaleHashes.set(n.id, currentHash);
168
+ staleNodeIds.push(n.id);
169
+ return true;
170
+ }
171
+ // Hash matches — skip (fresh); no need to cache hash for skipped nodes
172
+ return false;
173
+ });
174
+ // DELETE stale embedding rows so they can be re-inserted
175
+ // (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
176
+ if (staleNodeIds.length > 0) {
177
+ if (isDev) {
178
+ console.log(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
179
+ }
180
+ try {
181
+ await executeWithReusedStatement(`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, staleNodeIds.map((nodeId) => ({ nodeId })));
182
+ }
183
+ catch (err) {
184
+ // "does not exist" = rows already gone — safe to proceed.
185
+ // All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT
186
+ // for vector-indexed properties) — propagate so the pipeline aborts cleanly.
187
+ const msg = err instanceof Error ? err.message : String(err);
188
+ if (!msg.includes('does not exist')) {
189
+ throw new Error(`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`);
190
+ }
191
+ }
192
+ }
148
193
  if (isDev) {
149
- console.log(`📦 Incremental embeddings: ${beforeCount} total, ${skipNodeIds.size} cached, ${nodes.length} to embed`);
194
+ console.log(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`);
150
195
  }
151
196
  }
152
197
  const totalNodes = nodes.length;
@@ -154,6 +199,10 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
154
199
  console.log(`📊 Found ${totalNodes} embeddable nodes`);
155
200
  }
156
201
  if (totalNodes === 0) {
202
+ // Ensure the vector index exists even when no new nodes need embedding.
203
+ // A prior crash or first-time incremental run may have left CodeEmbedding
204
+ // rows without ever reaching index creation.
205
+ await createVectorIndex(executeQuery);
157
206
  onProgress({
158
207
  phase: 'ready',
159
208
  percent: 100,
@@ -186,6 +235,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
186
235
  const updates = batch.map((node, i) => ({
187
236
  id: node.id,
188
237
  embedding: embeddingToArray(embeddings[i]),
238
+ contentHash: computedStaleHashes.get(node.id) ?? contentHashForNode(node, finalConfig),
189
239
  }));
190
240
  await batchInsertEmbeddings(executeWithReusedStatement, updates);
191
241
  processedNodes += batch.length;
@@ -256,7 +306,7 @@ export const semanticSearch = async (executeQuery, query, k = 10, maxDistance =
256
306
  const queryVecStr = `[${queryVec.join(',')}]`;
257
307
  // Query the vector index on CodeEmbedding to get nodeIds and distances
258
308
  const vectorQuery = `
259
- CALL QUERY_VECTOR_INDEX('CodeEmbedding', 'code_embedding_idx',
309
+ CALL QUERY_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}',
260
310
  CAST(${queryVecStr} AS FLOAT[${queryVec.length}]), ${k})
261
311
  YIELD node AS emb, distance
262
312
  WITH emb, distance
@@ -68,14 +68,29 @@ export function manifestSymbolUid(repo, contractId) {
68
68
  }
69
69
  export class ManifestExtractor {
70
70
  async extractFromManifest(links, dbExecutors) {
71
- const contracts = [];
72
- const crossLinks = [];
73
- for (const link of links) {
71
+ const resolveCache = new Map();
72
+ const resolveOnce = (repo, link) => {
73
+ const key = `${repo}\u0000${link.type}\u0000${link.contract}`;
74
+ let pending = resolveCache.get(key);
75
+ if (!pending) {
76
+ pending = this.resolveSymbol(repo, link, dbExecutors);
77
+ resolveCache.set(key, pending);
78
+ }
79
+ return pending;
80
+ };
81
+ const perLink = await Promise.all(links.map(async (link) => {
74
82
  const contractId = this.buildContractId(link.type, link.contract);
75
83
  const providerRepo = link.role === 'provider' ? link.from : link.to;
76
84
  const consumerRepo = link.role === 'provider' ? link.to : link.from;
77
- const providerSymbol = await this.resolveSymbol(providerRepo, link, dbExecutors);
78
- const consumerSymbol = await this.resolveSymbol(consumerRepo, link, dbExecutors);
85
+ const [providerSymbol, consumerSymbol] = await Promise.all([
86
+ resolveOnce(providerRepo, link),
87
+ resolveOnce(consumerRepo, link),
88
+ ]);
89
+ return { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol };
90
+ }));
91
+ const contracts = [];
92
+ const crossLinks = [];
93
+ for (const { link, contractId, providerRepo, consumerRepo, providerSymbol, consumerSymbol, } of perLink) {
79
94
  const providerRef = providerSymbol || { filePath: '', name: link.contract };
80
95
  const consumerRef = consumerSymbol || { filePath: '', name: link.contract };
81
96
  // When the resolver finds a real graph symbol we keep its uid, otherwise
@@ -6,6 +6,7 @@ import { readRegistry } from '../../storage/repo-manager.js';
6
6
  import { HttpRouteExtractor } from './extractors/http-route-extractor.js';
7
7
  import { GrpcExtractor } from './extractors/grpc-extractor.js';
8
8
  import { TopicExtractor } from './extractors/topic-extractor.js';
9
+ import { ManifestExtractor } from './extractors/manifest-extractor.js';
9
10
  import { runExactMatch } from './matching.js';
10
11
  import { detectServiceBoundaries, assignService } from './service-boundary-detector.js';
11
12
  import { writeContractRegistry } from './storage.js';
@@ -34,10 +35,28 @@ function defaultResolveHandle(allEntries) {
34
35
  };
35
36
  };
36
37
  }
38
+ /**
39
+ * Dedupe cross-links that point from the same consumer endpoint to the same
40
+ * provider endpoint for the same contract. Preserves first-seen order so the
41
+ * caller controls precedence (e.g., pass manifest links first).
42
+ */
43
+ function dedupeCrossLinks(links) {
44
+ const seen = new Set();
45
+ const out = [];
46
+ for (const link of links) {
47
+ const key = `${link.from.repo}::${link.from.symbolUid}|${link.to.repo}::${link.to.symbolUid}|${link.type}|${link.contractId}`;
48
+ if (seen.has(key))
49
+ continue;
50
+ seen.add(key);
51
+ out.push(link);
52
+ }
53
+ return out;
54
+ }
37
55
  export async function syncGroup(config, opts) {
38
56
  const missingRepos = [];
39
57
  const repoSnapshots = {};
40
58
  let autoContracts = [];
59
+ let manifestCrossLinks = [];
41
60
  let dbExecutors;
42
61
  const eo = opts?.extractorOverride;
43
62
  if (eo && eo.length === 0) {
@@ -124,8 +143,37 @@ export async function syncGroup(config, opts) {
124
143
  }
125
144
  }
126
145
  }
146
+ // Process manifest links declared in group.yaml.
147
+ // ManifestExtractor is fully implemented but was never wired into this
148
+ // pipeline — config.links were parsed and validated but silently dropped.
149
+ // Placed after the DB try/finally: resolveSymbol falls back to synthetic
150
+ // UIDs when dbExecutors is undefined or a pool is closed, so cross-links
151
+ // are always generated regardless of whether real DB executors are available.
152
+ if (config.links.length > 0) {
153
+ // Warn about dangling links that reference repos not declared in config.repos.
154
+ // They still generate cross-links via synthetic UIDs (determinism is preserved),
155
+ // but the operator probably meant something that now silently does nothing useful.
156
+ const knownRepos = new Set(Object.keys(config.repos));
157
+ for (const link of config.links) {
158
+ const dangling = [link.from, link.to].filter((r) => !knownRepos.has(r));
159
+ if (dangling.length > 0) {
160
+ console.warn(`[group/sync] manifest link ${link.type}:${link.contract} references repos not in config.repos: ${dangling.join(', ')} — cross-links will use synthetic UIDs`);
161
+ }
162
+ }
163
+ const manifestEx = new ManifestExtractor();
164
+ const manifestResult = await manifestEx.extractFromManifest(config.links, dbExecutors);
165
+ autoContracts.push(...manifestResult.contracts);
166
+ manifestCrossLinks = manifestResult.crossLinks;
167
+ if (opts?.verbose) {
168
+ console.log(` manifest: ${manifestCrossLinks.length} cross-links from ${config.links.length} declared links`);
169
+ }
170
+ }
127
171
  const { matched, unmatched } = runExactMatch(autoContracts);
128
- const crossLinks = matched;
172
+ // Dedupe cross-links. Manifest contracts participate in runExactMatch, so a
173
+ // manifest-declared link can also emit a matchType:'exact' CrossLink with the
174
+ // same endpoints. Prefer the manifest version — it reflects operator intent
175
+ // and carries matchType:'manifest' which downstream consumers may rely on.
176
+ const crossLinks = dedupeCrossLinks([...manifestCrossLinks, ...matched]);
129
177
  const allContracts = autoContracts;
130
178
  const registry = {
131
179
  version: 1,
@@ -98,8 +98,18 @@ export declare const loadCachedEmbeddings: () => Promise<{
98
98
  embeddings: Array<{
99
99
  nodeId: string;
100
100
  embedding: number[];
101
+ contentHash?: string;
101
102
  }>;
102
103
  }>;
104
+ /**
105
+ * Fetch existing embedding hashes from CodeEmbedding table for incremental embedding.
106
+ * Returns a Map<nodeId, contentHash> suitable for passing to `runEmbeddingPipeline`.
107
+ * Handles legacy DBs without the `contentHash` column (all rows treated as stale with empty hash).
108
+ * Returns undefined if the CodeEmbedding table does not exist.
109
+ *
110
+ * @param execQuery - Cypher query executor (typically pool-adapter's `executeQuery`)
111
+ */
112
+ export declare const fetchExistingEmbeddingHashes: (execQuery: (cypher: string) => Promise<any[]>) => Promise<Map<string, string> | undefined>;
103
113
  export declare const closeLbug: () => Promise<void>;
104
114
  export declare const isLbugReady: () => boolean;
105
115
  /**
@@ -5,7 +5,7 @@ import { once } from 'events';
5
5
  import { finished } from 'stream/promises';
6
6
  import path from 'path';
7
7
  import lbug from '@ladybugdb/core';
8
- import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
8
+ import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, STALE_HASH_SENTINEL, } from './schema.js';
9
9
  import { streamAllCSVsToDisk } from './csv-generator.js';
10
10
  /**
11
11
  * Split a relationship CSV into per-label-pair files on disk.
@@ -110,6 +110,14 @@ let conn = null;
110
110
  let currentDbPath = null;
111
111
  let ftsLoaded = false;
112
112
  let vectorExtensionLoaded = false;
113
+ /**
114
+ * Check if an error indicates a missing column or table (schema-level problem)
115
+ * rather than a transient/connection error. Used for legacy DB fallback logic.
116
+ */
117
+ const isMissingColumnOrTableError = (msg) => msg.includes('does not exist') ||
118
+ // Kuzu-specific: "(table|column|property) ... not found" — narrow enough to avoid
119
+ // matching transient errors like "connection not found" or "key not found".
120
+ /(table|column|property).*not found/i.test(msg);
113
121
  /** Expose the current Database for pool adapter reuse in tests. */
114
122
  export const getDatabase = () => db;
115
123
  // Global session lock for operations that touch module-level lbug globals.
@@ -791,7 +799,24 @@ export const loadCachedEmbeddings = async () => {
791
799
  const embeddingNodeIds = new Set();
792
800
  const embeddings = [];
793
801
  try {
794
- const rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.embedding AS embedding`);
802
+ // Try to read contentHash alongside the embedding
803
+ let rows;
804
+ let hasContentHash = true;
805
+ try {
806
+ rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.embedding AS embedding, e.contentHash AS contentHash`);
807
+ }
808
+ catch (err) {
809
+ // Only fall back for missing-column errors (legacy DBs without contentHash).
810
+ // Rethrow transient / connection errors so callers see them.
811
+ const msg = err?.message ?? '';
812
+ if (isMissingColumnOrTableError(msg)) {
813
+ hasContentHash = false;
814
+ rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.embedding AS embedding`);
815
+ }
816
+ else {
817
+ throw err;
818
+ }
819
+ }
795
820
  const result = Array.isArray(rows) ? rows[0] : rows;
796
821
  for (const row of await result.getAll()) {
797
822
  const nodeId = String(row.nodeId ?? row[0] ?? '');
@@ -805,6 +830,7 @@ export const loadCachedEmbeddings = async () => {
805
830
  embedding: Array.isArray(embedding)
806
831
  ? embedding.map(Number)
807
832
  : Array.from(embedding).map(Number),
833
+ contentHash: hasContentHash ? (row.contentHash ?? row[2] ?? undefined) : undefined,
808
834
  });
809
835
  }
810
836
  }
@@ -814,6 +840,59 @@ export const loadCachedEmbeddings = async () => {
814
840
  }
815
841
  return { embeddingNodeIds, embeddings };
816
842
  };
843
+ /**
844
+ * Fetch existing embedding hashes from CodeEmbedding table for incremental embedding.
845
+ * Returns a Map<nodeId, contentHash> suitable for passing to `runEmbeddingPipeline`.
846
+ * Handles legacy DBs without the `contentHash` column (all rows treated as stale with empty hash).
847
+ * Returns undefined if the CodeEmbedding table does not exist.
848
+ *
849
+ * @param execQuery - Cypher query executor (typically pool-adapter's `executeQuery`)
850
+ */
851
+ export const fetchExistingEmbeddingHashes = async (execQuery) => {
852
+ try {
853
+ const rows = await execQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.contentHash AS contentHash`);
854
+ if (!rows || rows.length === 0)
855
+ return undefined;
856
+ const map = new Map();
857
+ for (const r of rows) {
858
+ const nodeId = r.nodeId ?? r[0];
859
+ const hash = r.contentHash ?? r[1] ?? STALE_HASH_SENTINEL;
860
+ if (nodeId) {
861
+ // Empty/null contentHash means legacy row — treat as stale so it gets re-embedded
862
+ map.set(nodeId, hash || STALE_HASH_SENTINEL);
863
+ }
864
+ }
865
+ return map;
866
+ }
867
+ catch (err) {
868
+ const msg = err?.message ?? '';
869
+ if (isMissingColumnOrTableError(msg)) {
870
+ // Column or table missing — try fallback without contentHash
871
+ try {
872
+ const rows = await execQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId`);
873
+ if (!rows || rows.length === 0)
874
+ return undefined;
875
+ const map = new Map();
876
+ for (const r of rows) {
877
+ const nodeId = r.nodeId ?? r[0];
878
+ if (nodeId)
879
+ map.set(nodeId, STALE_HASH_SENTINEL); // no contentHash — treat as stale
880
+ }
881
+ console.log(`[embed] ${map.size} nodes in legacy DB (no contentHash) — all treated as stale`);
882
+ return map;
883
+ }
884
+ catch (fallbackErr) {
885
+ const fallbackMsg = fallbackErr?.message ?? '';
886
+ if (isMissingColumnOrTableError(fallbackMsg)) {
887
+ console.log(`[embed] CodeEmbedding table not yet present — full embedding run (${fallbackMsg})`);
888
+ return undefined;
889
+ }
890
+ throw fallbackErr;
891
+ }
892
+ }
893
+ throw err;
894
+ }
895
+ };
817
896
  export const closeLbug = async () => {
818
897
  if (conn) {
819
898
  try {
@@ -43,6 +43,13 @@ export declare const TOOL_SCHEMA = "\nCREATE NODE TABLE Tool (\n id STRING,\n
43
43
  export declare const SECTION_SCHEMA = "\nCREATE NODE TABLE Section (\n id STRING,\n name STRING,\n filePath STRING,\n startLine INT64,\n endLine INT64,\n level INT64,\n content STRING,\n description STRING,\n PRIMARY KEY (id)\n)";
44
44
  export declare const RELATION_SCHEMA = "\nCREATE REL TABLE CodeRelation (\n FROM File TO File,\n FROM File TO Folder,\n FROM File TO Function,\n FROM File TO Class,\n FROM File TO Interface,\n FROM File TO Method,\n FROM File TO CodeElement,\n FROM File TO `Struct`,\n FROM File TO `Enum`,\n FROM File TO `Macro`,\n FROM File TO `Typedef`,\n FROM File TO `Union`,\n FROM File TO `Namespace`,\n FROM File TO `Trait`,\n FROM File TO `Impl`,\n FROM File TO `TypeAlias`,\n FROM File TO `Const`,\n FROM File TO `Static`,\n FROM File TO `Property`,\n FROM File TO `Record`,\n FROM File TO `Delegate`,\n FROM File TO `Annotation`,\n FROM File TO `Constructor`,\n FROM File TO `Template`,\n FROM File TO `Module`,\n FROM File TO Section,\n FROM Folder TO Folder,\n FROM Folder TO File,\n FROM Function TO Function,\n FROM Function TO Method,\n FROM Function TO Class,\n FROM Function TO Community,\n FROM Function TO `Macro`,\n FROM Function TO `Struct`,\n FROM Function TO `Template`,\n FROM Function TO `Enum`,\n FROM Function TO `Namespace`,\n FROM Function TO `TypeAlias`,\n FROM Function TO `Module`,\n FROM Function TO `Impl`,\n FROM Function TO Interface,\n FROM Function TO `Constructor`,\n FROM Function TO `Const`,\n FROM Function TO `Typedef`,\n FROM Function TO `Union`,\n FROM Function TO `Property`,\n FROM Function TO CodeElement,\n FROM Class TO Method,\n FROM Class TO Function,\n FROM Class TO Class,\n FROM Class TO Interface,\n FROM Class TO Community,\n FROM Class TO `Template`,\n FROM Class TO `TypeAlias`,\n FROM Class TO `Struct`,\n FROM Class TO `Enum`,\n FROM Class TO `Annotation`,\n FROM Class TO `Constructor`,\n FROM Class TO `Trait`,\n FROM Class TO `Macro`,\n FROM Class TO `Impl`,\n FROM Class TO `Union`,\n FROM Class TO `Namespace`,\n FROM Class TO `Typedef`,\n FROM Class TO `Property`,\n FROM Method TO Function,\n FROM Method TO Method,\n FROM Method TO Class,\n FROM Method TO Community,\n FROM Method TO `Template`,\n FROM Method TO `Struct`,\n FROM Method TO `TypeAlias`,\n FROM Method TO `Enum`,\n FROM Method TO `Macro`,\n FROM Method TO `Namespace`,\n FROM Method TO `Module`,\n FROM Method TO `Impl`,\n FROM Method TO Interface,\n FROM Method TO `Constructor`,\n FROM Method TO `Property`,\n FROM Method TO CodeElement,\n FROM `Template` TO `Template`,\n FROM `Template` TO Function,\n FROM `Template` TO Method,\n FROM `Template` TO Class,\n FROM `Template` TO `Struct`,\n FROM `Template` TO `TypeAlias`,\n FROM `Template` TO `Enum`,\n FROM `Template` TO `Macro`,\n FROM `Template` TO Interface,\n FROM `Template` TO `Constructor`,\n FROM `Module` TO `Module`,\n FROM Section TO Section,\n FROM Section TO File,\n FROM File TO Route,\n FROM Function TO Route,\n FROM Method TO Route,\n FROM File TO Tool,\n FROM Function TO Tool,\n FROM Method TO Tool,\n FROM CodeElement TO Community,\n FROM Interface TO Community,\n FROM Interface TO Function,\n FROM Interface TO Method,\n FROM Interface TO Class,\n FROM Interface TO Interface,\n FROM Interface TO `TypeAlias`,\n FROM Interface TO `Struct`,\n FROM Interface TO `Constructor`,\n FROM Interface TO `Property`,\n FROM `Struct` TO Community,\n FROM `Struct` TO `Trait`,\n FROM `Struct` TO `Struct`,\n FROM `Struct` TO Class,\n FROM `Struct` TO `Enum`,\n FROM `Struct` TO Function,\n FROM `Struct` TO Method,\n FROM `Struct` TO Interface,\n FROM `Struct` TO `Constructor`,\n FROM `Struct` TO `Property`,\n FROM `Enum` TO `Enum`,\n FROM `Enum` TO Community,\n FROM `Enum` TO Class,\n FROM `Enum` TO Interface,\n FROM `Macro` TO Community,\n FROM `Macro` TO Function,\n FROM `Macro` TO Method,\n FROM `Module` TO Function,\n FROM `Module` TO Method,\n FROM `Typedef` TO Community,\n FROM `Union` TO Community,\n FROM `Namespace` TO Community,\n FROM `Namespace` TO `Struct`,\n FROM `Trait` TO Method,\n FROM `Trait` TO `Constructor`,\n FROM `Trait` TO `Property`,\n FROM `Trait` TO Community,\n FROM `Impl` TO Method,\n FROM `Impl` TO `Constructor`,\n FROM `Impl` TO `Property`,\n FROM `Impl` TO Community,\n FROM `Impl` TO `Trait`,\n FROM `Impl` TO `Struct`,\n FROM `Impl` TO `Impl`,\n FROM `TypeAlias` TO Community,\n FROM `TypeAlias` TO `Trait`,\n FROM `TypeAlias` TO Class,\n FROM `Const` TO Community,\n FROM `Static` TO Community,\n FROM `Property` TO Community,\n FROM `Record` TO Method,\n FROM `Record` TO `Constructor`,\n FROM `Record` TO `Property`,\n FROM `Record` TO Community,\n FROM `Delegate` TO Community,\n FROM `Annotation` TO Community,\n FROM `Constructor` TO Community,\n FROM `Constructor` TO Interface,\n FROM `Constructor` TO Class,\n FROM `Constructor` TO Method,\n FROM `Constructor` TO Function,\n FROM `Constructor` TO `Constructor`,\n FROM `Constructor` TO `Struct`,\n FROM `Constructor` TO `Macro`,\n FROM `Constructor` TO `Template`,\n FROM `Constructor` TO `TypeAlias`,\n FROM `Constructor` TO `Enum`,\n FROM `Constructor` TO `Annotation`,\n FROM `Constructor` TO `Impl`,\n FROM `Constructor` TO `Namespace`,\n FROM `Constructor` TO `Module`,\n FROM `Constructor` TO `Property`,\n FROM `Constructor` TO `Typedef`,\n FROM `Template` TO Community,\n FROM `Module` TO Community,\n FROM Function TO Process,\n FROM Method TO Process,\n FROM Class TO Process,\n FROM Interface TO Process,\n FROM `Struct` TO Process,\n FROM `Constructor` TO Process,\n FROM `Module` TO Process,\n FROM `Macro` TO Process,\n FROM `Impl` TO Process,\n FROM `Typedef` TO Process,\n FROM `TypeAlias` TO Process,\n FROM `Enum` TO Process,\n FROM `Union` TO Process,\n FROM `Namespace` TO Process,\n FROM `Trait` TO Process,\n FROM `Const` TO Process,\n FROM `Static` TO Process,\n FROM `Property` TO Process,\n FROM `Record` TO Process,\n FROM `Delegate` TO Process,\n FROM `Annotation` TO Process,\n FROM `Template` TO Process,\n FROM CodeElement TO Process,\n FROM Route TO Process,\n FROM Tool TO Process,\n type STRING,\n confidence DOUBLE,\n reason STRING,\n step INT32\n)";
45
45
  export declare const EMBEDDING_DIMS: number;
46
+ /** HNSW vector index name for the CodeEmbedding table. */
47
+ export declare const EMBEDDING_INDEX_NAME = "code_embedding_idx";
48
+ /**
49
+ * Sentinel value for "no content hash available" — used in legacy DBs and null rows.
50
+ * Nodes with this hash are always treated as stale and re-embedded.
51
+ */
52
+ export declare const STALE_HASH_SENTINEL = "";
46
53
  export declare const EMBEDDING_SCHEMA: string;
47
54
  /**
48
55
  * Create vector index for semantic search
@@ -410,10 +410,18 @@ if (Number.isNaN(_rawDims) || _rawDims <= 0) {
410
410
  throw new Error(`GITNEXUS_EMBEDDING_DIMS must be a positive integer, got "${process.env.GITNEXUS_EMBEDDING_DIMS}"`);
411
411
  }
412
412
  export const EMBEDDING_DIMS = _rawDims;
413
+ /** HNSW vector index name for the CodeEmbedding table. */
414
+ export const EMBEDDING_INDEX_NAME = 'code_embedding_idx';
415
+ /**
416
+ * Sentinel value for "no content hash available" — used in legacy DBs and null rows.
417
+ * Nodes with this hash are always treated as stale and re-embedded.
418
+ */
419
+ export const STALE_HASH_SENTINEL = '';
413
420
  export const EMBEDDING_SCHEMA = `
414
421
  CREATE NODE TABLE ${EMBEDDING_TABLE_NAME} (
415
422
  nodeId STRING,
416
423
  embedding FLOAT[${EMBEDDING_DIMS}],
424
+ contentHash STRING,
417
425
  PRIMARY KEY (nodeId)
418
426
  )`;
419
427
  /**
@@ -421,7 +429,7 @@ CREATE NODE TABLE ${EMBEDDING_TABLE_NAME} (
421
429
  * Uses HNSW (Hierarchical Navigable Small World) algorithm with cosine similarity
422
430
  */
423
431
  export const CREATE_VECTOR_INDEX_QUERY = `
424
- CALL CREATE_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', 'code_embedding_idx', 'embedding', metric := 'cosine')
432
+ CALL CREATE_VECTOR_INDEX('${EMBEDDING_TABLE_NAME}', '${EMBEDDING_INDEX_NAME}', 'embedding', metric := 'cosine')
425
433
  `;
426
434
  // ============================================================================
427
435
  // ALL SCHEMA QUERIES IN ORDER
@@ -15,6 +15,8 @@ import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReuse
15
15
  import { getStoragePaths, saveMeta, loadMeta, addToGitignore, registerRepo, cleanupOldKuzuFiles, } from '../storage/repo-manager.js';
16
16
  import { getCurrentCommit, hasGitDir } from '../storage/git.js';
17
17
  import { generateAIContextFiles } from '../cli/ai-context.js';
18
+ import { EMBEDDING_TABLE_NAME } from './lbug/schema.js';
19
+ import { STALE_HASH_SENTINEL } from './lbug/schema.js';
18
20
  /** Threshold: auto-skip embeddings for repos with more nodes than this */
19
21
  const EMBEDDING_NODE_LIMIT = 50_000;
20
22
  export const PHASE_LABELS = {
@@ -147,9 +149,13 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
147
149
  const EMBED_BATCH = 200;
148
150
  for (let i = 0; i < cachedEmbeddings.length; i += EMBED_BATCH) {
149
151
  const batch = cachedEmbeddings.slice(i, i + EMBED_BATCH);
150
- const paramsList = batch.map((e) => ({ nodeId: e.nodeId, embedding: e.embedding }));
152
+ const paramsList = batch.map((e) => ({
153
+ nodeId: e.nodeId,
154
+ embedding: e.embedding,
155
+ contentHash: e.contentHash ?? STALE_HASH_SENTINEL,
156
+ }));
151
157
  try {
152
- await executeWithReusedStatement(`MERGE (e:CodeEmbedding {nodeId: $nodeId}) SET e.embedding = $embedding`, paramsList);
158
+ await executeWithReusedStatement(`MERGE (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) SET e.embedding = $embedding, e.contentHash = $contentHash`, paramsList);
153
159
  }
154
160
  catch {
155
161
  /* some may fail if node was removed, that's fine */
@@ -170,6 +176,14 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
170
176
  const httpMode = isHttpMode();
171
177
  progress('embeddings', 90, httpMode ? 'Connecting to embedding endpoint...' : 'Loading embedding model...');
172
178
  const { runEmbeddingPipeline } = await import('./embeddings/embedding-pipeline.js');
179
+ // Build a Map<nodeId, contentHash> from cached embeddings for incremental mode
180
+ let existingEmbeddings;
181
+ if (cachedEmbeddingNodeIds.size > 0) {
182
+ existingEmbeddings = new Map();
183
+ for (const e of cachedEmbeddings) {
184
+ existingEmbeddings.set(e.nodeId, e.contentHash ?? STALE_HASH_SENTINEL);
185
+ }
186
+ }
173
187
  await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
174
188
  const scaled = 90 + Math.round((p.percent / 100) * 8);
175
189
  const label = p.phase === 'loading-model'
@@ -178,14 +192,14 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
178
192
  : 'Loading embedding model...'
179
193
  : `Embedding ${p.nodesProcessed || 0}/${p.totalNodes || '?'}`;
180
194
  progress('embeddings', scaled, label);
181
- }, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined);
195
+ }, {}, existingEmbeddings);
182
196
  }
183
197
  // ── Phase 5: Finalize (98–100%) ───────────────────────────────────
184
198
  progress('done', 98, 'Saving metadata...');
185
199
  // Count embeddings in the index (cached + newly generated)
186
200
  let embeddingCount = 0;
187
201
  try {
188
- const embResult = await executeQuery(`MATCH (e:CodeEmbedding) RETURN count(e) AS cnt`);
202
+ const embResult = await executeQuery(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN count(e) AS cnt`);
189
203
  embeddingCount = embResult?.[0]?.cnt ?? 0;
190
204
  }
191
205
  catch {
@@ -1277,25 +1277,12 @@ export const createServer = async (port, host = '127.0.0.1') => {
1277
1277
  const lbugPath = path.join(entry.storagePath, 'lbug');
1278
1278
  await withLbugDb(lbugPath, async () => {
1279
1279
  const { runEmbeddingPipeline } = await import('../core/embeddings/embedding-pipeline.js');
1280
- // Skip nodes that already have embeddings — Kuzu forbids SET on vector-indexed properties.
1281
- let skipNodeIds;
1282
- try {
1283
- const rows = await executeQuery('MATCH (e:CodeEmbedding) RETURN e.nodeId AS nodeId');
1284
- if (rows && rows.length > 0) {
1285
- skipNodeIds = new Set(rows.map((r) => r.nodeId ?? r[0]).filter(Boolean));
1286
- console.log(`[embed] ${skipNodeIds.size} nodes already embedded — skipping in incremental run`);
1287
- }
1288
- }
1289
- catch (err) {
1290
- // Swallow only "table does not exist" — let real connection errors propagate.
1291
- // Log so ops can see this path fire if Kuzu ever changes error wording.
1292
- const msg = err?.message ?? '';
1293
- if (msg.includes('does not exist') || msg.includes('not found')) {
1294
- console.log(`[embed] CodeEmbedding table not yet present — full embedding run (${msg})`);
1295
- }
1296
- else {
1297
- throw err;
1298
- }
1280
+ // Fetch existing content hashes for incremental embedding.
1281
+ // Delegated to lbug-adapter which owns the DB query logic and legacy-fallback handling.
1282
+ const { fetchExistingEmbeddingHashes } = await import('../core/lbug/lbug-adapter.js');
1283
+ const existingEmbeddings = await fetchExistingEmbeddingHashes(executeQuery);
1284
+ if (existingEmbeddings && existingEmbeddings.size > 0) {
1285
+ console.log(`[embed] ${existingEmbeddings.size} nodes already embedded incremental run with content-hash comparison`);
1299
1286
  }
1300
1287
  await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
1301
1288
  embedJobManager.updateJob(job.id, {
@@ -1313,8 +1300,8 @@ export const createServer = async (port, host = '127.0.0.1') => {
1313
1300
  : `${p.phase} (${p.percent}%)`,
1314
1301
  },
1315
1302
  });
1316
- }, {}, // config: use defaults (runEmbeddingPipeline signature: executeQuery, executeWithReusedStatement, onProgress, config, skipNodeIds)
1317
- skipNodeIds);
1303
+ }, {}, // config: use defaults
1304
+ existingEmbeddings);
1318
1305
  });
1319
1306
  clearTimeout(embedTimeout);
1320
1307
  releaseRepoLock(repoLockPath);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "gitnexus",
3
- "version": "1.6.2-rc.3",
3
+ "version": "1.6.2-rc.5",
4
4
  "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
5
5
  "author": "Abhigyan Patwari",
6
6
  "license": "PolyForm-Noncommercial-1.0.0",
@@ -71,7 +71,7 @@
71
71
  "pandemonium": "^2.4.0",
72
72
  "tree-sitter": "^0.21.1",
73
73
  "tree-sitter-c": "0.23.2",
74
- "tree-sitter-c-sharp": "^0.23.1",
74
+ "tree-sitter-c-sharp": "0.23.1",
75
75
  "tree-sitter-cpp": "^0.23.4",
76
76
  "tree-sitter-go": "^0.23.0",
77
77
  "tree-sitter-java": "^0.23.5",