gitnexus 1.6.9-rc.32 → 1.6.9-rc.34

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -234,5 +234,5 @@ export declare const en: {
234
234
  readonly 'help.option.group.contracts.type': "Filter by contract type";
235
235
  readonly 'help.option.group.contracts.repo': "Filter by repo";
236
236
  readonly 'help.option.group.contracts.unmatched': "Show only unmatched contracts";
237
- readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
237
+ readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
238
238
  };
@@ -234,5 +234,5 @@ export const en = {
234
234
  'help.option.group.contracts.type': 'Filter by contract type',
235
235
  'help.option.group.contracts.repo': 'Filter by repo',
236
236
  'help.option.group.contracts.unmatched': 'Show only unmatched contracts',
237
- 'help.analyze.environment': '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).',
237
+ 'help.analyze.environment': '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).',
238
238
  };
@@ -235,7 +235,7 @@ export declare const cliResources: {
235
235
  readonly 'help.option.group.contracts.type': "Filter by contract type";
236
236
  readonly 'help.option.group.contracts.repo': "Filter by repo";
237
237
  readonly 'help.option.group.contracts.unmatched': "Show only unmatched contracts";
238
- readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
238
+ readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
239
239
  };
240
240
  readonly 'zh-CN': {
241
241
  'common.notIndexed': string;
@@ -234,5 +234,5 @@ export const zhCN = {
234
234
  'help.option.group.contracts.type': '按契约类型过滤',
235
235
  'help.option.group.contracts.repo': '按仓库过滤',
236
236
  'help.option.group.contracts.unmatched': '仅显示未匹配契约',
237
- 'help.analyze.environment': '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n\n当参数和对应环境变量同时提供时,参数优先。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。',
237
+ 'help.analyze.environment': '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离(0 < N <= 2;超出则钳制为 2)。MCP 默认 0.6,其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时,参数优先。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。',
238
238
  };
@@ -1,2 +1,17 @@
1
1
  import { type EmbeddingConfig } from './types.js';
2
+ export declare const DEFAULT_VECTOR_MAX_DISTANCE = 0.5;
3
+ export declare const DEFAULT_MCP_VECTOR_MAX_DISTANCE = 0.6;
4
+ /**
5
+ * Cosine distance over normalized embeddings is bounded to [0, 2], so any threshold
6
+ * above this accepts every row and silently disables the relevance filter. Values
7
+ * over the ceiling are clamped to it rather than passed through.
8
+ */
9
+ export declare const VECTOR_MAX_DISTANCE_CEILING = 2;
10
+ /**
11
+ * Resolve the effective max accepted vector/semantic cosine distance.
12
+ * Reads `GITNEXUS_VECTOR_MAX_DISTANCE`. Unset/empty/whitespace → silent fallback.
13
+ * Invalid (non-numeric, <= 0, non-finite) → fallback plus a one-time warning.
14
+ * Values above the cosine ceiling (2) are clamped to it with a one-time warning.
15
+ */
16
+ export declare const getVectorMaxDistance: (fallback?: number) => number;
2
17
  export declare const resolveEmbeddingConfig: (overrides?: Partial<EmbeddingConfig>) => EmbeddingConfig;
@@ -1,5 +1,42 @@
1
1
  import { defaultEmbeddingThreads } from '../platform/capabilities.js';
2
+ import { logger } from '../logger.js';
2
3
  import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
4
+ export const DEFAULT_VECTOR_MAX_DISTANCE = 0.5;
5
+ export const DEFAULT_MCP_VECTOR_MAX_DISTANCE = 0.6;
6
+ /**
7
+ * Cosine distance over normalized embeddings is bounded to [0, 2], so any threshold
8
+ * above this accepts every row and silently disables the relevance filter. Values
9
+ * over the ceiling are clamped to it rather than passed through.
10
+ */
11
+ export const VECTOR_MAX_DISTANCE_CEILING = 2;
12
+ const warned = new Set();
13
+ const warnOnce = (key, message) => {
14
+ if (warned.has(key))
15
+ return;
16
+ warned.add(key);
17
+ logger.warn(message);
18
+ };
19
+ /**
20
+ * Resolve the effective max accepted vector/semantic cosine distance.
21
+ * Reads `GITNEXUS_VECTOR_MAX_DISTANCE`. Unset/empty/whitespace → silent fallback.
22
+ * Invalid (non-numeric, <= 0, non-finite) → fallback plus a one-time warning.
23
+ * Values above the cosine ceiling (2) are clamped to it with a one-time warning.
24
+ */
25
+ export const getVectorMaxDistance = (fallback = DEFAULT_VECTOR_MAX_DISTANCE) => {
26
+ const raw = process.env.GITNEXUS_VECTOR_MAX_DISTANCE;
27
+ if (raw === undefined || raw.trim() === '')
28
+ return fallback;
29
+ const parsed = Number(raw);
30
+ if (!Number.isFinite(parsed) || parsed <= 0) {
31
+ warnOnce(`invalid:${raw}`, ` GITNEXUS_VECTOR_MAX_DISTANCE must be a positive number in (0, ${VECTOR_MAX_DISTANCE_CEILING}], got "${raw}" — using default ${fallback}`);
32
+ return fallback;
33
+ }
34
+ if (parsed > VECTOR_MAX_DISTANCE_CEILING) {
35
+ warnOnce(`clamp:${raw}`, ` GITNEXUS_VECTOR_MAX_DISTANCE=${parsed} exceeds the cosine-distance ceiling (${VECTOR_MAX_DISTANCE_CEILING}) — clamping`);
36
+ return VECTOR_MAX_DISTANCE_CEILING;
37
+ }
38
+ return parsed;
39
+ };
3
40
  const parsePositiveInt = (name, value, fallback) => {
4
41
  if (value === undefined)
5
42
  return fallback;
@@ -8,7 +8,7 @@
8
8
  * 4. Update LadybugDB with chunk-aware embeddings
9
9
  * 5. Create vector index for semantic search
10
10
  */
11
- import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult, type EmbeddingContext } from './types.js';
11
+ import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult } from './types.js';
12
12
  import type { ExtensionInstallPolicy } from '../lbug/extension-loader.js';
13
13
  /**
14
14
  * Resolve the extension-install policy for the embedding WRITE path (analyze).
@@ -28,7 +28,7 @@ export declare const resolveEmbeddingInstallPolicy: () => ExtensionInstallPolicy
28
28
  * invalidate existing vectors, such as metadata/header shape changes,
29
29
  * structural container context changes, or preceding-context formatting rules.
30
30
  */
31
- export declare const EMBEDDING_TEXT_VERSION = "v2";
31
+ export declare const EMBEDDING_TEXT_VERSION = "v4";
32
32
  /**
33
33
  * Compute a stable content fingerprint for an embeddable node.
34
34
  * Used to detect when the underlying text has changed so stale vectors
@@ -65,13 +65,11 @@ export interface EmbeddingPipelineResult {
65
65
  * @param onProgress - Callback for progress updates
66
66
  * @param config - Optional configuration override
67
67
  * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
68
- * @param context - Optional repo/server context for metadata enrichment
69
68
  * @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
70
69
  * Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
71
70
  * and re-embedded; nodes not in the map are embedded fresh.
72
-
73
71
  */
74
- export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>, context?: EmbeddingContext, existingEmbeddings?: Map<string, string>) => Promise<EmbeddingPipelineResult>;
72
+ export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>, existingEmbeddings?: Map<string, string>) => Promise<EmbeddingPipelineResult>;
75
73
  /**
76
74
  * Perform semantic search using the vector index with chunk deduplication
77
75
  */
@@ -14,7 +14,7 @@ import { generateEmbeddingText } from './text-generator.js';
14
14
  import { chunkNode, characterChunk } from './chunker.js';
15
15
  import { extractStructuralNames } from './structural-extractor.js';
16
16
  import { EMBEDDABLE_LABELS, isShortLabel, LABEL_METHOD, LABELS_WITH_EXPORTED, STRUCTURAL_LABELS, collectBestChunks, } from './types.js';
17
- import { resolveEmbeddingConfig } from './config.js';
17
+ import { DEFAULT_VECTOR_MAX_DISTANCE, getVectorMaxDistance, resolveEmbeddingConfig, } from './config.js';
18
18
  import { rankExactEmbeddingRows } from './exact-search.js';
19
19
  import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, STALE_HASH_SENTINEL } from '../lbug/schema.js';
20
20
  import { loadVectorExtension, createVectorIndex } from '../lbug/lbug-adapter.js';
@@ -51,7 +51,7 @@ const ensureVectorExtensionAvailable = async () => {
51
51
  * invalidate existing vectors, such as metadata/header shape changes,
52
52
  * structural container context changes, or preceding-context formatting rules.
53
53
  */
54
- export const EMBEDDING_TEXT_VERSION = 'v2';
54
+ export const EMBEDDING_TEXT_VERSION = 'v4';
55
55
  /**
56
56
  * Compute a stable content fingerprint for an embeddable node.
57
57
  * Used to detect when the underlying text has changed so stale vectors
@@ -185,6 +185,32 @@ const buildVectorIndex = async () => {
185
185
  return false;
186
186
  }
187
187
  };
188
+ /**
189
+ * DELETE stale embedding rows for the given nodeIds so they can be re-inserted.
190
+ *
191
+ * Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the
192
+ * sanctioned pattern. A `"does not exist"` error means the rows are already gone
193
+ * (safe to proceed); any other error risks vector-index corruption, so it
194
+ * propagates and aborts the pipeline.
195
+ *
196
+ * Called per-batch (just before each batch's INSERT), not once up front — see
197
+ * the caller comment / KTD7: an up-front bulk delete of every stale row leaves
198
+ * the whole index deleted-not-reinserted if the re-embed is interrupted. Per-batch
199
+ * interleaving bounds that window to a single batch.
200
+ */
201
+ const deleteStaleEmbeddingRows = async (executeWithReusedStatement, nodeIds) => {
202
+ if (nodeIds.length === 0)
203
+ return;
204
+ try {
205
+ await executeWithReusedStatement(`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, nodeIds.map((nodeId) => ({ nodeId })));
206
+ }
207
+ catch (err) {
208
+ const msg = err instanceof Error ? err.message : String(err);
209
+ if (!msg.includes('does not exist')) {
210
+ throw new Error(`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`);
211
+ }
212
+ }
213
+ };
188
214
  /**
189
215
  * Run the embedding pipeline
190
216
  *
@@ -193,13 +219,11 @@ const buildVectorIndex = async () => {
193
219
  * @param onProgress - Callback for progress updates
194
220
  * @param config - Optional configuration override
195
221
  * @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
196
- * @param context - Optional repo/server context for metadata enrichment
197
222
  * @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
198
223
  * Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
199
224
  * and re-embedded; nodes not in the map are embedded fresh.
200
-
201
225
  */
202
- export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds, context, existingEmbeddings) => {
226
+ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds, existingEmbeddings) => {
203
227
  const finalConfig = resolveEmbeddingConfig(config);
204
228
  let totalChunks = 0;
205
229
  try {
@@ -233,20 +257,16 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
233
257
  }
234
258
  // Phase 2: Query embeddable nodes
235
259
  let nodes = await queryEmbeddableNodes(executeQuery);
236
- // Apply context metadata
237
- if (context?.repoName) {
238
- for (const node of nodes) {
239
- node.repoName = context.repoName;
240
- node.serverName = context.serverName;
241
- }
242
- }
243
260
  // Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
244
261
  // Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
245
262
  // (avoids double computation).
246
263
  const computedStaleHashes = new Map();
264
+ // Stale rows are DELETE'd per-batch (just before each batch's INSERT) rather
265
+ // than all up front — see U6 / KTD7. `staleNodeIds` is consulted inside the
266
+ // batch loop; it stays empty in full (non-incremental) mode so no deletes fire.
267
+ const staleNodeIds = new Set();
247
268
  if (existingEmbeddings && existingEmbeddings.size > 0) {
248
269
  const beforeCount = nodes.length;
249
- const staleNodeIds = [];
250
270
  nodes = nodes.filter((n) => {
251
271
  const existingHash = existingEmbeddings.get(n.id);
252
272
  if (existingHash === undefined) {
@@ -257,33 +277,14 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
257
277
  if (currentHash !== existingHash) {
258
278
  // Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
259
279
  computedStaleHashes.set(n.id, currentHash);
260
- staleNodeIds.push(n.id);
280
+ staleNodeIds.add(n.id);
261
281
  return true;
262
282
  }
263
283
  // Hash matches — skip (fresh); no need to cache hash for skipped nodes
264
284
  return false;
265
285
  });
266
- // DELETE stale embedding rows so they can be re-inserted
267
- // (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
268
- if (staleNodeIds.length > 0) {
269
- if (isDev) {
270
- logger.info(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
271
- }
272
- try {
273
- await executeWithReusedStatement(`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, staleNodeIds.map((nodeId) => ({ nodeId })));
274
- }
275
- catch (err) {
276
- // "does not exist" = rows already gone — safe to proceed.
277
- // All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT
278
- // for vector-indexed properties) — propagate so the pipeline aborts cleanly.
279
- const msg = err instanceof Error ? err.message : String(err);
280
- if (!msg.includes('does not exist')) {
281
- throw new Error(`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`);
282
- }
283
- }
284
- }
285
286
  if (isDev) {
286
- logger.info(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.length} stale, ${nodes.length} to embed`);
287
+ logger.info(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.size} stale, ${nodes.length} to embed`);
287
288
  }
288
289
  }
289
290
  const totalNodes = nodes.length;
@@ -373,6 +374,11 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
373
374
  prevTail = overlap > 0 ? chunk.text.slice(-overlap) : '';
374
375
  }
375
376
  }
377
+ // U6 / KTD7: delete this batch's stale rows immediately before its inserts,
378
+ // so an interrupted re-embed loses at most one batch (not the whole index).
379
+ // Preserves Kuzu's required DELETE-before-INSERT for vector-indexed rows.
380
+ const batchStaleIds = batch.filter((n) => staleNodeIds.has(n.id)).map((n) => n.id);
381
+ await deleteStaleEmbeddingRows(executeWithReusedStatement, batchStaleIds);
376
382
  // Embed chunk texts in sub-batches to control memory
377
383
  const EMBED_SUB_BATCH = finalConfig.subBatchSize;
378
384
  for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) {
@@ -447,7 +453,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
447
453
  /**
448
454
  * Perform semantic search using the vector index with chunk deduplication
449
455
  */
450
- export const semanticSearch = async (executeQuery, query, k = 10, maxDistance = 0.5) => {
456
+ export const semanticSearch = async (executeQuery, query, k = 10, maxDistance = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE)) => {
451
457
  if (!isEmbedderReady()) {
452
458
  throw new Error('Embedding model not initialized. Run embedding pipeline first.');
453
459
  }
@@ -569,7 +575,7 @@ export const semanticSearch = async (executeQuery, query, k = 10, maxDistance =
569
575
  * Semantic search with graph expansion (flattened results)
570
576
  */
571
577
  export const semanticSearchWithContext = async (executeQuery, query, k = 5, _hops = 1) => {
572
- const results = await semanticSearch(executeQuery, query, k, 0.5);
578
+ const results = await semanticSearch(executeQuery, query, k);
573
579
  return results.map((r) => ({
574
580
  matchId: r.nodeId,
575
581
  matchName: r.name,
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Text Generator Module
3
3
  *
4
- * Generates enriched embedding text from code nodes with metadata.
4
+ * Generates compact, description-forward embedding text from code nodes.
5
5
  * Supports chunkable labels (Function/Method with AST chunking),
6
6
  * Class-specific structural text, and short-node direct embed.
7
7
  *
@@ -1,7 +1,7 @@
1
1
  /**
2
2
  * Text Generator Module
3
3
  *
4
- * Generates enriched embedding text from code nodes with metadata.
4
+ * Generates compact, description-forward embedding text from code nodes.
5
5
  * Supports chunkable labels (Function/Method with AST chunking),
6
6
  * Class-specific structural text, and short-node direct embed.
7
7
  *
@@ -41,27 +41,48 @@ const cleanContent = (content) => {
41
41
  .trim();
42
42
  };
43
43
  /**
44
- * Build metadata header for a node
44
+ * Compact location signal for the embedding header: the last 1-2 path segments
45
+ * (immediate parent dir + basename), never the full deep path.
46
+ *
47
+ * #2333 / PR #2334 tri-review: U1 dropped the location entirely, which regressed
48
+ * path/service-qualified semantic search (e.g. `billing/handler` vs
49
+ * `identity/handler` in a monorepo) — and FTS indexes only name/content/description,
50
+ * never `filePath`, so there is no keyword backfill. The bounded form restores the
51
+ * discriminating tokens (service dir + filename-concept) at a fraction of the
52
+ * dilution the full path caused.
45
53
  */
46
- const buildMetadataHeader = (node, config) => {
54
+ const boundedLocation = (filePath) => {
55
+ const segments = filePath.replace(/\\/g, '/').split('/').filter(Boolean);
56
+ return segments.slice(-2).join('/');
57
+ };
58
+ /**
59
+ * Build a compact, description-forward header for embedding text.
60
+ *
61
+ * Issue #2333 (sub-issue of #2326), Option A: lead the embedding text with the
62
+ * symbol name + doc-comment description and drop the low-signal metadata lines
63
+ * (`Repo`/`Server`/`Export` and the verbose full `Path`). For short doc comments
64
+ * those lines used to be ~25-30% of the embedding text, diluting the description's
65
+ * semantic weight in the vector and weakening description-shaped search — worst
66
+ * for CJK, where a complete concept is often 4-20 characters.
67
+ *
68
+ * A *bounded* location signal (last 1-2 path segments) is kept after the
69
+ * description — see `boundedLocation` for why the full path drop was reversed.
70
+ *
71
+ * Full metadata is unaffected: it lives on the graph node properties, which is
72
+ * what display/context tools read. Only the embedding text changes here.
73
+ *
74
+ * Option B (reorder only, keep metadata) was rejected — mean-pooled embeddings
75
+ * weight by token proportion, not position, so reordering alone barely moves the
76
+ * signal. Option C (a separate description-only embedding + hybrid merge) is
77
+ * deferred to follow-up; build it only if Option A proves insufficient against
78
+ * real measurement. Any change to this template MUST bump EMBEDDING_TEXT_VERSION.
79
+ */
80
+ const buildEmbeddingHeader = (node, config) => {
47
81
  const parts = [];
48
82
  // Label + name
49
83
  parts.push(`${node.label}: ${node.name}`);
50
- // Repo name
51
- if (node.repoName) {
52
- parts.push(`Repo: ${node.repoName}`);
53
- }
54
- // Server name (optional)
55
- if (node.serverName) {
56
- parts.push(`Server: ${node.serverName}`);
57
- }
58
- // Full file path
59
- parts.push(`Path: ${node.filePath}`);
60
- // Export status
61
- if (node.isExported !== undefined) {
62
- parts.push(`Export: ${node.isExported}`);
63
- }
64
- // Description (truncated)
84
+ // Description hoisted above everything else so its semantic signal dominates
85
+ // the embedding vector and is never the part lost to token-limit truncation.
65
86
  if (node.description) {
66
87
  const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength;
67
88
  const truncated = truncateDescription(node.description, maxLen);
@@ -69,10 +90,19 @@ const buildMetadataHeader = (node, config) => {
69
90
  parts.push(truncated);
70
91
  }
71
92
  }
93
+ // Bounded location signal — placed after the description so the description
94
+ // still leads the vector. Restores path/service disambiguation lost when the
95
+ // full Path line was dropped (FTS does not index filePath to backfill it).
96
+ if (node.filePath) {
97
+ const loc = boundedLocation(node.filePath);
98
+ if (loc) {
99
+ parts.push(`Loc: ${loc}`);
100
+ }
101
+ }
72
102
  return parts.join('\n');
73
103
  };
74
104
  const generateCodeBodyText = (node, codeBody, config, prevTail) => {
75
- const header = buildMetadataHeader(node, config);
105
+ const header = buildEmbeddingHeader(node, config);
76
106
  const parts = [header];
77
107
  if (prevTail) {
78
108
  parts.push(`[preceding context]: ...${cleanContent(prevTail)}`);
@@ -87,7 +117,7 @@ const getCompactContainerContext = (cleanedContent, declarationOnly) => {
87
117
  return firstLine ? `Container: ${firstLine}` : undefined;
88
118
  };
89
119
  const generateStructuralTypeText = (node, codeBody, config, chunkIndex, prevTail) => {
90
- const header = buildMetadataHeader(node, config);
120
+ const header = buildEmbeddingHeader(node, config);
91
121
  const parts = [header];
92
122
  const isFirstChunk = chunkIndex === undefined || chunkIndex === 0;
93
123
  const cleanedContent = cleanContent(node.content);
@@ -192,7 +222,7 @@ export const extractDeclarationOnly = (content) => {
192
222
  */
193
223
  export const generateEmbeddingText = (node, codeBody, config = {}, chunkIndex, prevTail) => {
194
224
  if (isShortLabel(node.label)) {
195
- const header = buildMetadataHeader(node, config);
225
+ const header = buildEmbeddingHeader(node, config);
196
226
  const cleaned = cleanContent(node.content);
197
227
  return `${header}\n\n${cleaned}`;
198
228
  }
@@ -168,13 +168,6 @@ export interface CachedEmbedding {
168
168
  embedding: number[];
169
169
  contentHash?: string;
170
170
  }
171
- /**
172
- * Context info for embedding pipeline (repo/server metadata enrichment)
173
- */
174
- export interface EmbeddingContext {
175
- repoName?: string;
176
- serverName?: string;
177
- }
178
171
  /**
179
172
  * Model download progress from transformers.js
180
173
  */
@@ -982,20 +982,6 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
982
982
  existingEmbeddings.set(e.nodeId, e.contentHash ?? STALE_HASH_SENTINEL);
983
983
  }
984
984
  }
985
- const { readServerMapping } = await import('./embeddings/server-mapping.js');
986
- // Mirror the registry's name-resolution chain so the server-mapping
987
- // lookup key stays aligned with the final registry name (#1259):
988
- // --name → remote-derived → canonical-root basename
989
- // (preserved-alias is intentionally NOT consulted here — server
990
- // mappings are addressed by the operationally-meaningful name the
991
- // user configures, not by a sticky registry-only alias they may not
992
- // know about. The previous canonical-only logic ignored both --name
993
- // and remote-derived names, silently breaking server-mapping for
994
- // anyone with a `--name` alias or remote-named repo.)
995
- const projectName = options.registryName ??
996
- getInferredRepoName(repoPath) ??
997
- path.basename(resolveRepoIdentityRoot(repoPath));
998
- const serverName = await readServerMapping(projectName);
999
985
  const embeddingResult = await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
1000
986
  const scaled = 90 + Math.round((p.percent / 100) * 8);
1001
987
  const label = p.phase === 'loading-model'
@@ -1004,7 +990,7 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
1004
990
  : 'Loading embedding model...'
1005
991
  : `Embedding ${p.nodesProcessed || 0}/${p.totalNodes || '?'}`;
1006
992
  progress('embeddings', scaled, label);
1007
- }, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined, { repoName: projectName, serverName }, existingEmbeddings);
993
+ }, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined, existingEmbeddings);
1008
994
  if (embeddingResult.semanticMode === 'exact-scan') {
1009
995
  semanticMode = 'exact-scan';
1010
996
  log('Semantic embeddings were generated without a VECTOR index; ' +
@@ -21,6 +21,7 @@ import { listRegisteredRepos, cleanupOldKuzuFiles, canonicalizePath, getStorageP
21
21
  import { GroupService, } from '../../core/group/service.js';
22
22
  import { resolveAtGroupMemberRepoPath } from '../../core/group/resolve-at-member.js';
23
23
  import { collectBestChunks } from '../../core/embeddings/types.js';
24
+ import { DEFAULT_MCP_VECTOR_MAX_DISTANCE, getVectorMaxDistance, } from '../../core/embeddings/config.js';
24
25
  import { rankExactEmbeddingRows, } from '../../core/embeddings/exact-search.js';
25
26
  import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME } from '../../core/lbug/schema.js';
26
27
  import { getExactScanLimit, isVectorExtensionSupportedByPlatform, } from '../../core/platform/capabilities.js';
@@ -1694,6 +1695,7 @@ export class LocalBackend {
1694
1695
  const queryVec = await embedQuery(query);
1695
1696
  const dims = getEmbeddingDims();
1696
1697
  const queryVecStr = `[${queryVec.join(',')}]`;
1698
+ const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE);
1697
1699
  let bestChunks = new Map();
1698
1700
  if (isVectorExtensionSupportedByPlatform()) {
1699
1701
  try {
@@ -1703,7 +1705,7 @@ export class LocalBackend {
1703
1705
  CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit})
1704
1706
  YIELD node AS emb, distance
1705
1707
  WITH emb, distance
1706
- WHERE distance < 0.6
1708
+ WHERE distance < ${maxDistance}
1707
1709
  RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex,
1708
1710
  emb.startLine AS startLine, emb.endLine AS endLine, distance
1709
1711
  ORDER BY distance
@@ -1747,7 +1749,7 @@ export class LocalBackend {
1747
1749
  endLine: row.endLine ?? row[3] ?? 0,
1748
1750
  embedding: row.embedding ?? row[4] ?? [],
1749
1751
  }));
1750
- bestChunks = new Map(rankExactEmbeddingRows(exactRows, queryVec, limit, 0.6).map((row) => [
1752
+ bestChunks = new Map(rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [
1751
1753
  row.nodeId,
1752
1754
  {
1753
1755
  distance: row.distance,
@@ -1520,7 +1520,6 @@ export const createServer = async (port, host = '127.0.0.1') => {
1520
1520
  });
1521
1521
  }, {}, // config: use defaults
1522
1522
  undefined, // skipNodeIds
1523
- undefined, // context
1524
1523
  existingEmbeddings);
1525
1524
  // Flush WAL so subsequent /api/search requests see the new
1526
1525
  // embeddings immediately (#1149). In the CLI path closeLbug()
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "gitnexus",
3
- "version": "1.6.9-rc.32",
3
+ "version": "1.6.9-rc.34",
4
4
  "description": "Graph-powered code intelligence for AI agents. Index any codebase, query via MCP or CLI.",
5
5
  "author": "Abhigyan Patwari",
6
6
  "license": "PolyForm-Noncommercial-1.0.0",