gitnexus 1.6.9-rc.32 → 1.6.9-rc.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/i18n/en.d.ts +1 -1
- package/dist/cli/i18n/en.js +1 -1
- package/dist/cli/i18n/resources.d.ts +1 -1
- package/dist/cli/i18n/zh-CN.js +1 -1
- package/dist/core/embeddings/config.d.ts +15 -0
- package/dist/core/embeddings/config.js +37 -0
- package/dist/core/embeddings/embedding-pipeline.d.ts +3 -5
- package/dist/core/embeddings/embedding-pipeline.js +42 -36
- package/dist/core/embeddings/text-generator.d.ts +1 -1
- package/dist/core/embeddings/text-generator.js +51 -21
- package/dist/core/embeddings/types.d.ts +0 -7
- package/dist/core/run-analyze.js +1 -15
- package/dist/mcp/local/local-backend.js +4 -2
- package/dist/server/api.js +0 -1
- package/package.json +1 -1
package/dist/cli/i18n/en.d.ts
CHANGED
|
@@ -234,5 +234,5 @@ export declare const en: {
|
|
|
234
234
|
readonly 'help.option.group.contracts.type': "Filter by contract type";
|
|
235
235
|
readonly 'help.option.group.contracts.repo': "Filter by repo";
|
|
236
236
|
readonly 'help.option.group.contracts.unmatched': "Show only unmatched contracts";
|
|
237
|
-
readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
|
|
237
|
+
readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
|
|
238
238
|
};
|
package/dist/cli/i18n/en.js
CHANGED
|
@@ -234,5 +234,5 @@ export const en = {
|
|
|
234
234
|
'help.option.group.contracts.type': 'Filter by contract type',
|
|
235
235
|
'help.option.group.contracts.repo': 'Filter by repo',
|
|
236
236
|
'help.option.group.contracts.unmatched': 'Show only unmatched contracts',
|
|
237
|
-
'help.analyze.environment': '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).',
|
|
237
|
+
'help.analyze.environment': '\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).',
|
|
238
238
|
};
|
|
@@ -235,7 +235,7 @@ export declare const cliResources: {
|
|
|
235
235
|
readonly 'help.option.group.contracts.type': "Filter by contract type";
|
|
236
236
|
readonly 'help.option.group.contracts.repo': "Filter by repo";
|
|
237
237
|
readonly 'help.option.group.contracts.unmatched': "Show only unmatched contracts";
|
|
238
|
-
readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
|
|
238
|
+
readonly 'help.analyze.environment': "\nEnvironment variables:\n GITNEXUS_NO_GITIGNORE=1 Skip .gitignore parsing (still reads .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N Override large-file skip threshold (KB). Default 512, max 32768.\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker idle timeout in milliseconds. Default 30000.\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL auto-checkpoint threshold in bytes (default 67108864 = 64 MiB; -1 keeps Ladybug stock ~16 MiB).\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker job byte budget. Default 8388608.\n GITNEXUS_WORKER_POOL_SIZE=N Parse worker count override. Default cores-1 capped at 16.\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N Concurrent in-flight parse chunks. Default 2.\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N Max replacement spawns per slot before drop. Default 3.\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N Total retry wall-time per job. Default 5x sub-batch timeout.\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N Per-slot deaths to trip circuit breaker. Default max(3, poolSize).\n GITNEXUS_EMBEDDING_THREADS=N Limit local ONNX CPU threads for --embeddings.\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N Max embedding chunks for exact-scan fallback. Default 10000.\n GITNEXUS_VECTOR_MAX_DISTANCE=N Max accepted semantic/vector cosine distance (0 < N <= 2; higher values clamp to 2). Default 0.6 for MCP, 0.5 elsewhere.\n\nFlags override the corresponding env vars when both are provided.\n\nTip: `.gitnexusignore` supports `.gitignore`-style negation. Add e.g.\n `!__tests__/` to index a directory that is auto-filtered by default (#771).";
|
|
239
239
|
};
|
|
240
240
|
readonly 'zh-CN': {
|
|
241
241
|
'common.notIndexed': string;
|
package/dist/cli/i18n/zh-CN.js
CHANGED
|
@@ -234,5 +234,5 @@ export const zhCN = {
|
|
|
234
234
|
'help.option.group.contracts.type': '按契约类型过滤',
|
|
235
235
|
'help.option.group.contracts.repo': '按仓库过滤',
|
|
236
236
|
'help.option.group.contracts.unmatched': '仅显示未匹配契约',
|
|
237
|
-
'help.analyze.environment': '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n\n当参数和对应环境变量同时提供时,参数优先。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。',
|
|
237
|
+
'help.analyze.environment': '\n环境变量:\n GITNEXUS_NO_GITIGNORE=1 跳过 .gitignore 解析(仍读取 .gitnexusignore)\n GITNEXUS_MAX_FILE_SIZE=N 覆盖大文件跳过阈值(KB)。默认 512,最大 32768。\n GITNEXUS_WORKER_SUB_BATCH_TIMEOUT_MS=N Worker 空闲超时(毫秒)。默认 30000。\n GITNEXUS_WAL_CHECKPOINT_THRESHOLD=N LadybugDB WAL 自动 checkpoint 阈值(字节,默认 67108864 = 64 MiB;-1 保持 Ladybug 默认约 16 MiB)。\n GITNEXUS_WORKER_SUB_BATCH_MAX_BYTES=N Worker 作业字节预算。默认 8388608。\n GITNEXUS_WORKER_POOL_SIZE=N 解析 worker 数量覆盖值。默认 cores-1,最多 16。\n GITNEXUS_PARSE_CHUNK_CONCURRENCY=N 并发进行中的解析分块数。默认 2。\n GITNEXUS_WORKER_MAX_RESPAWNS_PER_SLOT=N 每个 slot 丢弃前允许的最大替换进程数。默认 3。\n GITNEXUS_WORKER_MAX_CUMULATIVE_TIMEOUT_MS=N 每个作业的总重试墙钟时间。默认 5 倍子批次超时。\n GITNEXUS_WORKER_CONSECUTIVE_FAILURE_THRESHOLD=N 每个 slot 触发熔断的死亡次数。默认 max(3, poolSize)。\n GITNEXUS_EMBEDDING_THREADS=N 限制 --embeddings 的本地 ONNX CPU 线程数。\n GITNEXUS_SEMANTIC_EXACT_SCAN_LIMIT=N exact-scan 回退的最大嵌入分块数。默认 10000。\n GITNEXUS_VECTOR_MAX_DISTANCE=N 语义/向量搜索接受的最大余弦距离(0 < N <= 2;超出则钳制为 2)。MCP 默认 0.6,其他路径默认 0.5。\n\n当参数和对应环境变量同时提供时,参数优先。\n\n提示:`.gitnexusignore` 支持 `.gitignore` 风格的取反。比如添加\n `!__tests__/` 可以索引默认自动过滤的目录(#771)。',
|
|
238
238
|
};
|
|
@@ -1,2 +1,17 @@
|
|
|
1
1
|
import { type EmbeddingConfig } from './types.js';
|
|
2
|
+
export declare const DEFAULT_VECTOR_MAX_DISTANCE = 0.5;
|
|
3
|
+
export declare const DEFAULT_MCP_VECTOR_MAX_DISTANCE = 0.6;
|
|
4
|
+
/**
|
|
5
|
+
* Cosine distance over normalized embeddings is bounded to [0, 2], so any threshold
|
|
6
|
+
* above this accepts every row and silently disables the relevance filter. Values
|
|
7
|
+
* over the ceiling are clamped to it rather than passed through.
|
|
8
|
+
*/
|
|
9
|
+
export declare const VECTOR_MAX_DISTANCE_CEILING = 2;
|
|
10
|
+
/**
|
|
11
|
+
* Resolve the effective max accepted vector/semantic cosine distance.
|
|
12
|
+
* Reads `GITNEXUS_VECTOR_MAX_DISTANCE`. Unset/empty/whitespace → silent fallback.
|
|
13
|
+
* Invalid (non-numeric, <= 0, non-finite) → fallback plus a one-time warning.
|
|
14
|
+
* Values above the cosine ceiling (2) are clamped to it with a one-time warning.
|
|
15
|
+
*/
|
|
16
|
+
export declare const getVectorMaxDistance: (fallback?: number) => number;
|
|
2
17
|
export declare const resolveEmbeddingConfig: (overrides?: Partial<EmbeddingConfig>) => EmbeddingConfig;
|
|
@@ -1,5 +1,42 @@
|
|
|
1
1
|
import { defaultEmbeddingThreads } from '../platform/capabilities.js';
|
|
2
|
+
import { logger } from '../logger.js';
|
|
2
3
|
import { DEFAULT_EMBEDDING_CONFIG } from './types.js';
|
|
4
|
+
export const DEFAULT_VECTOR_MAX_DISTANCE = 0.5;
|
|
5
|
+
export const DEFAULT_MCP_VECTOR_MAX_DISTANCE = 0.6;
|
|
6
|
+
/**
|
|
7
|
+
* Cosine distance over normalized embeddings is bounded to [0, 2], so any threshold
|
|
8
|
+
* above this accepts every row and silently disables the relevance filter. Values
|
|
9
|
+
* over the ceiling are clamped to it rather than passed through.
|
|
10
|
+
*/
|
|
11
|
+
export const VECTOR_MAX_DISTANCE_CEILING = 2;
|
|
12
|
+
const warned = new Set();
|
|
13
|
+
const warnOnce = (key, message) => {
|
|
14
|
+
if (warned.has(key))
|
|
15
|
+
return;
|
|
16
|
+
warned.add(key);
|
|
17
|
+
logger.warn(message);
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* Resolve the effective max accepted vector/semantic cosine distance.
|
|
21
|
+
* Reads `GITNEXUS_VECTOR_MAX_DISTANCE`. Unset/empty/whitespace → silent fallback.
|
|
22
|
+
* Invalid (non-numeric, <= 0, non-finite) → fallback plus a one-time warning.
|
|
23
|
+
* Values above the cosine ceiling (2) are clamped to it with a one-time warning.
|
|
24
|
+
*/
|
|
25
|
+
export const getVectorMaxDistance = (fallback = DEFAULT_VECTOR_MAX_DISTANCE) => {
|
|
26
|
+
const raw = process.env.GITNEXUS_VECTOR_MAX_DISTANCE;
|
|
27
|
+
if (raw === undefined || raw.trim() === '')
|
|
28
|
+
return fallback;
|
|
29
|
+
const parsed = Number(raw);
|
|
30
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
31
|
+
warnOnce(`invalid:${raw}`, ` GITNEXUS_VECTOR_MAX_DISTANCE must be a positive number in (0, ${VECTOR_MAX_DISTANCE_CEILING}], got "${raw}" — using default ${fallback}`);
|
|
32
|
+
return fallback;
|
|
33
|
+
}
|
|
34
|
+
if (parsed > VECTOR_MAX_DISTANCE_CEILING) {
|
|
35
|
+
warnOnce(`clamp:${raw}`, ` GITNEXUS_VECTOR_MAX_DISTANCE=${parsed} exceeds the cosine-distance ceiling (${VECTOR_MAX_DISTANCE_CEILING}) — clamping`);
|
|
36
|
+
return VECTOR_MAX_DISTANCE_CEILING;
|
|
37
|
+
}
|
|
38
|
+
return parsed;
|
|
39
|
+
};
|
|
3
40
|
const parsePositiveInt = (name, value, fallback) => {
|
|
4
41
|
if (value === undefined)
|
|
5
42
|
return fallback;
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* 4. Update LadybugDB with chunk-aware embeddings
|
|
9
9
|
* 5. Create vector index for semantic search
|
|
10
10
|
*/
|
|
11
|
-
import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult
|
|
11
|
+
import { type EmbeddingProgress, type EmbeddingConfig, type EmbeddableNode, type SemanticSearchResult } from './types.js';
|
|
12
12
|
import type { ExtensionInstallPolicy } from '../lbug/extension-loader.js';
|
|
13
13
|
/**
|
|
14
14
|
* Resolve the extension-install policy for the embedding WRITE path (analyze).
|
|
@@ -28,7 +28,7 @@ export declare const resolveEmbeddingInstallPolicy: () => ExtensionInstallPolicy
|
|
|
28
28
|
* invalidate existing vectors, such as metadata/header shape changes,
|
|
29
29
|
* structural container context changes, or preceding-context formatting rules.
|
|
30
30
|
*/
|
|
31
|
-
export declare const EMBEDDING_TEXT_VERSION = "
|
|
31
|
+
export declare const EMBEDDING_TEXT_VERSION = "v4";
|
|
32
32
|
/**
|
|
33
33
|
* Compute a stable content fingerprint for an embeddable node.
|
|
34
34
|
* Used to detect when the underlying text has changed so stale vectors
|
|
@@ -65,13 +65,11 @@ export interface EmbeddingPipelineResult {
|
|
|
65
65
|
* @param onProgress - Callback for progress updates
|
|
66
66
|
* @param config - Optional configuration override
|
|
67
67
|
* @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
|
|
68
|
-
* @param context - Optional repo/server context for metadata enrichment
|
|
69
68
|
* @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
|
|
70
69
|
* Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
|
|
71
70
|
* and re-embedded; nodes not in the map are embedded fresh.
|
|
72
|
-
|
|
73
71
|
*/
|
|
74
|
-
export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>,
|
|
72
|
+
export declare const runEmbeddingPipeline: (executeQuery: (cypher: string) => Promise<any[]>, executeWithReusedStatement: (cypher: string, paramsList: Array<Record<string, any>>) => Promise<void>, onProgress: EmbeddingProgressCallback, config?: Partial<EmbeddingConfig>, skipNodeIds?: Set<string>, existingEmbeddings?: Map<string, string>) => Promise<EmbeddingPipelineResult>;
|
|
75
73
|
/**
|
|
76
74
|
* Perform semantic search using the vector index with chunk deduplication
|
|
77
75
|
*/
|
|
@@ -14,7 +14,7 @@ import { generateEmbeddingText } from './text-generator.js';
|
|
|
14
14
|
import { chunkNode, characterChunk } from './chunker.js';
|
|
15
15
|
import { extractStructuralNames } from './structural-extractor.js';
|
|
16
16
|
import { EMBEDDABLE_LABELS, isShortLabel, LABEL_METHOD, LABELS_WITH_EXPORTED, STRUCTURAL_LABELS, collectBestChunks, } from './types.js';
|
|
17
|
-
import { resolveEmbeddingConfig } from './config.js';
|
|
17
|
+
import { DEFAULT_VECTOR_MAX_DISTANCE, getVectorMaxDistance, resolveEmbeddingConfig, } from './config.js';
|
|
18
18
|
import { rankExactEmbeddingRows } from './exact-search.js';
|
|
19
19
|
import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME, STALE_HASH_SENTINEL } from '../lbug/schema.js';
|
|
20
20
|
import { loadVectorExtension, createVectorIndex } from '../lbug/lbug-adapter.js';
|
|
@@ -51,7 +51,7 @@ const ensureVectorExtensionAvailable = async () => {
|
|
|
51
51
|
* invalidate existing vectors, such as metadata/header shape changes,
|
|
52
52
|
* structural container context changes, or preceding-context formatting rules.
|
|
53
53
|
*/
|
|
54
|
-
export const EMBEDDING_TEXT_VERSION = '
|
|
54
|
+
export const EMBEDDING_TEXT_VERSION = 'v4';
|
|
55
55
|
/**
|
|
56
56
|
* Compute a stable content fingerprint for an embeddable node.
|
|
57
57
|
* Used to detect when the underlying text has changed so stale vectors
|
|
@@ -185,6 +185,32 @@ const buildVectorIndex = async () => {
|
|
|
185
185
|
return false;
|
|
186
186
|
}
|
|
187
187
|
};
|
|
188
|
+
/**
|
|
189
|
+
* DELETE stale embedding rows for the given nodeIds so they can be re-inserted.
|
|
190
|
+
*
|
|
191
|
+
* Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the
|
|
192
|
+
* sanctioned pattern. A `"does not exist"` error means the rows are already gone
|
|
193
|
+
* (safe to proceed); any other error risks vector-index corruption, so it
|
|
194
|
+
* propagates and aborts the pipeline.
|
|
195
|
+
*
|
|
196
|
+
* Called per-batch (just before each batch's INSERT), not once up front — see
|
|
197
|
+
* the caller comment / KTD7: an up-front bulk delete of every stale row leaves
|
|
198
|
+
* the whole index deleted-not-reinserted if the re-embed is interrupted. Per-batch
|
|
199
|
+
* interleaving bounds that window to a single batch.
|
|
200
|
+
*/
|
|
201
|
+
const deleteStaleEmbeddingRows = async (executeWithReusedStatement, nodeIds) => {
|
|
202
|
+
if (nodeIds.length === 0)
|
|
203
|
+
return;
|
|
204
|
+
try {
|
|
205
|
+
await executeWithReusedStatement(`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, nodeIds.map((nodeId) => ({ nodeId })));
|
|
206
|
+
}
|
|
207
|
+
catch (err) {
|
|
208
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
209
|
+
if (!msg.includes('does not exist')) {
|
|
210
|
+
throw new Error(`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`);
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
};
|
|
188
214
|
/**
|
|
189
215
|
* Run the embedding pipeline
|
|
190
216
|
*
|
|
@@ -193,13 +219,11 @@ const buildVectorIndex = async () => {
|
|
|
193
219
|
* @param onProgress - Callback for progress updates
|
|
194
220
|
* @param config - Optional configuration override
|
|
195
221
|
* @param skipNodeIds - Optional set of node IDs that already have embeddings (incremental mode)
|
|
196
|
-
* @param context - Optional repo/server context for metadata enrichment
|
|
197
222
|
* @param existingEmbeddings - Optional map of nodeId → contentHash for incremental mode.
|
|
198
223
|
* Nodes whose hash matches are skipped; nodes with a changed hash are DELETE'd
|
|
199
224
|
* and re-embedded; nodes not in the map are embedded fresh.
|
|
200
|
-
|
|
201
225
|
*/
|
|
202
|
-
export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds,
|
|
226
|
+
export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatement, onProgress, config = {}, skipNodeIds, existingEmbeddings) => {
|
|
203
227
|
const finalConfig = resolveEmbeddingConfig(config);
|
|
204
228
|
let totalChunks = 0;
|
|
205
229
|
try {
|
|
@@ -233,20 +257,16 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
233
257
|
}
|
|
234
258
|
// Phase 2: Query embeddable nodes
|
|
235
259
|
let nodes = await queryEmbeddableNodes(executeQuery);
|
|
236
|
-
// Apply context metadata
|
|
237
|
-
if (context?.repoName) {
|
|
238
|
-
for (const node of nodes) {
|
|
239
|
-
node.repoName = context.repoName;
|
|
240
|
-
node.serverName = context.serverName;
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
260
|
// Incremental mode: compare content hashes, delete stale rows, skip fresh ones.
|
|
244
261
|
// Computed hashes for stale nodes are cached so batchInsertEmbeddings can reuse them
|
|
245
262
|
// (avoids double computation).
|
|
246
263
|
const computedStaleHashes = new Map();
|
|
264
|
+
// Stale rows are DELETE'd per-batch (just before each batch's INSERT) rather
|
|
265
|
+
// than all up front — see U6 / KTD7. `staleNodeIds` is consulted inside the
|
|
266
|
+
// batch loop; it stays empty in full (non-incremental) mode so no deletes fire.
|
|
267
|
+
const staleNodeIds = new Set();
|
|
247
268
|
if (existingEmbeddings && existingEmbeddings.size > 0) {
|
|
248
269
|
const beforeCount = nodes.length;
|
|
249
|
-
const staleNodeIds = [];
|
|
250
270
|
nodes = nodes.filter((n) => {
|
|
251
271
|
const existingHash = existingEmbeddings.get(n.id);
|
|
252
272
|
if (existingHash === undefined) {
|
|
@@ -257,33 +277,14 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
257
277
|
if (currentHash !== existingHash) {
|
|
258
278
|
// Content changed — cache hash for reuse during insert, mark for DELETE + re-embed
|
|
259
279
|
computedStaleHashes.set(n.id, currentHash);
|
|
260
|
-
staleNodeIds.
|
|
280
|
+
staleNodeIds.add(n.id);
|
|
261
281
|
return true;
|
|
262
282
|
}
|
|
263
283
|
// Hash matches — skip (fresh); no need to cache hash for skipped nodes
|
|
264
284
|
return false;
|
|
265
285
|
});
|
|
266
|
-
// DELETE stale embedding rows so they can be re-inserted
|
|
267
|
-
// (Kuzu forbids SET on vector-indexed properties; DELETE-then-INSERT is the sanctioned pattern)
|
|
268
|
-
if (staleNodeIds.length > 0) {
|
|
269
|
-
if (isDev) {
|
|
270
|
-
logger.info(`🔄 Deleting ${staleNodeIds.length} stale embedding rows for re-embed`);
|
|
271
|
-
}
|
|
272
|
-
try {
|
|
273
|
-
await executeWithReusedStatement(`MATCH (e:${EMBEDDING_TABLE_NAME} {nodeId: $nodeId}) DELETE e`, staleNodeIds.map((nodeId) => ({ nodeId })));
|
|
274
|
-
}
|
|
275
|
-
catch (err) {
|
|
276
|
-
// "does not exist" = rows already gone — safe to proceed.
|
|
277
|
-
// All other errors risk vector-index corruption (Kuzu requires DELETE-before-INSERT
|
|
278
|
-
// for vector-indexed properties) — propagate so the pipeline aborts cleanly.
|
|
279
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
280
|
-
if (!msg.includes('does not exist')) {
|
|
281
|
-
throw new Error(`[embed] Failed to delete stale embedding rows — aborting to prevent vector-index corruption: ${msg}`);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
}
|
|
285
286
|
if (isDev) {
|
|
286
|
-
logger.info(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.
|
|
287
|
+
logger.info(`📦 Incremental embeddings: ${beforeCount} total, ${existingEmbeddings.size} cached, ${staleNodeIds.size} stale, ${nodes.length} to embed`);
|
|
287
288
|
}
|
|
288
289
|
}
|
|
289
290
|
const totalNodes = nodes.length;
|
|
@@ -373,6 +374,11 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
373
374
|
prevTail = overlap > 0 ? chunk.text.slice(-overlap) : '';
|
|
374
375
|
}
|
|
375
376
|
}
|
|
377
|
+
// U6 / KTD7: delete this batch's stale rows immediately before its inserts,
|
|
378
|
+
// so an interrupted re-embed loses at most one batch (not the whole index).
|
|
379
|
+
// Preserves Kuzu's required DELETE-before-INSERT for vector-indexed rows.
|
|
380
|
+
const batchStaleIds = batch.filter((n) => staleNodeIds.has(n.id)).map((n) => n.id);
|
|
381
|
+
await deleteStaleEmbeddingRows(executeWithReusedStatement, batchStaleIds);
|
|
376
382
|
// Embed chunk texts in sub-batches to control memory
|
|
377
383
|
const EMBED_SUB_BATCH = finalConfig.subBatchSize;
|
|
378
384
|
for (let si = 0; si < allTexts.length; si += EMBED_SUB_BATCH) {
|
|
@@ -447,7 +453,7 @@ export const runEmbeddingPipeline = async (executeQuery, executeWithReusedStatem
|
|
|
447
453
|
/**
|
|
448
454
|
* Perform semantic search using the vector index with chunk deduplication
|
|
449
455
|
*/
|
|
450
|
-
export const semanticSearch = async (executeQuery, query, k = 10, maxDistance =
|
|
456
|
+
export const semanticSearch = async (executeQuery, query, k = 10, maxDistance = getVectorMaxDistance(DEFAULT_VECTOR_MAX_DISTANCE)) => {
|
|
451
457
|
if (!isEmbedderReady()) {
|
|
452
458
|
throw new Error('Embedding model not initialized. Run embedding pipeline first.');
|
|
453
459
|
}
|
|
@@ -569,7 +575,7 @@ export const semanticSearch = async (executeQuery, query, k = 10, maxDistance =
|
|
|
569
575
|
* Semantic search with graph expansion (flattened results)
|
|
570
576
|
*/
|
|
571
577
|
export const semanticSearchWithContext = async (executeQuery, query, k = 5, _hops = 1) => {
|
|
572
|
-
const results = await semanticSearch(executeQuery, query, k
|
|
578
|
+
const results = await semanticSearch(executeQuery, query, k);
|
|
573
579
|
return results.map((r) => ({
|
|
574
580
|
matchId: r.nodeId,
|
|
575
581
|
matchName: r.name,
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Text Generator Module
|
|
3
3
|
*
|
|
4
|
-
* Generates
|
|
4
|
+
* Generates compact, description-forward embedding text from code nodes.
|
|
5
5
|
* Supports chunkable labels (Function/Method with AST chunking),
|
|
6
6
|
* Class-specific structural text, and short-node direct embed.
|
|
7
7
|
*
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Text Generator Module
|
|
3
3
|
*
|
|
4
|
-
* Generates
|
|
4
|
+
* Generates compact, description-forward embedding text from code nodes.
|
|
5
5
|
* Supports chunkable labels (Function/Method with AST chunking),
|
|
6
6
|
* Class-specific structural text, and short-node direct embed.
|
|
7
7
|
*
|
|
@@ -41,27 +41,48 @@ const cleanContent = (content) => {
|
|
|
41
41
|
.trim();
|
|
42
42
|
};
|
|
43
43
|
/**
|
|
44
|
-
*
|
|
44
|
+
* Compact location signal for the embedding header: the last 1-2 path segments
|
|
45
|
+
* (immediate parent dir + basename), never the full deep path.
|
|
46
|
+
*
|
|
47
|
+
* #2333 / PR #2334 tri-review: U1 dropped the location entirely, which regressed
|
|
48
|
+
* path/service-qualified semantic search (e.g. `billing/handler` vs
|
|
49
|
+
* `identity/handler` in a monorepo) — and FTS indexes only name/content/description,
|
|
50
|
+
* never `filePath`, so there is no keyword backfill. The bounded form restores the
|
|
51
|
+
* discriminating tokens (service dir + filename-concept) at a fraction of the
|
|
52
|
+
* dilution the full path caused.
|
|
45
53
|
*/
|
|
46
|
-
const
|
|
54
|
+
const boundedLocation = (filePath) => {
|
|
55
|
+
const segments = filePath.replace(/\\/g, '/').split('/').filter(Boolean);
|
|
56
|
+
return segments.slice(-2).join('/');
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Build a compact, description-forward header for embedding text.
|
|
60
|
+
*
|
|
61
|
+
* Issue #2333 (sub-issue of #2326), Option A: lead the embedding text with the
|
|
62
|
+
* symbol name + doc-comment description and drop the low-signal metadata lines
|
|
63
|
+
* (`Repo`/`Server`/`Export` and the verbose full `Path`). For short doc comments
|
|
64
|
+
* those lines used to be ~25-30% of the embedding text, diluting the description's
|
|
65
|
+
* semantic weight in the vector and weakening description-shaped search — worst
|
|
66
|
+
* for CJK, where a complete concept is often 4-20 characters.
|
|
67
|
+
*
|
|
68
|
+
* A *bounded* location signal (last 1-2 path segments) is kept after the
|
|
69
|
+
* description — see `boundedLocation` for why the full path drop was reversed.
|
|
70
|
+
*
|
|
71
|
+
* Full metadata is unaffected: it lives on the graph node properties, which is
|
|
72
|
+
* what display/context tools read. Only the embedding text changes here.
|
|
73
|
+
*
|
|
74
|
+
* Option B (reorder only, keep metadata) was rejected — mean-pooled embeddings
|
|
75
|
+
* weight by token proportion, not position, so reordering alone barely moves the
|
|
76
|
+
* signal. Option C (a separate description-only embedding + hybrid merge) is
|
|
77
|
+
* deferred to follow-up; build it only if Option A proves insufficient against
|
|
78
|
+
* real measurement. Any change to this template MUST bump EMBEDDING_TEXT_VERSION.
|
|
79
|
+
*/
|
|
80
|
+
const buildEmbeddingHeader = (node, config) => {
|
|
47
81
|
const parts = [];
|
|
48
82
|
// Label + name
|
|
49
83
|
parts.push(`${node.label}: ${node.name}`);
|
|
50
|
-
//
|
|
51
|
-
|
|
52
|
-
parts.push(`Repo: ${node.repoName}`);
|
|
53
|
-
}
|
|
54
|
-
// Server name (optional)
|
|
55
|
-
if (node.serverName) {
|
|
56
|
-
parts.push(`Server: ${node.serverName}`);
|
|
57
|
-
}
|
|
58
|
-
// Full file path
|
|
59
|
-
parts.push(`Path: ${node.filePath}`);
|
|
60
|
-
// Export status
|
|
61
|
-
if (node.isExported !== undefined) {
|
|
62
|
-
parts.push(`Export: ${node.isExported}`);
|
|
63
|
-
}
|
|
64
|
-
// Description (truncated)
|
|
84
|
+
// Description hoisted above everything else so its semantic signal dominates
|
|
85
|
+
// the embedding vector and is never the part lost to token-limit truncation.
|
|
65
86
|
if (node.description) {
|
|
66
87
|
const maxLen = config.maxDescriptionLength ?? DEFAULT_EMBEDDING_CONFIG.maxDescriptionLength;
|
|
67
88
|
const truncated = truncateDescription(node.description, maxLen);
|
|
@@ -69,10 +90,19 @@ const buildMetadataHeader = (node, config) => {
|
|
|
69
90
|
parts.push(truncated);
|
|
70
91
|
}
|
|
71
92
|
}
|
|
93
|
+
// Bounded location signal — placed after the description so the description
|
|
94
|
+
// still leads the vector. Restores path/service disambiguation lost when the
|
|
95
|
+
// full Path line was dropped (FTS does not index filePath to backfill it).
|
|
96
|
+
if (node.filePath) {
|
|
97
|
+
const loc = boundedLocation(node.filePath);
|
|
98
|
+
if (loc) {
|
|
99
|
+
parts.push(`Loc: ${loc}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
72
102
|
return parts.join('\n');
|
|
73
103
|
};
|
|
74
104
|
const generateCodeBodyText = (node, codeBody, config, prevTail) => {
|
|
75
|
-
const header =
|
|
105
|
+
const header = buildEmbeddingHeader(node, config);
|
|
76
106
|
const parts = [header];
|
|
77
107
|
if (prevTail) {
|
|
78
108
|
parts.push(`[preceding context]: ...${cleanContent(prevTail)}`);
|
|
@@ -87,7 +117,7 @@ const getCompactContainerContext = (cleanedContent, declarationOnly) => {
|
|
|
87
117
|
return firstLine ? `Container: ${firstLine}` : undefined;
|
|
88
118
|
};
|
|
89
119
|
const generateStructuralTypeText = (node, codeBody, config, chunkIndex, prevTail) => {
|
|
90
|
-
const header =
|
|
120
|
+
const header = buildEmbeddingHeader(node, config);
|
|
91
121
|
const parts = [header];
|
|
92
122
|
const isFirstChunk = chunkIndex === undefined || chunkIndex === 0;
|
|
93
123
|
const cleanedContent = cleanContent(node.content);
|
|
@@ -192,7 +222,7 @@ export const extractDeclarationOnly = (content) => {
|
|
|
192
222
|
*/
|
|
193
223
|
export const generateEmbeddingText = (node, codeBody, config = {}, chunkIndex, prevTail) => {
|
|
194
224
|
if (isShortLabel(node.label)) {
|
|
195
|
-
const header =
|
|
225
|
+
const header = buildEmbeddingHeader(node, config);
|
|
196
226
|
const cleaned = cleanContent(node.content);
|
|
197
227
|
return `${header}\n\n${cleaned}`;
|
|
198
228
|
}
|
|
@@ -168,13 +168,6 @@ export interface CachedEmbedding {
|
|
|
168
168
|
embedding: number[];
|
|
169
169
|
contentHash?: string;
|
|
170
170
|
}
|
|
171
|
-
/**
|
|
172
|
-
* Context info for embedding pipeline (repo/server metadata enrichment)
|
|
173
|
-
*/
|
|
174
|
-
export interface EmbeddingContext {
|
|
175
|
-
repoName?: string;
|
|
176
|
-
serverName?: string;
|
|
177
|
-
}
|
|
178
171
|
/**
|
|
179
172
|
* Model download progress from transformers.js
|
|
180
173
|
*/
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -982,20 +982,6 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
|
|
|
982
982
|
existingEmbeddings.set(e.nodeId, e.contentHash ?? STALE_HASH_SENTINEL);
|
|
983
983
|
}
|
|
984
984
|
}
|
|
985
|
-
const { readServerMapping } = await import('./embeddings/server-mapping.js');
|
|
986
|
-
// Mirror the registry's name-resolution chain so the server-mapping
|
|
987
|
-
// lookup key stays aligned with the final registry name (#1259):
|
|
988
|
-
// --name → remote-derived → canonical-root basename
|
|
989
|
-
// (preserved-alias is intentionally NOT consulted here — server
|
|
990
|
-
// mappings are addressed by the operationally-meaningful name the
|
|
991
|
-
// user configures, not by a sticky registry-only alias they may not
|
|
992
|
-
// know about. The previous canonical-only logic ignored both --name
|
|
993
|
-
// and remote-derived names, silently breaking server-mapping for
|
|
994
|
-
// anyone with a `--name` alias or remote-named repo.)
|
|
995
|
-
const projectName = options.registryName ??
|
|
996
|
-
getInferredRepoName(repoPath) ??
|
|
997
|
-
path.basename(resolveRepoIdentityRoot(repoPath));
|
|
998
|
-
const serverName = await readServerMapping(projectName);
|
|
999
985
|
const embeddingResult = await runEmbeddingPipeline(executeQuery, executeWithReusedStatement, (p) => {
|
|
1000
986
|
const scaled = 90 + Math.round((p.percent / 100) * 8);
|
|
1001
987
|
const label = p.phase === 'loading-model'
|
|
@@ -1004,7 +990,7 @@ export async function runFullAnalysis(repoPath, options, callbacks) {
|
|
|
1004
990
|
: 'Loading embedding model...'
|
|
1005
991
|
: `Embedding ${p.nodesProcessed || 0}/${p.totalNodes || '?'}`;
|
|
1006
992
|
progress('embeddings', scaled, label);
|
|
1007
|
-
}, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined,
|
|
993
|
+
}, {}, cachedEmbeddingNodeIds.size > 0 ? cachedEmbeddingNodeIds : undefined, existingEmbeddings);
|
|
1008
994
|
if (embeddingResult.semanticMode === 'exact-scan') {
|
|
1009
995
|
semanticMode = 'exact-scan';
|
|
1010
996
|
log('Semantic embeddings were generated without a VECTOR index; ' +
|
|
@@ -21,6 +21,7 @@ import { listRegisteredRepos, cleanupOldKuzuFiles, canonicalizePath, getStorageP
|
|
|
21
21
|
import { GroupService, } from '../../core/group/service.js';
|
|
22
22
|
import { resolveAtGroupMemberRepoPath } from '../../core/group/resolve-at-member.js';
|
|
23
23
|
import { collectBestChunks } from '../../core/embeddings/types.js';
|
|
24
|
+
import { DEFAULT_MCP_VECTOR_MAX_DISTANCE, getVectorMaxDistance, } from '../../core/embeddings/config.js';
|
|
24
25
|
import { rankExactEmbeddingRows, } from '../../core/embeddings/exact-search.js';
|
|
25
26
|
import { EMBEDDING_TABLE_NAME, EMBEDDING_INDEX_NAME } from '../../core/lbug/schema.js';
|
|
26
27
|
import { getExactScanLimit, isVectorExtensionSupportedByPlatform, } from '../../core/platform/capabilities.js';
|
|
@@ -1694,6 +1695,7 @@ export class LocalBackend {
|
|
|
1694
1695
|
const queryVec = await embedQuery(query);
|
|
1695
1696
|
const dims = getEmbeddingDims();
|
|
1696
1697
|
const queryVecStr = `[${queryVec.join(',')}]`;
|
|
1698
|
+
const maxDistance = getVectorMaxDistance(DEFAULT_MCP_VECTOR_MAX_DISTANCE);
|
|
1697
1699
|
let bestChunks = new Map();
|
|
1698
1700
|
if (isVectorExtensionSupportedByPlatform()) {
|
|
1699
1701
|
try {
|
|
@@ -1703,7 +1705,7 @@ export class LocalBackend {
|
|
|
1703
1705
|
CAST(${queryVecStr} AS FLOAT[${dims}]), ${fetchLimit})
|
|
1704
1706
|
YIELD node AS emb, distance
|
|
1705
1707
|
WITH emb, distance
|
|
1706
|
-
WHERE distance <
|
|
1708
|
+
WHERE distance < ${maxDistance}
|
|
1707
1709
|
RETURN emb.nodeId AS nodeId, emb.chunkIndex AS chunkIndex,
|
|
1708
1710
|
emb.startLine AS startLine, emb.endLine AS endLine, distance
|
|
1709
1711
|
ORDER BY distance
|
|
@@ -1747,7 +1749,7 @@ export class LocalBackend {
|
|
|
1747
1749
|
endLine: row.endLine ?? row[3] ?? 0,
|
|
1748
1750
|
embedding: row.embedding ?? row[4] ?? [],
|
|
1749
1751
|
}));
|
|
1750
|
-
bestChunks = new Map(rankExactEmbeddingRows(exactRows, queryVec, limit,
|
|
1752
|
+
bestChunks = new Map(rankExactEmbeddingRows(exactRows, queryVec, limit, maxDistance).map((row) => [
|
|
1751
1753
|
row.nodeId,
|
|
1752
1754
|
{
|
|
1753
1755
|
distance: row.distance,
|
package/dist/server/api.js
CHANGED
|
@@ -1520,7 +1520,6 @@ export const createServer = async (port, host = '127.0.0.1') => {
|
|
|
1520
1520
|
});
|
|
1521
1521
|
}, {}, // config: use defaults
|
|
1522
1522
|
undefined, // skipNodeIds
|
|
1523
|
-
undefined, // context
|
|
1524
1523
|
existingEmbeddings);
|
|
1525
1524
|
// Flush WAL so subsequent /api/search requests see the new
|
|
1526
1525
|
// embeddings immediately (#1149). In the CLI path closeLbug()
|
package/package.json
CHANGED