noteconnection 1.6.8 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -21
- package/README.md +258 -64
- package/dist/src/agent_workspace.contract.parity.test.js +475 -0
- package/dist/src/agent_workspace.frontend.test.js +9989 -0
- package/dist/src/agent_workspace.locale.contract.test.js +95 -0
- package/dist/src/agent_workspace.runtime.behavior.test.js +5072 -0
- package/dist/src/copy.assets.contract.test.js +130 -0
- package/dist/src/core/PathBridge.js +28 -3
- package/dist/src/export/WorkspaceExportBundle.js +1130 -0
- package/dist/src/export/WorkspaceExportBundle.test.js +1480 -0
- package/dist/src/export/index.js +18 -0
- package/dist/src/export/types.js +2 -0
- package/dist/src/fixrisk.issue.verifier.contract.test.js +3 -1
- package/dist/src/foundation.ann.runtime.contract.test.js +81 -0
- package/dist/src/foundation.release.evidence.contract.test.js +406 -0
- package/dist/src/foundation.sqlite.runtime.contract.test.js +82 -0
- package/dist/src/frontend/README.md +258 -64
- package/dist/src/frontend/agent_workspace.js +5353 -0
- package/dist/src/frontend/agent_workspace_runtime.js +4434 -0
- package/dist/src/frontend/analysis.js +31 -4
- package/dist/src/frontend/app.js +3597 -141
- package/dist/src/frontend/focus_mode_interactions.js +33 -0
- package/dist/src/frontend/godot_future_path_renderer.js +540 -0
- package/dist/src/frontend/godot_tree_interactions.js +178 -0
- package/dist/src/frontend/graph_state.mjs +105 -0
- package/dist/src/frontend/hosted_future_path_runtime.js +157 -0
- package/dist/src/frontend/i18n.mjs +186 -0
- package/dist/src/frontend/index.html +641 -266
- package/dist/src/frontend/layout_gpu.js +12 -4
- package/dist/src/frontend/locales/en.json +721 -5
- package/dist/src/frontend/locales/zh.json +721 -5
- package/dist/src/frontend/main.mjs +60 -0
- package/dist/src/frontend/markdown_runtime.js +827 -0
- package/dist/src/frontend/notemd.css +49 -0
- package/dist/src/frontend/notemd.html +7 -1
- package/dist/src/frontend/notemd.js +64 -0
- package/dist/src/frontend/path.html +107 -0
- package/dist/src/frontend/path_app.js +2189 -150
- package/dist/src/frontend/path_layout.mjs +143 -0
- package/dist/src/frontend/path_mermaid_utils.mjs +108 -0
- package/dist/src/frontend/path_modules_bridge.js +486 -0
- package/dist/src/frontend/path_state.mjs +118 -0
- package/dist/src/frontend/path_styles.css +146 -0
- package/dist/src/frontend/path_worker_bridge.mjs +85 -0
- package/dist/src/frontend/reader.js +522 -27
- package/dist/src/frontend/runtime_bridge.js +67 -54
- package/dist/src/frontend/runtime_bridge.mjs +279 -0
- package/dist/src/frontend/settings.js +130 -12
- package/dist/src/frontend/simulationWorker.js +241 -6
- package/dist/src/frontend/source_manager.js +190 -21
- package/dist/src/frontend/styles.css +2853 -72
- package/dist/src/frontend/workbench_state.mjs +101 -0
- package/dist/src/frontend/workspace_panes.js +10168 -0
- package/dist/src/frontend.locale.contract.test.js +62 -0
- package/dist/src/godot.sidecar.bootstrap.contract.test.js +244 -0
- package/dist/src/indexing/IndexLifecycle.js +195 -0
- package/dist/src/indexing/IndexLifecycle.test.js +49 -0
- package/dist/src/indexing/SegmentBuilder.js +64 -0
- package/dist/src/indexing/UnitBuilder.js +48 -0
- package/dist/src/indexing/types.js +2 -0
- package/dist/src/knowledge.api.contract.test.js +170 -0
- package/dist/src/learning/KnowledgeLearningPlatform.js +10386 -0
- package/dist/src/learning/KnowledgeLearningPlatform.persistence.test.js +327 -0
- package/dist/src/learning/KnowledgeLearningPlatform.program-f.test.js +99 -0
- package/dist/src/learning/KnowledgeLearningPlatform.test.js +2971 -0
- package/dist/src/learning/KnowledgeWorkspaceConversationRegression.js +2974 -0
- package/dist/src/learning/KnowledgeWorkspaceConversationRegression.test.js +3928 -0
- package/dist/src/learning/answerReleaseReview.js +4319 -0
- package/dist/src/learning/answerReleaseReview.test.js +2888 -0
- package/dist/src/learning/api.js +2 -0
- package/dist/src/learning/conversationComposer.js +1480 -0
- package/dist/src/learning/conversationComposer.test.js +1817 -0
- package/dist/src/learning/domains/ConversationManager.js +53 -0
- package/dist/src/learning/domains/KnowledgeIngestor.js +238 -0
- package/dist/src/learning/domains/KnowledgeQuerier.js +187 -0
- package/dist/src/learning/domains/MasteryEngine.js +387 -0
- package/dist/src/learning/domains/MemoryPolicyManager.js +408 -0
- package/dist/src/learning/domains/QualityEvaluator.js +307 -0
- package/dist/src/learning/domains/TutorRouter.js +313 -0
- package/dist/src/learning/domains/index.js +33 -0
- package/dist/src/learning/domains/types.js +7 -0
- package/dist/src/learning/errors.js +29 -0
- package/dist/src/learning/evidenceContextAssembler.js +1176 -0
- package/dist/src/learning/evidenceContextAssembler.test.js +6332 -0
- package/dist/src/learning/graphContextAssembler.js +870 -0
- package/dist/src/learning/graphContextAssembler.test.js +1033 -0
- package/dist/src/learning/index.js +28 -0
- package/dist/src/learning/queryBackend.js +1898 -0
- package/dist/src/learning/queryBackend.test.js +955 -0
- package/dist/src/learning/ragContextPack.js +257 -0
- package/dist/src/learning/ragContextPack.test.js +160 -0
- package/dist/src/learning/ragPublicText.js +38 -0
- package/dist/src/learning/ragSufficiencyJudge.js +161 -0
- package/dist/src/learning/ragSufficiencyJudge.test.js +177 -0
- package/dist/src/learning/ragSufficiencyProviderJudge.js +227 -0
- package/dist/src/learning/ragSufficiencyProviderJudge.test.js +156 -0
- package/dist/src/learning/requestNormalization.js +198 -0
- package/dist/src/learning/runtimeCapability.js +4677 -0
- package/dist/src/learning/runtimeCapability.test.js +3635 -0
- package/dist/src/learning/store.js +1240 -0
- package/dist/src/learning/store.test.js +1126 -0
- package/dist/src/learning/tutorAdapter.js +2 -0
- package/dist/src/learning/types.js +2 -0
- package/dist/src/learning/vectorAccelerationAdapter.js +942 -0
- package/dist/src/learning/vectorAccelerationAdapter.test.js +382 -0
- package/dist/src/lfs.asset.policy.contract.test.js +153 -0
- package/dist/src/license.policy.contract.test.js +66 -0
- package/dist/src/memory/MemoryGovernance.js +74 -0
- package/dist/src/memory/MemoryGovernance.test.js +46 -0
- package/dist/src/memory/types.js +2 -0
- package/dist/src/mermaid.frontend.guard.contract.test.js +77 -0
- package/dist/src/middleware/auth.js +17 -0
- package/dist/src/middleware/body-parser.js +45 -0
- package/dist/src/middleware/cors.js +44 -0
- package/dist/src/middleware/index.js +21 -0
- package/dist/src/middleware/request-trace.js +96 -0
- package/dist/src/notemd/AppConfigToml.js +6 -4
- package/dist/src/notemd/MermaidProcessor.js +400 -50
- package/dist/src/notemd/NotemdService.js +498 -13
- package/dist/src/notemd/PromptManager.js +15 -0
- package/dist/src/notemd/cli/commands.js +357 -0
- package/dist/src/notemd/cli/dispatcher.js +225 -0
- package/dist/src/notemd/cli/index.js +169 -0
- package/dist/src/notemd/cli/parser.js +68 -0
- package/dist/src/notemd/cli/types.js +2 -0
- package/dist/src/notemd/constants.js +43 -0
- package/dist/src/notemd/diagram/diagramGenerationService.js +78 -0
- package/dist/src/notemd/diagram/diagramSpec.js +79 -0
- package/dist/src/notemd/diagram/diagramSpecResponseParser.js +131 -0
- package/dist/src/notemd/diagram/intent.js +95 -0
- package/dist/src/notemd/diagram/planner.js +71 -0
- package/dist/src/notemd/diagram/prompts/diagramSpecPrompt.js +42 -0
- package/dist/src/notemd/diagram/types.js +18 -0
- package/dist/src/notemd/index.js +26 -0
- package/dist/src/notemd/operations/capabilityManifest.js +23 -0
- package/dist/src/notemd/operations/cliContracts.js +17 -0
- package/dist/src/notemd/operations/configProfileCommands.js +85 -0
- package/dist/src/notemd/operations/registry.contract.test.js +95 -0
- package/dist/src/notemd/operations/registry.js +991 -0
- package/dist/src/notemd/operations/types.js +2 -0
- package/dist/src/notemd/providerDiagnostics.js +220 -0
- package/dist/src/notemd/providerProfiles.js +42 -0
- package/dist/src/notemd/providerTemplates.js +231 -0
- package/dist/src/notemd/search/DuckDuckGoProvider.js +39 -0
- package/dist/src/notemd/search/SearchManager.js +13 -0
- package/dist/src/notemd/search/SearchProvider.js +2 -0
- package/dist/src/notemd/search/TavilyProvider.js +44 -0
- package/dist/src/notemd.agent.manifest.test.js +85 -0
- package/dist/src/notemd.api.contract.test.js +14 -1
- package/dist/src/notemd.app_config_toml.test.js +2 -1
- package/dist/src/notemd.batch.workflow.test.js +117 -0
- package/dist/src/notemd.cli.e2e.test.js +136 -0
- package/dist/src/notemd.core.test.js +51 -0
- package/dist/src/notemd.diagram.pipeline.test.js +233 -0
- package/dist/src/notemd.providerTemplates.test.js +34 -0
- package/dist/src/notemd.server.integration.test.js +143 -35
- package/dist/src/notemd.workflow.pipeline.test.js +162 -0
- package/dist/src/pathbridge.handshake.contract.test.js +16 -2
- package/dist/src/pathmode.background.contract.test.js +69 -0
- package/dist/src/pathmode.settings.api.contract.test.js +9 -0
- package/dist/src/pkg.sidecar.contract.test.js +9 -3
- package/dist/src/platform/ExportProfile.js +58 -0
- package/dist/src/platform/PlatformCapabilities.js +45 -0
- package/dist/src/platform/PlatformCapabilities.test.js +30 -0
- package/dist/src/platform/RenderMaterializer.js +33 -0
- package/dist/src/platform/RenderMaterializer.test.js +32 -0
- package/dist/src/query_backend.external_http.integration.test.js +410 -0
- package/dist/src/reader_renderer.js +404 -3
- package/dist/src/reader_renderer.test.js +87 -0
- package/dist/src/release.godot.mirror.contract.test.js +73 -0
- package/dist/src/resources/ResourceRegistry.js +223 -0
- package/dist/src/resources/ResourceRegistry.test.js +61 -0
- package/dist/src/resources/types.js +2 -0
- package/dist/src/routes/agentWorkspaceDiagnostics.js +173 -0
- package/dist/src/routes/data.js +267 -0
- package/dist/src/routes/diagnostics.js +51 -0
- package/dist/src/routes/index.js +23 -0
- package/dist/src/routes/knowledge.js +968 -0
- package/dist/src/routes/markdown.js +287 -0
- package/dist/src/routes/notemd.js +565 -0
- package/dist/src/routes/registry.contract.test.js +130 -0
- package/dist/src/routes/render.js +285 -0
- package/dist/src/routes/runtimeRunbookRouteOps.js +149 -0
- package/dist/src/routes/runtimeRunbookRouteOps.test.js +194 -0
- package/dist/src/routes/settings.js +6 -0
- package/dist/src/routes/staticFiles.js +94 -0
- package/dist/src/routes/types.js +2 -0
- package/dist/src/runtime.transport.adapter.contract.test.js +81 -0
- package/dist/src/server.js +11090 -1549
- package/dist/src/server.migration.test.js +193 -21
- package/dist/src/server.port.fallback.contract.test.js +63 -0
- package/dist/src/session/SessionStateStore.js +81 -0
- package/dist/src/session/SessionStateStore.test.js +58 -0
- package/dist/src/session/types.js +2 -0
- package/dist/src/settings.runtime.contract.test.js +50 -0
- package/dist/src/shared/types.contract.test.js +107 -0
- package/dist/src/shared/types.js +22 -0
- package/dist/src/sidecar.replacement.boundary.contract.test.js +128 -0
- package/dist/src/sidecar.supply.readiness.contract.test.js +144 -0
- package/dist/src/source_manager.loadflow.test.js +46 -0
- package/dist/src/startup.layout.snapshot.contract.test.js +57 -0
- package/dist/src/tauri.frontend.build.contract.test.js +60 -0
- package/dist/src/tauri.sidecar.cleanup.contract.test.js +21 -0
- package/dist/src/utils/RuntimePaths.js +4 -13
- package/dist/src/utils/platform.js +153 -0
- package/dist/src/workflows/WorkflowArtifactStore.js +96 -0
- package/dist/src/workflows/WorkflowArtifactStore.test.js +80 -0
- package/dist/src/workflows/types.js +2 -0
- package/dist/src/workspace/WorkspaceRegistry.js +122 -0
- package/dist/src/workspace/WorkspaceRegistry.test.js +29 -0
- package/dist/src/workspace/types.js +2 -0
- package/package.json +61 -10
|
@@ -0,0 +1,1898 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.LocalVectorGraphQueryBackend = exports.KeywordOnlyGraphQueryBackend = exports.LocalHybridGraphQueryBackend = void 0;
|
|
37
|
+
exports.evaluateAnnRunbookHealth = evaluateAnnRunbookHealth;
|
|
38
|
+
exports.normalizeGraphQueryBackendType = normalizeGraphQueryBackendType;
|
|
39
|
+
exports.normalizeLocalVectorAccelerationFailureMode = normalizeLocalVectorAccelerationFailureMode;
|
|
40
|
+
exports.createGraphQueryBackend = createGraphQueryBackend;
|
|
41
|
+
exports.listGraphQueryBackendCatalog = listGraphQueryBackendCatalog;
|
|
42
|
+
const crypto_1 = require("crypto");
|
|
43
|
+
const fs = __importStar(require("fs"));
|
|
44
|
+
const path = __importStar(require("path"));
|
|
45
|
+
function evaluateAnnRunbookHealth(params) {
|
|
46
|
+
const checks = [];
|
|
47
|
+
const threshold = params.failureThreshold ?? 3;
|
|
48
|
+
const minEffectiveness = params.minPrefilterEffectivenessRatio ?? 0.01;
|
|
49
|
+
// Check 1: Circuit state
|
|
50
|
+
const circuitHealthy = params.circuitState !== 'open';
|
|
51
|
+
checks.push({
|
|
52
|
+
name: 'circuit_state',
|
|
53
|
+
passed: circuitHealthy,
|
|
54
|
+
value: params.circuitState ?? 'unknown',
|
|
55
|
+
threshold: 'closed',
|
|
56
|
+
operator: 'eq',
|
|
57
|
+
message: circuitHealthy ? 'Circuit closed' : 'Circuit open — requests blocked',
|
|
58
|
+
});
|
|
59
|
+
// Check 2: Consecutive failures
|
|
60
|
+
const failuresHealthy = (params.consecutiveFailures ?? 0) < threshold;
|
|
61
|
+
checks.push({
|
|
62
|
+
name: 'consecutive_failures',
|
|
63
|
+
passed: failuresHealthy,
|
|
64
|
+
value: params.consecutiveFailures ?? 0,
|
|
65
|
+
threshold,
|
|
66
|
+
operator: 'lt',
|
|
67
|
+
message: failuresHealthy ? 'Within threshold' : `Exceeded threshold (${params.consecutiveFailures} >= ${threshold})`,
|
|
68
|
+
});
|
|
69
|
+
// Check 3: Prefilter effectiveness
|
|
70
|
+
const effectiveness = params.prefilterEffectivenessRatio ?? 1;
|
|
71
|
+
const prefilterHealthy = effectiveness >= minEffectiveness;
|
|
72
|
+
checks.push({
|
|
73
|
+
name: 'prefilter_effectiveness',
|
|
74
|
+
passed: prefilterHealthy,
|
|
75
|
+
value: effectiveness,
|
|
76
|
+
threshold: minEffectiveness,
|
|
77
|
+
operator: 'gte',
|
|
78
|
+
message: prefilterHealthy ? 'Effective prefilter' : 'Prefilter too narrow — may miss relevant atoms',
|
|
79
|
+
});
|
|
80
|
+
// Check 4: Representation consistency
|
|
81
|
+
const repHealthy = params.representationStatus !== 'mismatch';
|
|
82
|
+
checks.push({
|
|
83
|
+
name: 'representation_consistency',
|
|
84
|
+
passed: repHealthy || !params.representationStrictMode,
|
|
85
|
+
value: params.representationStatus ?? 'unknown',
|
|
86
|
+
threshold: 'aligned',
|
|
87
|
+
operator: 'eq',
|
|
88
|
+
message: repHealthy ? 'Aligned' : `Mismatch: ${params.representationStatus}`,
|
|
89
|
+
});
|
|
90
|
+
const failed = checks.filter(c => !c.passed);
|
|
91
|
+
const status = failed.length === 0 ? 'healthy'
|
|
92
|
+
: failed.length <= 1 && checks[0].passed ? 'degraded'
|
|
93
|
+
: 'unhealthy';
|
|
94
|
+
return {
|
|
95
|
+
status,
|
|
96
|
+
checks,
|
|
97
|
+
summary: failed.length === 0
|
|
98
|
+
? 'All ANN health checks passed.'
|
|
99
|
+
: `${failed.length}/${checks.length} checks failed: ${failed.map(c => c.name).join(', ')}.`,
|
|
100
|
+
checkedAt: new Date().toISOString(),
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
const GRAPH_QUERY_BACKEND_CATALOG = [
|
|
104
|
+
{
|
|
105
|
+
backend: 'local_hybrid',
|
|
106
|
+
backendId: 'local-hybrid-v1',
|
|
107
|
+
label: 'Local Hybrid',
|
|
108
|
+
aliases: ['local_hybrid', 'local-hybrid', 'hybrid'],
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
backend: 'keyword_only',
|
|
112
|
+
backendId: 'keyword-only-v1',
|
|
113
|
+
label: 'Keyword Only',
|
|
114
|
+
aliases: ['keyword_only', 'keyword-only', 'keyword'],
|
|
115
|
+
},
|
|
116
|
+
{
|
|
117
|
+
backend: 'local_vector',
|
|
118
|
+
backendId: 'local-vector-v1',
|
|
119
|
+
label: 'Local Vector',
|
|
120
|
+
aliases: ['local_vector', 'local-vector', 'vector', 'semantic_vector', 'semantic-vector'],
|
|
121
|
+
},
|
|
122
|
+
];
|
|
123
|
+
const SEMANTIC_CONTENT_TOKEN_LIMIT = 180;
|
|
124
|
+
const SEMANTIC_TITLE_TOKEN_LIMIT = 24;
|
|
125
|
+
const SEMANTIC_KEYWORD_TOKEN_LIMIT = 40;
|
|
126
|
+
const SEMANTIC_QUERY_TOKEN_LIMIT = 48;
|
|
127
|
+
const SEMANTIC_CJK_NGRAM_LIMIT = 96;
|
|
128
|
+
const LOCAL_VECTOR_INDEX_SCHEMA_VERSION = 2;
|
|
129
|
+
const LOCAL_VECTOR_INDEX_BACKEND_ID = 'local-vector-v1';
|
|
130
|
+
const LOCAL_VECTOR_INDEX_SIGNATURE_VERSION = 'local-vector-index-signature-v2';
|
|
131
|
+
const LOCAL_VECTOR_ACCELERATION_ADAPTER_ID = 'local-vector-acceleration-ann-v1';
|
|
132
|
+
const LOCAL_VECTOR_REPRESENTATION_VERSION = 'local-vector-representation-v2';
|
|
133
|
+
const LOCAL_VECTOR_EMBEDDING_MODEL_ID = 'local-semantic-tfidf-unicode-v2';
|
|
134
|
+
const LOCAL_VECTOR_ANN_MIN_ATOMS = 96;
|
|
135
|
+
const LOCAL_VECTOR_ANN_TOP_QUERY_TOKENS = 8;
|
|
136
|
+
const LOCAL_VECTOR_ANN_CANDIDATE_MULTIPLIER = 28;
|
|
137
|
+
const LOCAL_VECTOR_ANN_CANDIDATE_MIN_FLOOR = 48;
|
|
138
|
+
const LOCAL_VECTOR_ANN_SIGNATURE_PREFIX_BITS = 14;
|
|
139
|
+
const LOCAL_VECTOR_ANN_SIGNATURE_SCORE_BONUS = 0.08;
|
|
140
|
+
const LOCAL_VECTOR_ANN_HASH_CACHE_SOFT_LIMIT = 50000;
|
|
141
|
+
const LOCAL_VECTOR_TOKEN_HASH_CACHE = new Map();
|
|
142
|
+
function clamp(value, minValue, maxValue) {
|
|
143
|
+
return Math.min(maxValue, Math.max(minValue, value));
|
|
144
|
+
}
|
|
145
|
+
function normalizeSemanticToken(rawValue) {
|
|
146
|
+
return String(rawValue || '')
|
|
147
|
+
.trim()
|
|
148
|
+
.toLowerCase()
|
|
149
|
+
.normalize('NFKC')
|
|
150
|
+
.replace(/[^\p{L}\p{N}]+/gu, '');
|
|
151
|
+
}
|
|
152
|
+
function stemSemanticToken(rawValue) {
|
|
153
|
+
const token = normalizeSemanticToken(rawValue);
|
|
154
|
+
if (!token) {
|
|
155
|
+
return '';
|
|
156
|
+
}
|
|
157
|
+
if (!/^[a-z0-9]+$/.test(token)) {
|
|
158
|
+
return token;
|
|
159
|
+
}
|
|
160
|
+
if (token.endsWith('ies') && token.length > 4) {
|
|
161
|
+
return `${token.slice(0, -3)}y`;
|
|
162
|
+
}
|
|
163
|
+
if (token.endsWith('ing') && token.length > 5) {
|
|
164
|
+
return token.slice(0, -3);
|
|
165
|
+
}
|
|
166
|
+
if (token.endsWith('ed') && token.length > 4) {
|
|
167
|
+
return token.slice(0, -2);
|
|
168
|
+
}
|
|
169
|
+
if (token.endsWith('es') && token.length > 4 && !token.endsWith('ses')) {
|
|
170
|
+
return token.slice(0, -2);
|
|
171
|
+
}
|
|
172
|
+
if (token.endsWith('s')
|
|
173
|
+
&& token.length > 3
|
|
174
|
+
&& !token.endsWith('ss')
|
|
175
|
+
&& !token.endsWith('is')
|
|
176
|
+
&& !token.endsWith('us')) {
|
|
177
|
+
return token.slice(0, -1);
|
|
178
|
+
}
|
|
179
|
+
return token;
|
|
180
|
+
}
|
|
181
|
+
function buildCjkSemanticNgrams(rawText, limit) {
|
|
182
|
+
if (limit <= 0) {
|
|
183
|
+
return [];
|
|
184
|
+
}
|
|
185
|
+
const segments = String(rawText || '')
|
|
186
|
+
.normalize('NFKC')
|
|
187
|
+
.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]+/gu) || [];
|
|
188
|
+
const tokens = [];
|
|
189
|
+
const seen = new Set();
|
|
190
|
+
for (const segment of segments) {
|
|
191
|
+
const chars = Array.from(segment);
|
|
192
|
+
if (chars.length === 1) {
|
|
193
|
+
const token = chars[0];
|
|
194
|
+
if (!seen.has(token)) {
|
|
195
|
+
seen.add(token);
|
|
196
|
+
tokens.push(token);
|
|
197
|
+
}
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
for (const gramSize of [2, 3]) {
|
|
201
|
+
if (chars.length < gramSize) {
|
|
202
|
+
continue;
|
|
203
|
+
}
|
|
204
|
+
for (let index = 0; index <= chars.length - gramSize; index += 1) {
|
|
205
|
+
const token = chars.slice(index, index + gramSize).join('');
|
|
206
|
+
if (!token || seen.has(token)) {
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
seen.add(token);
|
|
210
|
+
tokens.push(token);
|
|
211
|
+
if (tokens.length >= limit) {
|
|
212
|
+
return tokens;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
return tokens;
|
|
218
|
+
}
|
|
219
|
+
function extractSemanticTokens(rawText, limit) {
|
|
220
|
+
if (limit <= 0) {
|
|
221
|
+
return [];
|
|
222
|
+
}
|
|
223
|
+
const tokens = String(rawText || '')
|
|
224
|
+
.normalize('NFKC')
|
|
225
|
+
.toLowerCase()
|
|
226
|
+
.match(/[\p{L}\p{N}]+(?:[_-][\p{L}\p{N}]+)*/gu) || [];
|
|
227
|
+
const collected = [];
|
|
228
|
+
const seen = new Set();
|
|
229
|
+
for (const token of tokens) {
|
|
230
|
+
const normalized = stemSemanticToken(token);
|
|
231
|
+
if (!normalized || seen.has(normalized)) {
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
seen.add(normalized);
|
|
235
|
+
collected.push(normalized);
|
|
236
|
+
if (collected.length >= limit) {
|
|
237
|
+
break;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
if (collected.length >= limit) {
|
|
241
|
+
return collected;
|
|
242
|
+
}
|
|
243
|
+
const cjkTokens = buildCjkSemanticNgrams(rawText, Math.min(limit - collected.length, SEMANTIC_CJK_NGRAM_LIMIT));
|
|
244
|
+
for (const token of cjkTokens) {
|
|
245
|
+
if (!token || seen.has(token)) {
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
seen.add(token);
|
|
249
|
+
collected.push(token);
|
|
250
|
+
if (collected.length >= limit) {
|
|
251
|
+
break;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return collected;
|
|
255
|
+
}
|
|
256
|
+
function uniqueTokens(tokens) {
|
|
257
|
+
const unique = [];
|
|
258
|
+
const seen = new Set();
|
|
259
|
+
tokens.forEach((token) => {
|
|
260
|
+
if (!token || seen.has(token)) {
|
|
261
|
+
return;
|
|
262
|
+
}
|
|
263
|
+
seen.add(token);
|
|
264
|
+
unique.push(token);
|
|
265
|
+
});
|
|
266
|
+
return unique;
|
|
267
|
+
}
|
|
268
|
+
function uniqueNormalizedStrings(values) {
|
|
269
|
+
const unique = [];
|
|
270
|
+
const seen = new Set();
|
|
271
|
+
values.forEach((value) => {
|
|
272
|
+
const normalized = String(value || '')
|
|
273
|
+
.normalize('NFKC')
|
|
274
|
+
.toLowerCase()
|
|
275
|
+
.replace(/\s+/g, ' ')
|
|
276
|
+
.trim();
|
|
277
|
+
if (!normalized || seen.has(normalized)) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
seen.add(normalized);
|
|
281
|
+
unique.push(normalized);
|
|
282
|
+
});
|
|
283
|
+
return unique;
|
|
284
|
+
}
|
|
285
|
+
function buildQueryTextVariants(context) {
|
|
286
|
+
return uniqueNormalizedStrings([
|
|
287
|
+
context.query,
|
|
288
|
+
...(Array.isArray(context.queryVariants) ? context.queryVariants : []),
|
|
289
|
+
]);
|
|
290
|
+
}
|
|
291
|
+
function buildAtomSemanticTokens(atom) {
|
|
292
|
+
const keywordTokens = uniqueTokens((atom.keywords || [])
|
|
293
|
+
.map((keyword) => stemSemanticToken(keyword))
|
|
294
|
+
.filter((token) => token.length > 0)
|
|
295
|
+
.slice(0, SEMANTIC_KEYWORD_TOKEN_LIMIT));
|
|
296
|
+
const titleTokens = extractSemanticTokens(atom.title, SEMANTIC_TITLE_TOKEN_LIMIT);
|
|
297
|
+
const contentTokens = extractSemanticTokens(atom.content, SEMANTIC_CONTENT_TOKEN_LIMIT);
|
|
298
|
+
return uniqueTokens([...keywordTokens, ...titleTokens, ...contentTokens]);
|
|
299
|
+
}
|
|
300
|
+
function buildQuerySemanticTokens(context) {
|
|
301
|
+
const directTokens = uniqueTokens((context.queryTokens || [])
|
|
302
|
+
.map((token) => stemSemanticToken(token))
|
|
303
|
+
.filter((token) => token.length > 0));
|
|
304
|
+
const variantTokens = buildQueryTextVariants(context).flatMap((variant) => (extractSemanticTokens(variant, SEMANTIC_QUERY_TOKEN_LIMIT)));
|
|
305
|
+
const fallbackTokens = extractSemanticTokens(context.query, SEMANTIC_QUERY_TOKEN_LIMIT);
|
|
306
|
+
return uniqueTokens([...directTokens, ...variantTokens, ...fallbackTokens]).slice(0, SEMANTIC_QUERY_TOKEN_LIMIT);
|
|
307
|
+
}
|
|
308
|
+
function computeJaccard(left, right) {
|
|
309
|
+
if (!left.length || !right.length) {
|
|
310
|
+
return 0;
|
|
311
|
+
}
|
|
312
|
+
const leftSet = new Set(left);
|
|
313
|
+
const rightSet = new Set(right);
|
|
314
|
+
let intersection = 0;
|
|
315
|
+
leftSet.forEach((token) => {
|
|
316
|
+
if (rightSet.has(token)) {
|
|
317
|
+
intersection += 1;
|
|
318
|
+
}
|
|
319
|
+
});
|
|
320
|
+
const union = leftSet.size + rightSet.size - intersection;
|
|
321
|
+
if (union <= 0) {
|
|
322
|
+
return 0;
|
|
323
|
+
}
|
|
324
|
+
return intersection / union;
|
|
325
|
+
}
|
|
326
|
+
function classifyGraphQueryIntent(query) {
|
|
327
|
+
const normalized = String(query || '').trim().toLowerCase();
|
|
328
|
+
if (!normalized) {
|
|
329
|
+
return 'generic';
|
|
330
|
+
}
|
|
331
|
+
if (normalized.includes('compare')
|
|
332
|
+
|| normalized.includes('difference')
|
|
333
|
+
|| normalized.includes('vs')
|
|
334
|
+
|| normalized.includes('对比')
|
|
335
|
+
|| normalized.includes('比较')
|
|
336
|
+
|| normalized.includes('区别')
|
|
337
|
+
|| normalized.includes('差异')) {
|
|
338
|
+
return 'compare';
|
|
339
|
+
}
|
|
340
|
+
if (normalized.includes('how to')
|
|
341
|
+
|| normalized.includes('how do')
|
|
342
|
+
|| normalized.includes('steps')
|
|
343
|
+
|| normalized.includes('plan')
|
|
344
|
+
|| normalized.includes('如何')
|
|
345
|
+
|| normalized.includes('怎么')
|
|
346
|
+
|| normalized.includes('步骤')
|
|
347
|
+
|| normalized.includes('方案')) {
|
|
348
|
+
return 'how_to';
|
|
349
|
+
}
|
|
350
|
+
if (normalized.includes('what is')
|
|
351
|
+
|| normalized.includes('why')
|
|
352
|
+
|| normalized.includes('explain')
|
|
353
|
+
|| normalized.includes('什么是')
|
|
354
|
+
|| normalized.includes('为什么')
|
|
355
|
+
|| normalized.includes('解释')) {
|
|
356
|
+
return 'explain';
|
|
357
|
+
}
|
|
358
|
+
return 'generic';
|
|
359
|
+
}
|
|
360
|
+
function buildAtomAnchorTokens(atom) {
|
|
361
|
+
return uniqueTokens([
|
|
362
|
+
...extractSemanticTokens(atom.title, Math.max(SEMANTIC_TITLE_TOKEN_LIMIT, 10)),
|
|
363
|
+
...extractSemanticTokens(atom.sourcePath, 10),
|
|
364
|
+
...extractSemanticTokens(atom.documentId, 6),
|
|
365
|
+
]);
|
|
366
|
+
}
|
|
367
|
+
function inferGraphAnchorAtomIds(context, querySemanticTokens) {
|
|
368
|
+
const queryVariants = buildQueryTextVariants(context);
|
|
369
|
+
const rankedAnchors = context.atoms
|
|
370
|
+
.map((atom, index) => {
|
|
371
|
+
const titleLower = String(atom.title || '').trim().toLowerCase();
|
|
372
|
+
const directMatchScore = titleLower && queryVariants.some((variant) => variant.includes(titleLower))
|
|
373
|
+
? 1
|
|
374
|
+
: 0;
|
|
375
|
+
const anchorTokenOverlap = computeJaccard(querySemanticTokens, buildAtomAnchorTokens(atom));
|
|
376
|
+
const keywordOverlap = computeJaccard(querySemanticTokens, uniqueTokens((atom.keywords || []).map((keyword) => stemSemanticToken(keyword)).filter(Boolean)));
|
|
377
|
+
const score = Number((directMatchScore
|
|
378
|
+
+ anchorTokenOverlap
|
|
379
|
+
+ (keywordOverlap * 0.35)).toFixed(6));
|
|
380
|
+
return {
|
|
381
|
+
atomId: atom.id,
|
|
382
|
+
score,
|
|
383
|
+
index,
|
|
384
|
+
};
|
|
385
|
+
})
|
|
386
|
+
.filter((entry) => entry.score > 0)
|
|
387
|
+
.sort((left, right) => (right.score - left.score
|
|
388
|
+
|| left.index - right.index));
|
|
389
|
+
if (rankedAnchors.length <= 0) {
|
|
390
|
+
return new Set();
|
|
391
|
+
}
|
|
392
|
+
const topScore = rankedAnchors[0].score;
|
|
393
|
+
return new Set(rankedAnchors
|
|
394
|
+
.filter((entry, index) => index < 3 && entry.score >= Math.max(0.2, topScore - 0.08))
|
|
395
|
+
.map((entry) => entry.atomId));
|
|
396
|
+
}
|
|
397
|
+
function buildGraphAdjacency(activeEdges) {
|
|
398
|
+
const outgoingByAtomId = new Map();
|
|
399
|
+
const incomingByAtomId = new Map();
|
|
400
|
+
const undirectedNeighborsByAtomId = new Map();
|
|
401
|
+
const degreeByAtomId = new Map();
|
|
402
|
+
const incidentRelationKindsByAtomId = new Map();
|
|
403
|
+
activeEdges.forEach((edge) => {
|
|
404
|
+
const confidence = clamp(Number(edge.confidence || 0), 0, 1);
|
|
405
|
+
const outgoing = outgoingByAtomId.get(edge.sourceAtomId) || [];
|
|
406
|
+
outgoing.push({
|
|
407
|
+
to: edge.targetAtomId,
|
|
408
|
+
relationKind: edge.relationKind,
|
|
409
|
+
confidence,
|
|
410
|
+
});
|
|
411
|
+
outgoingByAtomId.set(edge.sourceAtomId, outgoing);
|
|
412
|
+
const incoming = incomingByAtomId.get(edge.targetAtomId) || [];
|
|
413
|
+
incoming.push({
|
|
414
|
+
to: edge.sourceAtomId,
|
|
415
|
+
relationKind: edge.relationKind,
|
|
416
|
+
confidence,
|
|
417
|
+
});
|
|
418
|
+
incomingByAtomId.set(edge.targetAtomId, incoming);
|
|
419
|
+
const sourceUndirected = undirectedNeighborsByAtomId.get(edge.sourceAtomId) || [];
|
|
420
|
+
sourceUndirected.push(edge.targetAtomId);
|
|
421
|
+
undirectedNeighborsByAtomId.set(edge.sourceAtomId, sourceUndirected);
|
|
422
|
+
const targetUndirected = undirectedNeighborsByAtomId.get(edge.targetAtomId) || [];
|
|
423
|
+
targetUndirected.push(edge.sourceAtomId);
|
|
424
|
+
undirectedNeighborsByAtomId.set(edge.targetAtomId, targetUndirected);
|
|
425
|
+
degreeByAtomId.set(edge.sourceAtomId, (degreeByAtomId.get(edge.sourceAtomId) || 0) + 1);
|
|
426
|
+
degreeByAtomId.set(edge.targetAtomId, (degreeByAtomId.get(edge.targetAtomId) || 0) + 1);
|
|
427
|
+
const sourceKinds = incidentRelationKindsByAtomId.get(edge.sourceAtomId) || new Set();
|
|
428
|
+
sourceKinds.add(edge.relationKind);
|
|
429
|
+
incidentRelationKindsByAtomId.set(edge.sourceAtomId, sourceKinds);
|
|
430
|
+
const targetKinds = incidentRelationKindsByAtomId.get(edge.targetAtomId) || new Set();
|
|
431
|
+
targetKinds.add(edge.relationKind);
|
|
432
|
+
incidentRelationKindsByAtomId.set(edge.targetAtomId, targetKinds);
|
|
433
|
+
});
|
|
434
|
+
return {
|
|
435
|
+
outgoingByAtomId,
|
|
436
|
+
incomingByAtomId,
|
|
437
|
+
undirectedNeighborsByAtomId,
|
|
438
|
+
degreeByAtomId,
|
|
439
|
+
incidentRelationKindsByAtomId,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
function buildUndirectedDistanceSignals(anchorAtomIds, undirectedNeighborsByAtomId, maxDepth) {
|
|
443
|
+
const distances = new Map();
|
|
444
|
+
const queue = [];
|
|
445
|
+
anchorAtomIds.forEach((anchorAtomId) => {
|
|
446
|
+
distances.set(anchorAtomId, 0);
|
|
447
|
+
queue.push({ atomId: anchorAtomId, distance: 0 });
|
|
448
|
+
});
|
|
449
|
+
while (queue.length > 0) {
|
|
450
|
+
const current = queue.shift();
|
|
451
|
+
if (!current) {
|
|
452
|
+
continue;
|
|
453
|
+
}
|
|
454
|
+
if (current.distance >= maxDepth) {
|
|
455
|
+
continue;
|
|
456
|
+
}
|
|
457
|
+
const neighbors = undirectedNeighborsByAtomId.get(current.atomId) || [];
|
|
458
|
+
neighbors.forEach((neighborAtomId) => {
|
|
459
|
+
const nextDistance = current.distance + 1;
|
|
460
|
+
const existingDistance = distances.get(neighborAtomId);
|
|
461
|
+
if (typeof existingDistance === 'number' && existingDistance <= nextDistance) {
|
|
462
|
+
return;
|
|
463
|
+
}
|
|
464
|
+
distances.set(neighborAtomId, nextDistance);
|
|
465
|
+
queue.push({
|
|
466
|
+
atomId: neighborAtomId,
|
|
467
|
+
distance: nextDistance,
|
|
468
|
+
});
|
|
469
|
+
});
|
|
470
|
+
}
|
|
471
|
+
return distances;
|
|
472
|
+
}
|
|
473
|
+
function shouldReplaceDirectedPathSignal(current, next) {
|
|
474
|
+
if (!current) {
|
|
475
|
+
return true;
|
|
476
|
+
}
|
|
477
|
+
if (next.distance < current.distance) {
|
|
478
|
+
return true;
|
|
479
|
+
}
|
|
480
|
+
return next.distance === current.distance && next.averageConfidence > current.averageConfidence;
|
|
481
|
+
}
|
|
482
|
+
function buildDirectedPathSignals(anchorAtomIds, adjacencyByAtomId, maxDepth) {
|
|
483
|
+
const signals = new Map();
|
|
484
|
+
const queue = [];
|
|
485
|
+
anchorAtomIds.forEach((anchorAtomId) => {
|
|
486
|
+
signals.set(anchorAtomId, {
|
|
487
|
+
distance: 0,
|
|
488
|
+
averageConfidence: 1,
|
|
489
|
+
relationKinds: [],
|
|
490
|
+
});
|
|
491
|
+
queue.push({
|
|
492
|
+
atomId: anchorAtomId,
|
|
493
|
+
distance: 0,
|
|
494
|
+
confidenceSum: 0,
|
|
495
|
+
relationKinds: [],
|
|
496
|
+
});
|
|
497
|
+
});
|
|
498
|
+
while (queue.length > 0) {
|
|
499
|
+
const current = queue.shift();
|
|
500
|
+
if (!current) {
|
|
501
|
+
continue;
|
|
502
|
+
}
|
|
503
|
+
if (current.distance >= maxDepth) {
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
const neighbors = adjacencyByAtomId.get(current.atomId) || [];
|
|
507
|
+
neighbors.forEach((neighbor) => {
|
|
508
|
+
const nextDistance = current.distance + 1;
|
|
509
|
+
const nextConfidenceSum = current.confidenceSum + clamp(Number(neighbor.confidence || 0), 0, 1);
|
|
510
|
+
const nextSignal = {
|
|
511
|
+
distance: nextDistance,
|
|
512
|
+
averageConfidence: Number((nextConfidenceSum / nextDistance).toFixed(4)),
|
|
513
|
+
relationKinds: current.relationKinds.includes(neighbor.relationKind)
|
|
514
|
+
? current.relationKinds
|
|
515
|
+
: [...current.relationKinds, neighbor.relationKind],
|
|
516
|
+
};
|
|
517
|
+
const currentSignal = signals.get(neighbor.to);
|
|
518
|
+
if (!shouldReplaceDirectedPathSignal(currentSignal, nextSignal)) {
|
|
519
|
+
return;
|
|
520
|
+
}
|
|
521
|
+
signals.set(neighbor.to, nextSignal);
|
|
522
|
+
queue.push({
|
|
523
|
+
atomId: neighbor.to,
|
|
524
|
+
distance: nextDistance,
|
|
525
|
+
confidenceSum: nextConfidenceSum,
|
|
526
|
+
relationKinds: nextSignal.relationKinds,
|
|
527
|
+
});
|
|
528
|
+
});
|
|
529
|
+
}
|
|
530
|
+
return signals;
|
|
531
|
+
}
|
|
532
|
+
function buildGraphFeatureIndex(context, querySemanticTokens) {
|
|
533
|
+
const queryIntent = classifyGraphQueryIntent(context.query);
|
|
534
|
+
const anchorAtomIds = inferGraphAnchorAtomIds(context, querySemanticTokens);
|
|
535
|
+
const { outgoingByAtomId, incomingByAtomId, undirectedNeighborsByAtomId, degreeByAtomId, incidentRelationKindsByAtomId, } = buildGraphAdjacency(context.activeEdges);
|
|
536
|
+
const maxDegree = Array.from(degreeByAtomId.values()).reduce((maxValue, degree) => Math.max(maxValue, degree), 0);
|
|
537
|
+
return {
|
|
538
|
+
queryIntent,
|
|
539
|
+
anchorAtomIds,
|
|
540
|
+
degreeByAtomId,
|
|
541
|
+
maxDegree,
|
|
542
|
+
incidentRelationKindsByAtomId,
|
|
543
|
+
undirectedDistanceByAtomId: buildUndirectedDistanceSignals(anchorAtomIds, undirectedNeighborsByAtomId, 4),
|
|
544
|
+
pathToAnchorByAtomId: buildDirectedPathSignals(anchorAtomIds, incomingByAtomId, 4),
|
|
545
|
+
pathFromAnchorByAtomId: buildDirectedPathSignals(anchorAtomIds, outgoingByAtomId, 4),
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
function computeAnchorDistanceBonus(atomId, graphFeatureIndex) {
|
|
549
|
+
const distance = graphFeatureIndex.undirectedDistanceByAtomId.get(atomId);
|
|
550
|
+
if (typeof distance !== 'number') {
|
|
551
|
+
return 0;
|
|
552
|
+
}
|
|
553
|
+
if (distance <= 0) {
|
|
554
|
+
return graphFeatureIndex.queryIntent === 'compare' ? 0.22 : 0.3;
|
|
555
|
+
}
|
|
556
|
+
if (distance === 1) {
|
|
557
|
+
return 0.18;
|
|
558
|
+
}
|
|
559
|
+
if (distance === 2) {
|
|
560
|
+
return 0.1;
|
|
561
|
+
}
|
|
562
|
+
if (distance === 3) {
|
|
563
|
+
return 0.05;
|
|
564
|
+
}
|
|
565
|
+
return 0;
|
|
566
|
+
}
|
|
567
|
+
function computeDirectedPathBonus(signal, intent, direction) {
|
|
568
|
+
if (!signal || signal.distance <= 0) {
|
|
569
|
+
return 0;
|
|
570
|
+
}
|
|
571
|
+
const baseBonus = signal.distance === 1
|
|
572
|
+
? 0.14
|
|
573
|
+
: signal.distance === 2
|
|
574
|
+
? 0.09
|
|
575
|
+
: 0.05;
|
|
576
|
+
const confidenceBonus = clamp(signal.averageConfidence, 0, 1) * 0.12;
|
|
577
|
+
let relationBonus = 0;
|
|
578
|
+
if (signal.relationKinds.includes('prerequisite') && direction === 'to_anchor' && intent !== 'compare') {
|
|
579
|
+
relationBonus += intent === 'how_to' ? 0.12 : 0.07;
|
|
580
|
+
}
|
|
581
|
+
if (signal.relationKinds.includes('sequence') && direction === 'from_anchor' && intent === 'how_to') {
|
|
582
|
+
relationBonus += 0.08;
|
|
583
|
+
}
|
|
584
|
+
if (signal.relationKinds.includes('application') && direction === 'from_anchor' && intent === 'how_to') {
|
|
585
|
+
relationBonus += 0.06;
|
|
586
|
+
}
|
|
587
|
+
const compareBranchSignalPresent = signal.relationKinds.includes('contrast') || signal.relationKinds.includes('analogy');
|
|
588
|
+
if (compareBranchSignalPresent && intent === 'compare') {
|
|
589
|
+
relationBonus += signal.distance === 1 ? 0.32 : 0.18;
|
|
590
|
+
}
|
|
591
|
+
const maxBonus = compareBranchSignalPresent && intent === 'compare' ? 0.6 : 0.4;
|
|
592
|
+
return Number(clamp(baseBonus + confidenceBonus + relationBonus, 0, maxBonus).toFixed(6));
|
|
593
|
+
}
|
|
594
|
+
function computeRelationIntentBonus(atomId, graphFeatureIndex) {
|
|
595
|
+
const relationKinds = graphFeatureIndex.incidentRelationKindsByAtomId.get(atomId);
|
|
596
|
+
if (!relationKinds || relationKinds.size <= 0) {
|
|
597
|
+
return 0;
|
|
598
|
+
}
|
|
599
|
+
let bonus = 0;
|
|
600
|
+
if (graphFeatureIndex.queryIntent === 'compare') {
|
|
601
|
+
if (relationKinds.has('contrast'))
|
|
602
|
+
bonus += 0.24;
|
|
603
|
+
if (relationKinds.has('analogy'))
|
|
604
|
+
bonus += 0.16;
|
|
605
|
+
if (relationKinds.has('reference'))
|
|
606
|
+
bonus += 0.02;
|
|
607
|
+
}
|
|
608
|
+
else if (graphFeatureIndex.queryIntent === 'how_to') {
|
|
609
|
+
if (relationKinds.has('prerequisite'))
|
|
610
|
+
bonus += 0.1;
|
|
611
|
+
if (relationKinds.has('sequence'))
|
|
612
|
+
bonus += 0.08;
|
|
613
|
+
if (relationKinds.has('application'))
|
|
614
|
+
bonus += 0.08;
|
|
615
|
+
if (relationKinds.has('causal'))
|
|
616
|
+
bonus += 0.04;
|
|
617
|
+
}
|
|
618
|
+
else if (graphFeatureIndex.queryIntent === 'explain') {
|
|
619
|
+
if (relationKinds.has('causal'))
|
|
620
|
+
bonus += 0.1;
|
|
621
|
+
if (relationKinds.has('reference'))
|
|
622
|
+
bonus += 0.06;
|
|
623
|
+
if (relationKinds.has('prerequisite'))
|
|
624
|
+
bonus += 0.05;
|
|
625
|
+
}
|
|
626
|
+
else {
|
|
627
|
+
if (relationKinds.has('reference'))
|
|
628
|
+
bonus += 0.04;
|
|
629
|
+
if (relationKinds.has('causal'))
|
|
630
|
+
bonus += 0.04;
|
|
631
|
+
}
|
|
632
|
+
return Number(clamp(bonus, 0, 0.18).toFixed(6));
|
|
633
|
+
}
|
|
634
|
+
function computeTemporalValidityBonus(context, atomId) {
|
|
635
|
+
const temporalSignal = context.atomTemporalValidity && typeof context.atomTemporalValidity === 'object'
|
|
636
|
+
? context.atomTemporalValidity[atomId]
|
|
637
|
+
: undefined;
|
|
638
|
+
if (!temporalSignal) {
|
|
639
|
+
return 0;
|
|
640
|
+
}
|
|
641
|
+
if (temporalSignal.isValid === false) {
|
|
642
|
+
return -Number((0.18 + Math.min(temporalSignal.reasonCount, 3) * 0.02).toFixed(6));
|
|
643
|
+
}
|
|
644
|
+
return 0;
|
|
645
|
+
}
|
|
646
|
+
function computeCompareBranchBonus(atomId, graphFeatureIndex) {
|
|
647
|
+
if (graphFeatureIndex.queryIntent !== 'compare') {
|
|
648
|
+
return 0;
|
|
649
|
+
}
|
|
650
|
+
const toAnchorSignal = graphFeatureIndex.pathToAnchorByAtomId.get(atomId);
|
|
651
|
+
const fromAnchorSignal = graphFeatureIndex.pathFromAnchorByAtomId.get(atomId);
|
|
652
|
+
const directContrastToAnchor = Boolean(toAnchorSignal
|
|
653
|
+
&& toAnchorSignal.distance === 1
|
|
654
|
+
&& (toAnchorSignal.relationKinds.includes('contrast') || toAnchorSignal.relationKinds.includes('analogy')));
|
|
655
|
+
const directContrastFromAnchor = Boolean(fromAnchorSignal
|
|
656
|
+
&& fromAnchorSignal.distance === 1
|
|
657
|
+
&& (fromAnchorSignal.relationKinds.includes('contrast') || fromAnchorSignal.relationKinds.includes('analogy')));
|
|
658
|
+
if (directContrastToAnchor || directContrastFromAnchor) {
|
|
659
|
+
return 0.24;
|
|
660
|
+
}
|
|
661
|
+
return 0;
|
|
662
|
+
}
|
|
663
|
+
function computeHubPenalty(atomId, graphFeatureIndex) {
|
|
664
|
+
if (graphFeatureIndex.maxDegree <= 0) {
|
|
665
|
+
return 0;
|
|
666
|
+
}
|
|
667
|
+
if (graphFeatureIndex.undirectedDistanceByAtomId.has(atomId)) {
|
|
668
|
+
return 0;
|
|
669
|
+
}
|
|
670
|
+
const degree = graphFeatureIndex.degreeByAtomId.get(atomId) || 0;
|
|
671
|
+
const normalizedDegree = degree / graphFeatureIndex.maxDegree;
|
|
672
|
+
if (normalizedDegree <= 0.7) {
|
|
673
|
+
return 0;
|
|
674
|
+
}
|
|
675
|
+
return Number(clamp((normalizedDegree - 0.7) * 0.25, 0, 0.12).toFixed(6));
|
|
676
|
+
}
|
|
677
|
+
function computeGraphFeatureScore(context, atomId, graphFeatureIndex) {
|
|
678
|
+
if (graphFeatureIndex.anchorAtomIds.size <= 0) {
|
|
679
|
+
return 0;
|
|
680
|
+
}
|
|
681
|
+
const anchorDistanceBonus = computeAnchorDistanceBonus(atomId, graphFeatureIndex);
|
|
682
|
+
const pathToAnchorBonus = computeDirectedPathBonus(graphFeatureIndex.pathToAnchorByAtomId.get(atomId), graphFeatureIndex.queryIntent, 'to_anchor');
|
|
683
|
+
const pathFromAnchorBonus = computeDirectedPathBonus(graphFeatureIndex.pathFromAnchorByAtomId.get(atomId), graphFeatureIndex.queryIntent, 'from_anchor');
|
|
684
|
+
const relationIntentBonus = computeRelationIntentBonus(atomId, graphFeatureIndex);
|
|
685
|
+
const temporalValidityBonus = computeTemporalValidityBonus(context, atomId);
|
|
686
|
+
const compareBranchBonus = computeCompareBranchBonus(atomId, graphFeatureIndex);
|
|
687
|
+
const hubPenalty = computeHubPenalty(atomId, graphFeatureIndex);
|
|
688
|
+
return Number((anchorDistanceBonus
|
|
689
|
+
+ pathToAnchorBonus
|
|
690
|
+
+ pathFromAnchorBonus
|
|
691
|
+
+ relationIntentBonus
|
|
692
|
+
+ temporalValidityBonus
|
|
693
|
+
+ compareBranchBonus
|
|
694
|
+
- hubPenalty).toFixed(6));
|
|
695
|
+
}
|
|
696
|
+
function buildTokenFrequency(tokens) {
|
|
697
|
+
const frequency = new Map();
|
|
698
|
+
tokens.forEach((token) => {
|
|
699
|
+
if (!token) {
|
|
700
|
+
return;
|
|
701
|
+
}
|
|
702
|
+
frequency.set(token, (frequency.get(token) || 0) + 1);
|
|
703
|
+
});
|
|
704
|
+
return frequency;
|
|
705
|
+
}
|
|
706
|
+
function buildTfIdfWeights(frequency, docFrequency, totalDocs) {
|
|
707
|
+
const weights = new Map();
|
|
708
|
+
const maxTermCount = Math.max(1, ...Array.from(frequency.values()).map((value) => Math.max(0, Math.floor(Number(value) || 0))));
|
|
709
|
+
frequency.forEach((count, token) => {
|
|
710
|
+
const tf = Math.max(0, Math.floor(Number(count) || 0)) / maxTermCount;
|
|
711
|
+
const df = Math.max(1, Math.floor(Number(docFrequency.get(token) || 0)));
|
|
712
|
+
const idf = Math.log((1 + totalDocs) / (1 + df)) + 1;
|
|
713
|
+
weights.set(token, Number((tf * idf).toFixed(8)));
|
|
714
|
+
});
|
|
715
|
+
return weights;
|
|
716
|
+
}
|
|
717
|
+
function computeCosineSimilarity(left, right) {
|
|
718
|
+
if (left.size <= 0 || right.size <= 0) {
|
|
719
|
+
return 0;
|
|
720
|
+
}
|
|
721
|
+
let dot = 0;
|
|
722
|
+
let leftNorm = 0;
|
|
723
|
+
let rightNorm = 0;
|
|
724
|
+
left.forEach((weight, token) => {
|
|
725
|
+
const rightWeight = right.get(token) || 0;
|
|
726
|
+
dot += weight * rightWeight;
|
|
727
|
+
leftNorm += weight * weight;
|
|
728
|
+
});
|
|
729
|
+
right.forEach((weight) => {
|
|
730
|
+
rightNorm += weight * weight;
|
|
731
|
+
});
|
|
732
|
+
const denominator = Math.sqrt(leftNorm) * Math.sqrt(rightNorm);
|
|
733
|
+
if (!Number.isFinite(denominator) || denominator <= 0) {
|
|
734
|
+
return 0;
|
|
735
|
+
}
|
|
736
|
+
return dot / denominator;
|
|
737
|
+
}
|
|
738
|
+
function normalizeVectorIndexPath(rawValue) {
|
|
739
|
+
const trimmed = String(rawValue || '').trim();
|
|
740
|
+
if (!trimmed) {
|
|
741
|
+
return undefined;
|
|
742
|
+
}
|
|
743
|
+
return path.resolve(trimmed);
|
|
744
|
+
}
|
|
745
|
+
function toSerializableWeightEntries(weights) {
|
|
746
|
+
const entries = [];
|
|
747
|
+
weights.forEach((weight, token) => {
|
|
748
|
+
if (!token) {
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
const numericWeight = Number(weight);
|
|
752
|
+
if (!Number.isFinite(numericWeight) || numericWeight <= 0) {
|
|
753
|
+
return;
|
|
754
|
+
}
|
|
755
|
+
entries.push([token, Number(numericWeight.toFixed(8))]);
|
|
756
|
+
});
|
|
757
|
+
entries.sort((left, right) => left[0].localeCompare(right[0]));
|
|
758
|
+
return entries;
|
|
759
|
+
}
|
|
760
|
+
function toWeightMap(entries) {
|
|
761
|
+
const weightMap = new Map();
|
|
762
|
+
if (!Array.isArray(entries)) {
|
|
763
|
+
return weightMap;
|
|
764
|
+
}
|
|
765
|
+
entries.forEach((entry) => {
|
|
766
|
+
if (!Array.isArray(entry) || entry.length < 2) {
|
|
767
|
+
return;
|
|
768
|
+
}
|
|
769
|
+
const token = String(entry[0] || '').trim().toLowerCase();
|
|
770
|
+
const weight = Number(entry[1]);
|
|
771
|
+
if (!token || !Number.isFinite(weight) || weight <= 0) {
|
|
772
|
+
return;
|
|
773
|
+
}
|
|
774
|
+
weightMap.set(token, Number(weight.toFixed(8)));
|
|
775
|
+
});
|
|
776
|
+
return weightMap;
|
|
777
|
+
}
|
|
778
|
+
function computeLocalVectorSignature(atoms) {
|
|
779
|
+
const hash = (0, crypto_1.createHash)('sha256');
|
|
780
|
+
hash.update(LOCAL_VECTOR_INDEX_SIGNATURE_VERSION);
|
|
781
|
+
const sortedAtoms = [...atoms].sort((left, right) => left.id.localeCompare(right.id));
|
|
782
|
+
sortedAtoms.forEach((atom) => {
|
|
783
|
+
const sourceHash = String(atom.metadata?.sourceHash || '').trim();
|
|
784
|
+
const version = Number(atom.metadata?.version || 0);
|
|
785
|
+
hash.update(atom.id);
|
|
786
|
+
hash.update('\u0000');
|
|
787
|
+
hash.update(String(atom.updatedAt || ''));
|
|
788
|
+
hash.update('\u0000');
|
|
789
|
+
hash.update(sourceHash);
|
|
790
|
+
hash.update('\u0000');
|
|
791
|
+
hash.update(String(Number.isFinite(version) ? version : 0));
|
|
792
|
+
hash.update('\u0000');
|
|
793
|
+
if (!sourceHash) {
|
|
794
|
+
hash.update(String(atom.title || ''));
|
|
795
|
+
hash.update('\u0000');
|
|
796
|
+
hash.update(String(atom.content || ''));
|
|
797
|
+
hash.update('\u0000');
|
|
798
|
+
hash.update((atom.keywords || []).join('|'));
|
|
799
|
+
}
|
|
800
|
+
hash.update('\u0001');
|
|
801
|
+
});
|
|
802
|
+
return hash.digest('hex');
|
|
803
|
+
}
|
|
804
|
+
function getLocalVectorTokenHash64(token) {
|
|
805
|
+
const normalizedToken = String(token || '').trim().toLowerCase();
|
|
806
|
+
if (!normalizedToken) {
|
|
807
|
+
return 0n;
|
|
808
|
+
}
|
|
809
|
+
const cached = LOCAL_VECTOR_TOKEN_HASH_CACHE.get(normalizedToken);
|
|
810
|
+
if (typeof cached !== 'undefined') {
|
|
811
|
+
return cached;
|
|
812
|
+
}
|
|
813
|
+
const tokenHashHex = (0, crypto_1.createHash)('sha1')
|
|
814
|
+
.update(`local_vector_ann:${normalizedToken}`)
|
|
815
|
+
.digest('hex')
|
|
816
|
+
.slice(0, 16);
|
|
817
|
+
const tokenHash = BigInt(`0x${tokenHashHex}`);
|
|
818
|
+
if (LOCAL_VECTOR_TOKEN_HASH_CACHE.size >= LOCAL_VECTOR_ANN_HASH_CACHE_SOFT_LIMIT) {
|
|
819
|
+
const firstKey = LOCAL_VECTOR_TOKEN_HASH_CACHE.keys().next().value;
|
|
820
|
+
if (typeof firstKey === 'string') {
|
|
821
|
+
LOCAL_VECTOR_TOKEN_HASH_CACHE.delete(firstKey);
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
LOCAL_VECTOR_TOKEN_HASH_CACHE.set(normalizedToken, tokenHash);
|
|
825
|
+
return tokenHash;
|
|
826
|
+
}
|
|
827
|
+
function computeWeightedSimHash(weights) {
|
|
828
|
+
if (weights.size <= 0) {
|
|
829
|
+
return 0n;
|
|
830
|
+
}
|
|
831
|
+
const accumulator = new Array(64).fill(0);
|
|
832
|
+
weights.forEach((weight, token) => {
|
|
833
|
+
const numericWeight = Number(weight);
|
|
834
|
+
if (!Number.isFinite(numericWeight) || numericWeight <= 0) {
|
|
835
|
+
return;
|
|
836
|
+
}
|
|
837
|
+
const tokenHash = getLocalVectorTokenHash64(token);
|
|
838
|
+
for (let bitIndex = 0; bitIndex < 64; bitIndex += 1) {
|
|
839
|
+
const bitMask = (tokenHash >> BigInt(bitIndex)) & 1n;
|
|
840
|
+
accumulator[bitIndex] += bitMask === 1n ? numericWeight : -numericWeight;
|
|
841
|
+
}
|
|
842
|
+
});
|
|
843
|
+
let signature = 0n;
|
|
844
|
+
for (let bitIndex = 0; bitIndex < 64; bitIndex += 1) {
|
|
845
|
+
if (accumulator[bitIndex] >= 0) {
|
|
846
|
+
signature |= (1n << BigInt(bitIndex));
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
return signature;
|
|
850
|
+
}
|
|
851
|
+
function computeSimHashPrefix(signature, prefixBits) {
|
|
852
|
+
const normalizedBits = Math.max(1, Math.min(64, Math.floor(Number(prefixBits) || 0)));
|
|
853
|
+
const shiftBits = Math.max(0, 64 - normalizedBits);
|
|
854
|
+
const prefixValue = shiftBits > 0
|
|
855
|
+
? (signature >> BigInt(shiftBits))
|
|
856
|
+
: signature;
|
|
857
|
+
const width = Math.max(1, Math.ceil(normalizedBits / 4));
|
|
858
|
+
return prefixValue.toString(16).padStart(width, '0');
|
|
859
|
+
}
|
|
860
|
+
function buildLocalVectorAnnStructures(entriesByAtomId) {
|
|
861
|
+
const tokenToAtomIds = new Map();
|
|
862
|
+
const signatureBuckets = new Map();
|
|
863
|
+
entriesByAtomId.forEach((entry, atomId) => {
|
|
864
|
+
entry.tokens.forEach((token) => {
|
|
865
|
+
if (!token) {
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
const postingList = tokenToAtomIds.get(token) || [];
|
|
869
|
+
postingList.push(atomId);
|
|
870
|
+
tokenToAtomIds.set(token, postingList);
|
|
871
|
+
});
|
|
872
|
+
const prefix = entry.signaturePrefix;
|
|
873
|
+
if (!prefix) {
|
|
874
|
+
return;
|
|
875
|
+
}
|
|
876
|
+
const bucket = signatureBuckets.get(prefix) || [];
|
|
877
|
+
bucket.push(atomId);
|
|
878
|
+
signatureBuckets.set(prefix, bucket);
|
|
879
|
+
});
|
|
880
|
+
return {
|
|
881
|
+
tokenToAtomIds,
|
|
882
|
+
signatureBuckets,
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
function selectLocalVectorAnnCandidatesByDefaultStrategy(input) {
|
|
886
|
+
if (!input.annPrefilterEnabled) {
|
|
887
|
+
return {
|
|
888
|
+
used: false,
|
|
889
|
+
candidateAtomIds: [],
|
|
890
|
+
mode: 'full_scan',
|
|
891
|
+
};
|
|
892
|
+
}
|
|
893
|
+
if (input.atomCount < LOCAL_VECTOR_ANN_MIN_ATOMS) {
|
|
894
|
+
return {
|
|
895
|
+
used: false,
|
|
896
|
+
candidateAtomIds: [],
|
|
897
|
+
mode: 'full_scan',
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
const prioritizedTokens = uniqueTokens([
|
|
901
|
+
...Array.from(input.queryWeights.entries())
|
|
902
|
+
.filter((entry) => entry[1] > 0)
|
|
903
|
+
.sort((left, right) => right[1] - left[1])
|
|
904
|
+
.map((entry) => entry[0]),
|
|
905
|
+
...input.queryTokens,
|
|
906
|
+
]).slice(0, LOCAL_VECTOR_ANN_TOP_QUERY_TOKENS);
|
|
907
|
+
if (prioritizedTokens.length <= 0) {
|
|
908
|
+
return {
|
|
909
|
+
used: false,
|
|
910
|
+
candidateAtomIds: [],
|
|
911
|
+
mode: 'full_scan',
|
|
912
|
+
};
|
|
913
|
+
}
|
|
914
|
+
const targetCandidateCount = Math.min(input.atomCount, Math.max(LOCAL_VECTOR_ANN_CANDIDATE_MIN_FLOOR, Math.max(1, input.topK) * LOCAL_VECTOR_ANN_CANDIDATE_MULTIPLIER));
|
|
915
|
+
const minimumCandidateCount = Math.min(input.atomCount, Math.max(Math.max(1, input.topK) * 2, Math.floor(LOCAL_VECTOR_ANN_CANDIDATE_MIN_FLOOR / 2)));
|
|
916
|
+
const candidateAtomIds = new Set();
|
|
917
|
+
let tokenPrefilterUsed = false;
|
|
918
|
+
prioritizedTokens.forEach((token) => {
|
|
919
|
+
if (candidateAtomIds.size >= targetCandidateCount) {
|
|
920
|
+
return;
|
|
921
|
+
}
|
|
922
|
+
const postingList = input.tokenToAtomIds.get(token);
|
|
923
|
+
if (!postingList || postingList.length <= 0) {
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
tokenPrefilterUsed = true;
|
|
927
|
+
postingList.forEach((atomId) => {
|
|
928
|
+
if (candidateAtomIds.size >= targetCandidateCount) {
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
const normalizedAtomId = String(atomId || '').trim();
|
|
932
|
+
if (normalizedAtomId) {
|
|
933
|
+
candidateAtomIds.add(normalizedAtomId);
|
|
934
|
+
}
|
|
935
|
+
});
|
|
936
|
+
});
|
|
937
|
+
let signaturePrefilterUsed = false;
|
|
938
|
+
if (candidateAtomIds.size < targetCandidateCount && input.queryWeights.size > 0) {
|
|
939
|
+
const querySignature = computeWeightedSimHash(new Map(input.queryWeights));
|
|
940
|
+
const queryPrefix = computeSimHashPrefix(querySignature, LOCAL_VECTOR_ANN_SIGNATURE_PREFIX_BITS);
|
|
941
|
+
const signatureBucket = input.signatureBuckets.get(queryPrefix);
|
|
942
|
+
if (signatureBucket && signatureBucket.length > 0) {
|
|
943
|
+
signaturePrefilterUsed = true;
|
|
944
|
+
signatureBucket.forEach((atomId) => {
|
|
945
|
+
if (candidateAtomIds.size >= targetCandidateCount) {
|
|
946
|
+
return;
|
|
947
|
+
}
|
|
948
|
+
const normalizedAtomId = String(atomId || '').trim();
|
|
949
|
+
if (normalizedAtomId) {
|
|
950
|
+
candidateAtomIds.add(normalizedAtomId);
|
|
951
|
+
}
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
if (!tokenPrefilterUsed || candidateAtomIds.size < minimumCandidateCount) {
|
|
956
|
+
return {
|
|
957
|
+
used: false,
|
|
958
|
+
candidateAtomIds: [],
|
|
959
|
+
mode: 'full_scan',
|
|
960
|
+
};
|
|
961
|
+
}
|
|
962
|
+
return {
|
|
963
|
+
used: true,
|
|
964
|
+
candidateAtomIds: Array.from(candidateAtomIds),
|
|
965
|
+
mode: signaturePrefilterUsed ? 'token_signature_prefilter' : 'token_prefilter',
|
|
966
|
+
};
|
|
967
|
+
}
|
|
968
|
+
const DEFAULT_LOCAL_VECTOR_ACCELERATION_ADAPTER = {
|
|
969
|
+
id: LOCAL_VECTOR_ACCELERATION_ADAPTER_ID,
|
|
970
|
+
syncIndex: () => ({
|
|
971
|
+
synced: true,
|
|
972
|
+
}),
|
|
973
|
+
selectCandidates: selectLocalVectorAnnCandidatesByDefaultStrategy,
|
|
974
|
+
getHealth: () => ({
|
|
975
|
+
status: 'ready',
|
|
976
|
+
message: 'local_ann_prefilter_active',
|
|
977
|
+
checkedAt: new Date().toISOString(),
|
|
978
|
+
indexSyncStatus: 'ready',
|
|
979
|
+
indexSyncMessage: 'local_adapter_index_sync_not_required',
|
|
980
|
+
representationVersion: LOCAL_VECTOR_REPRESENTATION_VERSION,
|
|
981
|
+
embeddingModelId: LOCAL_VECTOR_EMBEDDING_MODEL_ID,
|
|
982
|
+
representationStatus: 'aligned',
|
|
983
|
+
representationStatusReason: 'local_adapter_representation_aligned',
|
|
984
|
+
}),
|
|
985
|
+
};
|
|
986
|
+
async function selectLocalVectorAnnCandidates(params) {
|
|
987
|
+
const adapterId = String(params.accelerationAdapter?.id || '').trim() || LOCAL_VECTOR_ACCELERATION_ADAPTER_ID;
|
|
988
|
+
try {
|
|
989
|
+
if (typeof params.accelerationAdapter.syncIndex === 'function') {
|
|
990
|
+
await params.accelerationAdapter.syncIndex({
|
|
991
|
+
atomCount: params.index.atomCount,
|
|
992
|
+
tokenToAtomIds: params.index.tokenToAtomIds,
|
|
993
|
+
signatureBuckets: params.index.signatureBuckets,
|
|
994
|
+
representationVersion: LOCAL_VECTOR_REPRESENTATION_VERSION,
|
|
995
|
+
embeddingModelId: LOCAL_VECTOR_EMBEDDING_MODEL_ID,
|
|
996
|
+
embeddingDimension: params.index.documentFrequency.size,
|
|
997
|
+
indexSignature: params.index.signature,
|
|
998
|
+
});
|
|
999
|
+
}
|
|
1000
|
+
const rawSelection = await params.accelerationAdapter.selectCandidates({
|
|
1001
|
+
atomCount: params.index.atomCount,
|
|
1002
|
+
queryTokens: [...params.queryTokens],
|
|
1003
|
+
queryWeights: params.queryWeights,
|
|
1004
|
+
topK: params.topK,
|
|
1005
|
+
tokenToAtomIds: params.index.tokenToAtomIds,
|
|
1006
|
+
signatureBuckets: params.index.signatureBuckets,
|
|
1007
|
+
annPrefilterEnabled: params.annPrefilterEnabled,
|
|
1008
|
+
representationVersion: LOCAL_VECTOR_REPRESENTATION_VERSION,
|
|
1009
|
+
embeddingModelId: LOCAL_VECTOR_EMBEDDING_MODEL_ID,
|
|
1010
|
+
embeddingDimension: params.index.documentFrequency.size,
|
|
1011
|
+
indexSignature: params.index.signature,
|
|
1012
|
+
});
|
|
1013
|
+
const mode = (rawSelection?.mode === 'token_prefilter'
|
|
1014
|
+
|| rawSelection?.mode === 'token_signature_prefilter'
|
|
1015
|
+
|| rawSelection?.mode === 'full_scan') ? rawSelection.mode : 'full_scan';
|
|
1016
|
+
const candidateAtomIds = new Set(Array.isArray(rawSelection?.candidateAtomIds)
|
|
1017
|
+
? rawSelection.candidateAtomIds
|
|
1018
|
+
.map((atomId) => String(atomId || '').trim())
|
|
1019
|
+
.filter((atomId) => atomId.length > 0)
|
|
1020
|
+
: []);
|
|
1021
|
+
const used = rawSelection?.used === true
|
|
1022
|
+
&& mode !== 'full_scan'
|
|
1023
|
+
&& candidateAtomIds.size > 0;
|
|
1024
|
+
return {
|
|
1025
|
+
used,
|
|
1026
|
+
candidateAtomIds: used ? candidateAtomIds : new Set(),
|
|
1027
|
+
mode: used ? mode : 'full_scan',
|
|
1028
|
+
adapterId,
|
|
1029
|
+
};
|
|
1030
|
+
}
|
|
1031
|
+
catch (error) {
|
|
1032
|
+
const normalizedError = String(error?.message || error || 'unknown_error')
|
|
1033
|
+
.replace(/\s+/g, ' ')
|
|
1034
|
+
.trim()
|
|
1035
|
+
.slice(0, 240);
|
|
1036
|
+
if (params.failureMode === 'fail_closed') {
|
|
1037
|
+
throw new Error(`vector_acceleration_adapter_failure:${adapterId}:${normalizedError || 'unknown_error'}`);
|
|
1038
|
+
}
|
|
1039
|
+
return {
|
|
1040
|
+
used: false,
|
|
1041
|
+
candidateAtomIds: new Set(),
|
|
1042
|
+
mode: 'full_scan',
|
|
1043
|
+
adapterId,
|
|
1044
|
+
adapterError: normalizedError,
|
|
1045
|
+
};
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
class LocalHybridGraphQueryBackend {
|
|
1049
|
+
constructor() {
|
|
1050
|
+
this.id = 'local-hybrid-v1';
|
|
1051
|
+
}
|
|
1052
|
+
async query(context) {
|
|
1053
|
+
const queryVariants = buildQueryTextVariants(context);
|
|
1054
|
+
const querySemanticTokens = buildQuerySemanticTokens(context);
|
|
1055
|
+
const graphFeatureIndex = buildGraphFeatureIndex(context, querySemanticTokens);
|
|
1056
|
+
const candidates = [];
|
|
1057
|
+
context.atoms.forEach((atom) => {
|
|
1058
|
+
const titleLower = atom.title.toLowerCase();
|
|
1059
|
+
const contentLower = atom.content.toLowerCase();
|
|
1060
|
+
const keywordMatches = context.queryTokens.filter((token) => atom.keywords.includes(token)).length;
|
|
1061
|
+
const titleMatchBonus = queryVariants.some((variant) => titleLower.includes(variant)) ? 2 : 0;
|
|
1062
|
+
const contentMatchBonus = context.queryTokens.filter((token) => contentLower.includes(token)).length * 0.25;
|
|
1063
|
+
const atomSemanticTokens = buildAtomSemanticTokens(atom);
|
|
1064
|
+
const semanticSimilarity = computeJaccard(querySemanticTokens, atomSemanticTokens);
|
|
1065
|
+
const semanticBonus = semanticSimilarity * 1.6;
|
|
1066
|
+
const graphFeatureBonus = computeGraphFeatureScore(context, atom.id, graphFeatureIndex);
|
|
1067
|
+
const score = keywordMatches + titleMatchBonus + contentMatchBonus + semanticBonus + graphFeatureBonus;
|
|
1068
|
+
if (score > 0) {
|
|
1069
|
+
candidates.push({
|
|
1070
|
+
atomId: atom.id,
|
|
1071
|
+
score: Number(clamp(score, 0, 100).toFixed(4)),
|
|
1072
|
+
});
|
|
1073
|
+
}
|
|
1074
|
+
});
|
|
1075
|
+
candidates.sort((left, right) => right.score - left.score);
|
|
1076
|
+
const maxCandidates = Math.max(context.topK, Math.min(context.atoms.length, context.topK * 4));
|
|
1077
|
+
return {
|
|
1078
|
+
candidates: candidates.slice(0, maxCandidates),
|
|
1079
|
+
trace: {
|
|
1080
|
+
retrievalModes: ['keyword', 'semantic_similarity', 'graph_anchor_distance', 'graph_path_confidence', 'graph_intent_match', 'temporal_filter'],
|
|
1081
|
+
modeWeights: {
|
|
1082
|
+
keyword: 0.32,
|
|
1083
|
+
semantic: 0.2,
|
|
1084
|
+
graph: 0.3,
|
|
1085
|
+
temporal: 0.18,
|
|
1086
|
+
},
|
|
1087
|
+
},
|
|
1088
|
+
};
|
|
1089
|
+
}
|
|
1090
|
+
getDiagnostics() {
|
|
1091
|
+
return {
|
|
1092
|
+
backendId: this.id,
|
|
1093
|
+
ready: true,
|
|
1094
|
+
};
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
exports.LocalHybridGraphQueryBackend = LocalHybridGraphQueryBackend;
|
|
1098
|
+
class KeywordOnlyGraphQueryBackend {
|
|
1099
|
+
constructor() {
|
|
1100
|
+
this.id = 'keyword-only-v1';
|
|
1101
|
+
}
|
|
1102
|
+
async query(context) {
|
|
1103
|
+
const queryVariants = buildQueryTextVariants(context);
|
|
1104
|
+
const candidates = [];
|
|
1105
|
+
context.atoms.forEach((atom) => {
|
|
1106
|
+
const titleLower = atom.title.toLowerCase();
|
|
1107
|
+
const contentLower = atom.content.toLowerCase();
|
|
1108
|
+
const keywordMatches = context.queryTokens.filter((token) => atom.keywords.includes(token)).length;
|
|
1109
|
+
const titleMatchBonus = queryVariants.some((variant) => titleLower.includes(variant)) ? 2 : 0;
|
|
1110
|
+
const contentMatchBonus = context.queryTokens.filter((token) => contentLower.includes(token)).length * 0.22;
|
|
1111
|
+
const score = keywordMatches + titleMatchBonus + contentMatchBonus;
|
|
1112
|
+
if (score > 0) {
|
|
1113
|
+
candidates.push({
|
|
1114
|
+
atomId: atom.id,
|
|
1115
|
+
score: Number(clamp(score, 0, 100).toFixed(4)),
|
|
1116
|
+
});
|
|
1117
|
+
}
|
|
1118
|
+
});
|
|
1119
|
+
candidates.sort((left, right) => right.score - left.score);
|
|
1120
|
+
const maxCandidates = Math.max(context.topK, Math.min(context.atoms.length, context.topK * 4));
|
|
1121
|
+
return {
|
|
1122
|
+
candidates: candidates.slice(0, maxCandidates),
|
|
1123
|
+
trace: {
|
|
1124
|
+
retrievalModes: ['keyword', 'temporal_filter'],
|
|
1125
|
+
modeWeights: {
|
|
1126
|
+
keyword: 0.72,
|
|
1127
|
+
graph: 0.1,
|
|
1128
|
+
temporal: 0.18,
|
|
1129
|
+
},
|
|
1130
|
+
},
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
getDiagnostics() {
|
|
1134
|
+
return {
|
|
1135
|
+
backendId: this.id,
|
|
1136
|
+
ready: true,
|
|
1137
|
+
};
|
|
1138
|
+
}
|
|
1139
|
+
}
|
|
1140
|
+
exports.KeywordOnlyGraphQueryBackend = KeywordOnlyGraphQueryBackend;
|
|
1141
|
+
class LocalVectorGraphQueryBackend {
|
|
1142
|
+
constructor(options = {}) {
|
|
1143
|
+
this.id = LOCAL_VECTOR_INDEX_BACKEND_ID;
|
|
1144
|
+
this.activeIndex = null;
|
|
1145
|
+
this.pendingIndexSignature = null;
|
|
1146
|
+
this.pendingIndexBuild = null;
|
|
1147
|
+
this.invalidated = false;
|
|
1148
|
+
this.lastAnnSelectionMode = 'full_scan';
|
|
1149
|
+
this.lastAnnCandidateCount = 0;
|
|
1150
|
+
this.lastAnnAdapterId = LOCAL_VECTOR_ACCELERATION_ADAPTER_ID;
|
|
1151
|
+
this.lastAnnAdapterHealthStatus = 'unknown';
|
|
1152
|
+
this.lastAnnIndexSyncStatus = 'unknown';
|
|
1153
|
+
this.lastAnnIndexSyncRequestCount = 0;
|
|
1154
|
+
this.lastAnnIndexSyncSuccessCount = 0;
|
|
1155
|
+
this.lastAnnIndexSyncFailureCount = 0;
|
|
1156
|
+
this.lastAnnAdapterCircuitState = 'unknown';
|
|
1157
|
+
this.lastAnnAdapterConsecutiveFailures = 0;
|
|
1158
|
+
this.lastAnnAdapterRequestCount = 0;
|
|
1159
|
+
this.lastAnnAdapterRetryCount = 0;
|
|
1160
|
+
this.lastAnnAdapterShortCircuitCount = 0;
|
|
1161
|
+
this.lastAnnAdapterSuccessCount = 0;
|
|
1162
|
+
this.lastAnnAdapterFailureCount = 0;
|
|
1163
|
+
this.lastAnnAdapterHalfOpenProbeSuccessCount = 0;
|
|
1164
|
+
this.lastAnnAdapterHalfOpenProbeFailureCount = 0;
|
|
1165
|
+
this.lastAnnAdapterLastRetryAfterMs = 0;
|
|
1166
|
+
this.lastAnnRepresentationStatus = 'unknown';
|
|
1167
|
+
this.localVectorIndexPath = normalizeVectorIndexPath(options.localVectorIndexPath);
|
|
1168
|
+
this.localVectorAnnPrefilterEnabled = options.localVectorAnnPrefilterEnabled !== false;
|
|
1169
|
+
const providedAdapter = options.localVectorAccelerationAdapter;
|
|
1170
|
+
const normalizedAdapterId = String(providedAdapter?.id || '').trim();
|
|
1171
|
+
this.localVectorAccelerationAdapter = (providedAdapter
|
|
1172
|
+
&& normalizedAdapterId
|
|
1173
|
+
&& typeof providedAdapter.selectCandidates === 'function')
|
|
1174
|
+
? providedAdapter
|
|
1175
|
+
: DEFAULT_LOCAL_VECTOR_ACCELERATION_ADAPTER;
|
|
1176
|
+
this.localVectorAccelerationFailureMode = normalizeLocalVectorAccelerationFailureMode(options.localVectorAccelerationFailureMode);
|
|
1177
|
+
this.localVectorAccelerationRepresentationStrict =
|
|
1178
|
+
options.localVectorAccelerationRepresentationStrict === true;
|
|
1179
|
+
this.refreshAccelerationAdapterHealth();
|
|
1180
|
+
}
|
|
1181
|
+
async query(context) {
|
|
1182
|
+
const indexAtoms = this.resolveIndexAtoms(context);
|
|
1183
|
+
const index = await this.ensureIndex(indexAtoms);
|
|
1184
|
+
const querySemanticTokens = buildQuerySemanticTokens(context);
|
|
1185
|
+
const queryFrequency = buildTokenFrequency(querySemanticTokens);
|
|
1186
|
+
const graphFeatureIndex = buildGraphFeatureIndex(context, querySemanticTokens);
|
|
1187
|
+
const totalDocs = Math.max(1, index.atomCount);
|
|
1188
|
+
const queryWeights = buildTfIdfWeights(queryFrequency, index.documentFrequency, totalDocs);
|
|
1189
|
+
const querySignaturePrefix = computeSimHashPrefix(computeWeightedSimHash(queryWeights), LOCAL_VECTOR_ANN_SIGNATURE_PREFIX_BITS);
|
|
1190
|
+
let annCandidateSelection;
|
|
1191
|
+
try {
|
|
1192
|
+
annCandidateSelection = await selectLocalVectorAnnCandidates({
|
|
1193
|
+
index,
|
|
1194
|
+
queryTokens: querySemanticTokens,
|
|
1195
|
+
queryWeights,
|
|
1196
|
+
topK: context.topK,
|
|
1197
|
+
annPrefilterEnabled: this.localVectorAnnPrefilterEnabled,
|
|
1198
|
+
accelerationAdapter: this.localVectorAccelerationAdapter,
|
|
1199
|
+
failureMode: this.localVectorAccelerationFailureMode,
|
|
1200
|
+
});
|
|
1201
|
+
}
|
|
1202
|
+
catch (error) {
|
|
1203
|
+
this.refreshAccelerationAdapterHealth();
|
|
1204
|
+
this.lastAnnSelectionMode = 'full_scan';
|
|
1205
|
+
this.lastAnnCandidateCount = index.atomCount;
|
|
1206
|
+
this.lastAnnAdapterId = String(this.localVectorAccelerationAdapter?.id || '').trim()
|
|
1207
|
+
|| LOCAL_VECTOR_ACCELERATION_ADAPTER_ID;
|
|
1208
|
+
this.lastAnnAdapterError = String(error?.message || error || 'unknown_error')
|
|
1209
|
+
.replace(/\s+/g, ' ')
|
|
1210
|
+
.trim()
|
|
1211
|
+
.slice(0, 240);
|
|
1212
|
+
this.lastError = this.lastAnnAdapterError;
|
|
1213
|
+
throw error;
|
|
1214
|
+
}
|
|
1215
|
+
this.refreshAccelerationAdapterHealth();
|
|
1216
|
+
this.lastAnnSelectionMode = annCandidateSelection.mode;
|
|
1217
|
+
this.lastAnnAdapterId = annCandidateSelection.adapterId;
|
|
1218
|
+
this.lastAnnAdapterError = annCandidateSelection.adapterError;
|
|
1219
|
+
const representationTelemetry = this.resolveAccelerationRepresentationTelemetry(index);
|
|
1220
|
+
if (this.localVectorAccelerationRepresentationStrict
|
|
1221
|
+
&& representationTelemetry.representationStatus === 'mismatch') {
|
|
1222
|
+
const mismatchReason = String(representationTelemetry.representationStatusReason || 'representation_mismatch').trim().replace(/\s+/g, '_').slice(0, 200) || 'representation_mismatch';
|
|
1223
|
+
const strictMismatchError = (`vector_acceleration_representation_mismatch:${this.lastAnnAdapterId || 'unknown'}:${mismatchReason}`);
|
|
1224
|
+
this.lastAnnAdapterError = strictMismatchError;
|
|
1225
|
+
this.lastError = strictMismatchError;
|
|
1226
|
+
throw new Error(strictMismatchError);
|
|
1227
|
+
}
|
|
1228
|
+
const scopedAtomIds = new Set(context.atoms.map((atom) => atom.id));
|
|
1229
|
+
const scopedAnnCandidateAtomIds = annCandidateSelection.used
|
|
1230
|
+
? Array.from(annCandidateSelection.candidateAtomIds.values())
|
|
1231
|
+
.filter((atomId) => scopedAtomIds.has(atomId))
|
|
1232
|
+
: [];
|
|
1233
|
+
const annScopedPrefilterUsable = (annCandidateSelection.used
|
|
1234
|
+
&& scopedAtomIds.size > 0
|
|
1235
|
+
&& scopedAnnCandidateAtomIds.length > 0);
|
|
1236
|
+
const atomIdsToScore = annScopedPrefilterUsable
|
|
1237
|
+
? scopedAnnCandidateAtomIds
|
|
1238
|
+
: context.atoms.map((atom) => atom.id);
|
|
1239
|
+
this.lastAnnCandidateCount = annScopedPrefilterUsable
|
|
1240
|
+
? scopedAnnCandidateAtomIds.length
|
|
1241
|
+
: scopedAtomIds.size;
|
|
1242
|
+
const accelerationMode = (this.localVectorAnnPrefilterEnabled
|
|
1243
|
+
&& annScopedPrefilterUsable
|
|
1244
|
+
&& annCandidateSelection.mode !== 'full_scan') ? 'ann_prefilter' : 'full_scan';
|
|
1245
|
+
const vectorAccelerationTrace = {
|
|
1246
|
+
mode: accelerationMode,
|
|
1247
|
+
selectionMode: annCandidateSelection.mode,
|
|
1248
|
+
failureMode: this.localVectorAccelerationFailureMode,
|
|
1249
|
+
candidateCount: this.lastAnnCandidateCount,
|
|
1250
|
+
adapterId: this.lastAnnAdapterId,
|
|
1251
|
+
adapterError: this.lastAnnAdapterError,
|
|
1252
|
+
healthStatus: this.lastAnnAdapterHealthStatus,
|
|
1253
|
+
circuitState: this.lastAnnAdapterCircuitState,
|
|
1254
|
+
lastRequestId: this.lastAnnAdapterLastRequestId,
|
|
1255
|
+
lastErrorCode: this.lastAnnAdapterLastErrorCode,
|
|
1256
|
+
lastRetryAfterMs: this.lastAnnAdapterLastRetryAfterMs > 0
|
|
1257
|
+
? this.lastAnnAdapterLastRetryAfterMs
|
|
1258
|
+
: undefined,
|
|
1259
|
+
representationVersion: representationTelemetry.representationVersion,
|
|
1260
|
+
embeddingModelId: representationTelemetry.embeddingModelId,
|
|
1261
|
+
embeddingDimension: representationTelemetry.embeddingDimension,
|
|
1262
|
+
indexSignature: representationTelemetry.indexSignature,
|
|
1263
|
+
representationStatus: representationTelemetry.representationStatus,
|
|
1264
|
+
representationStatusReason: representationTelemetry.representationStatusReason,
|
|
1265
|
+
representationStrictMode: this.localVectorAccelerationRepresentationStrict,
|
|
1266
|
+
};
|
|
1267
|
+
const candidates = [];
|
|
1268
|
+
atomIdsToScore.forEach((atomId) => {
|
|
1269
|
+
const entry = index.entriesByAtomId.get(atomId);
|
|
1270
|
+
if (!entry) {
|
|
1271
|
+
return;
|
|
1272
|
+
}
|
|
1273
|
+
const cosineSimilarity = computeCosineSimilarity(queryWeights, entry.weights);
|
|
1274
|
+
const semanticOverlap = computeJaccard(querySemanticTokens, entry.tokens);
|
|
1275
|
+
const graphFeatureBonus = computeGraphFeatureScore(context, atomId, graphFeatureIndex);
|
|
1276
|
+
const signatureBonus = entry.signaturePrefix === querySignaturePrefix
|
|
1277
|
+
? LOCAL_VECTOR_ANN_SIGNATURE_SCORE_BONUS
|
|
1278
|
+
: 0;
|
|
1279
|
+
const score = (cosineSimilarity * 4.2) + (semanticOverlap * 1.1) + graphFeatureBonus + signatureBonus;
|
|
1280
|
+
if (score <= 0) {
|
|
1281
|
+
return;
|
|
1282
|
+
}
|
|
1283
|
+
candidates.push({
|
|
1284
|
+
atomId,
|
|
1285
|
+
score: Number(clamp(score, 0, 100).toFixed(4)),
|
|
1286
|
+
});
|
|
1287
|
+
});
|
|
1288
|
+
candidates.sort((left, right) => right.score - left.score);
|
|
1289
|
+
const maxCandidates = Math.max(context.topK, Math.min(context.atoms.length, context.topK * 4));
|
|
1290
|
+
return {
|
|
1291
|
+
candidates: candidates.slice(0, maxCandidates),
|
|
1292
|
+
trace: {
|
|
1293
|
+
retrievalModes: [
|
|
1294
|
+
'vector_similarity',
|
|
1295
|
+
'semantic_similarity',
|
|
1296
|
+
'graph_anchor_distance',
|
|
1297
|
+
'graph_path_confidence',
|
|
1298
|
+
'graph_intent_match',
|
|
1299
|
+
'temporal_filter',
|
|
1300
|
+
...(annScopedPrefilterUsable ? ['ann_prefilter'] : []),
|
|
1301
|
+
],
|
|
1302
|
+
modeWeights: {
|
|
1303
|
+
vector: 0.58,
|
|
1304
|
+
semantic: 0.14,
|
|
1305
|
+
graph: 0.1,
|
|
1306
|
+
temporal: 0.18,
|
|
1307
|
+
},
|
|
1308
|
+
vectorAcceleration: vectorAccelerationTrace,
|
|
1309
|
+
},
|
|
1310
|
+
};
|
|
1311
|
+
}
|
|
1312
|
+
resolveIndexAtoms(context) {
|
|
1313
|
+
const atomsById = new Map();
|
|
1314
|
+
const candidateIndexAtoms = (Array.isArray(context.indexAtoms)
|
|
1315
|
+
&& context.indexAtoms.length >= context.atoms.length)
|
|
1316
|
+
? context.indexAtoms
|
|
1317
|
+
: context.atoms;
|
|
1318
|
+
candidateIndexAtoms.forEach((atom) => {
|
|
1319
|
+
if (atom && atom.id) {
|
|
1320
|
+
atomsById.set(atom.id, atom);
|
|
1321
|
+
}
|
|
1322
|
+
});
|
|
1323
|
+
context.atoms.forEach((atom) => {
|
|
1324
|
+
if (atom && atom.id && !atomsById.has(atom.id)) {
|
|
1325
|
+
atomsById.set(atom.id, atom);
|
|
1326
|
+
}
|
|
1327
|
+
});
|
|
1328
|
+
return Array.from(atomsById.values());
|
|
1329
|
+
}
|
|
1330
|
+
getDiagnostics() {
|
|
1331
|
+
const index = this.activeIndex;
|
|
1332
|
+
const ready = (Boolean(index) && !this.invalidated) || !this.lastError;
|
|
1333
|
+
const status = (index && !this.invalidated)
|
|
1334
|
+
? 'ready'
|
|
1335
|
+
: (this.lastError ? 'unavailable' : 'stale');
|
|
1336
|
+
const accelerationMode = (this.localVectorAnnPrefilterEnabled
|
|
1337
|
+
&& this.lastAnnSelectionMode !== 'full_scan') ? 'ann_prefilter' : 'full_scan';
|
|
1338
|
+
const representationTelemetry = index
|
|
1339
|
+
? this.resolveAccelerationRepresentationTelemetry(index)
|
|
1340
|
+
: {
|
|
1341
|
+
representationVersion: this.lastAnnRepresentationVersion,
|
|
1342
|
+
embeddingModelId: this.lastAnnEmbeddingModelId,
|
|
1343
|
+
embeddingDimension: this.lastAnnEmbeddingDimension,
|
|
1344
|
+
indexSignature: this.lastAnnIndexSignature,
|
|
1345
|
+
representationStatus: this.lastAnnRepresentationStatus,
|
|
1346
|
+
representationStatusReason: this.lastAnnRepresentationStatusReason,
|
|
1347
|
+
};
|
|
1348
|
+
return {
|
|
1349
|
+
backendId: this.id,
|
|
1350
|
+
ready,
|
|
1351
|
+
lastError: this.lastError,
|
|
1352
|
+
vectorIndex: {
|
|
1353
|
+
enabled: true,
|
|
1354
|
+
status,
|
|
1355
|
+
location: this.localVectorIndexPath,
|
|
1356
|
+
signature: index?.signature,
|
|
1357
|
+
builtAt: index?.builtAt,
|
|
1358
|
+
atomCount: index?.atomCount,
|
|
1359
|
+
loadedFromDisk: index?.loadedFromDisk,
|
|
1360
|
+
persisted: index?.persisted,
|
|
1361
|
+
acceleration: {
|
|
1362
|
+
enabled: this.localVectorAnnPrefilterEnabled,
|
|
1363
|
+
mode: accelerationMode,
|
|
1364
|
+
failureMode: this.localVectorAccelerationFailureMode,
|
|
1365
|
+
indexSyncStatus: this.lastAnnIndexSyncStatus,
|
|
1366
|
+
indexSyncMessage: this.lastAnnIndexSyncMessage,
|
|
1367
|
+
lastSyncAt: this.lastAnnIndexLastSyncAt,
|
|
1368
|
+
syncRequestCount: this.lastAnnIndexSyncRequestCount,
|
|
1369
|
+
syncSuccessCount: this.lastAnnIndexSyncSuccessCount,
|
|
1370
|
+
syncFailureCount: this.lastAnnIndexSyncFailureCount,
|
|
1371
|
+
syncedIndexSignature: this.lastAnnIndexSyncedSignature,
|
|
1372
|
+
syncedAtomCount: this.lastAnnIndexSyncedAtomCount,
|
|
1373
|
+
lastSelectionMode: this.lastAnnSelectionMode,
|
|
1374
|
+
lastCandidateCount: this.lastAnnCandidateCount,
|
|
1375
|
+
adapterId: this.lastAnnAdapterId,
|
|
1376
|
+
adapterError: this.lastAnnAdapterError,
|
|
1377
|
+
healthStatus: this.lastAnnAdapterHealthStatus,
|
|
1378
|
+
healthMessage: this.lastAnnAdapterHealthMessage,
|
|
1379
|
+
healthCheckedAt: this.lastAnnAdapterHealthCheckedAt,
|
|
1380
|
+
circuitState: this.lastAnnAdapterCircuitState,
|
|
1381
|
+
consecutiveFailures: this.lastAnnAdapterConsecutiveFailures,
|
|
1382
|
+
requestCount: this.lastAnnAdapterRequestCount,
|
|
1383
|
+
retryCount: this.lastAnnAdapterRetryCount,
|
|
1384
|
+
shortCircuitCount: this.lastAnnAdapterShortCircuitCount,
|
|
1385
|
+
successCount: this.lastAnnAdapterSuccessCount,
|
|
1386
|
+
failureCount: this.lastAnnAdapterFailureCount,
|
|
1387
|
+
halfOpenProbeSuccessCount: this.lastAnnAdapterHalfOpenProbeSuccessCount,
|
|
1388
|
+
halfOpenProbeFailureCount: this.lastAnnAdapterHalfOpenProbeFailureCount,
|
|
1389
|
+
lastSuccessAt: this.lastAnnAdapterLastSuccessAt,
|
|
1390
|
+
lastFailureAt: this.lastAnnAdapterLastFailureAt,
|
|
1391
|
+
circuitOpenedAt: this.lastAnnAdapterCircuitOpenedAt,
|
|
1392
|
+
lastRequestId: this.lastAnnAdapterLastRequestId,
|
|
1393
|
+
lastErrorCode: this.lastAnnAdapterLastErrorCode,
|
|
1394
|
+
lastRetryAfterMs: this.lastAnnAdapterLastRetryAfterMs > 0
|
|
1395
|
+
? this.lastAnnAdapterLastRetryAfterMs
|
|
1396
|
+
: undefined,
|
|
1397
|
+
representationVersion: representationTelemetry.representationVersion,
|
|
1398
|
+
embeddingModelId: representationTelemetry.embeddingModelId,
|
|
1399
|
+
embeddingDimension: representationTelemetry.embeddingDimension,
|
|
1400
|
+
indexSignature: representationTelemetry.indexSignature,
|
|
1401
|
+
representationStatus: representationTelemetry.representationStatus,
|
|
1402
|
+
representationStatusReason: representationTelemetry.representationStatusReason,
|
|
1403
|
+
representationStrictMode: this.localVectorAccelerationRepresentationStrict,
|
|
1404
|
+
},
|
|
1405
|
+
},
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
invalidate(reason) {
|
|
1409
|
+
this.invalidated = true;
|
|
1410
|
+
this.invalidationReason = String(reason || '').trim().slice(0, 160) || undefined;
|
|
1411
|
+
}
|
|
1412
|
+
refreshAccelerationAdapterHealth() {
|
|
1413
|
+
if (!this.localVectorAccelerationAdapter || typeof this.localVectorAccelerationAdapter.getHealth !== 'function') {
|
|
1414
|
+
this.lastAnnAdapterHealthStatus = 'unknown';
|
|
1415
|
+
this.lastAnnAdapterHealthMessage = 'adapter_health_not_reported';
|
|
1416
|
+
this.lastAnnAdapterHealthCheckedAt = new Date().toISOString();
|
|
1417
|
+
this.lastAnnIndexSyncStatus = 'unknown';
|
|
1418
|
+
this.lastAnnIndexSyncMessage = 'adapter_health_not_reported';
|
|
1419
|
+
this.lastAnnIndexLastSyncAt = undefined;
|
|
1420
|
+
this.lastAnnIndexSyncRequestCount = 0;
|
|
1421
|
+
this.lastAnnIndexSyncSuccessCount = 0;
|
|
1422
|
+
this.lastAnnIndexSyncFailureCount = 0;
|
|
1423
|
+
this.lastAnnIndexSyncedSignature = undefined;
|
|
1424
|
+
this.lastAnnIndexSyncedAtomCount = undefined;
|
|
1425
|
+
this.lastAnnAdapterCircuitState = 'unknown';
|
|
1426
|
+
this.lastAnnAdapterConsecutiveFailures = 0;
|
|
1427
|
+
this.lastAnnAdapterRequestCount = 0;
|
|
1428
|
+
this.lastAnnAdapterRetryCount = 0;
|
|
1429
|
+
this.lastAnnAdapterShortCircuitCount = 0;
|
|
1430
|
+
this.lastAnnAdapterSuccessCount = 0;
|
|
1431
|
+
this.lastAnnAdapterFailureCount = 0;
|
|
1432
|
+
this.lastAnnAdapterHalfOpenProbeSuccessCount = 0;
|
|
1433
|
+
this.lastAnnAdapterHalfOpenProbeFailureCount = 0;
|
|
1434
|
+
this.lastAnnAdapterLastSuccessAt = undefined;
|
|
1435
|
+
this.lastAnnAdapterLastFailureAt = undefined;
|
|
1436
|
+
this.lastAnnAdapterCircuitOpenedAt = undefined;
|
|
1437
|
+
this.lastAnnAdapterLastRequestId = undefined;
|
|
1438
|
+
this.lastAnnAdapterLastErrorCode = undefined;
|
|
1439
|
+
this.lastAnnAdapterLastRetryAfterMs = 0;
|
|
1440
|
+
this.lastAnnRepresentationVersion = undefined;
|
|
1441
|
+
this.lastAnnEmbeddingModelId = undefined;
|
|
1442
|
+
this.lastAnnEmbeddingDimension = undefined;
|
|
1443
|
+
this.lastAnnIndexSignature = undefined;
|
|
1444
|
+
this.lastAnnRepresentationStatus = 'unknown';
|
|
1445
|
+
this.lastAnnRepresentationStatusReason = 'adapter_health_not_reported';
|
|
1446
|
+
return;
|
|
1447
|
+
}
|
|
1448
|
+
try {
|
|
1449
|
+
const health = this.localVectorAccelerationAdapter.getHealth();
|
|
1450
|
+
const normalizedStatus = String(health?.status || '').trim().toLowerCase();
|
|
1451
|
+
this.lastAnnAdapterHealthStatus = (normalizedStatus === 'ready'
|
|
1452
|
+
|| normalizedStatus === 'degraded'
|
|
1453
|
+
|| normalizedStatus === 'unavailable'
|
|
1454
|
+
|| normalizedStatus === 'unknown') ? normalizedStatus : 'unknown';
|
|
1455
|
+
this.lastAnnAdapterHealthMessage = String(health?.message || '').trim().slice(0, 240) || undefined;
|
|
1456
|
+
this.lastAnnAdapterHealthCheckedAt = String(health?.checkedAt || '').trim() || new Date().toISOString();
|
|
1457
|
+
const normalizedIndexSyncStatus = String(health?.indexSyncStatus || '').trim().toLowerCase();
|
|
1458
|
+
this.lastAnnIndexSyncStatus = (normalizedIndexSyncStatus === 'ready'
|
|
1459
|
+
|| normalizedIndexSyncStatus === 'degraded'
|
|
1460
|
+
|| normalizedIndexSyncStatus === 'unavailable'
|
|
1461
|
+
|| normalizedIndexSyncStatus === 'unknown') ? normalizedIndexSyncStatus : 'unknown';
|
|
1462
|
+
this.lastAnnIndexSyncMessage = String(health?.indexSyncMessage || '').trim().slice(0, 240) || undefined;
|
|
1463
|
+
this.lastAnnIndexLastSyncAt = String(health?.lastSyncAt || '').trim() || undefined;
|
|
1464
|
+
this.lastAnnIndexSyncRequestCount = Math.max(0, Math.floor(Number(health?.syncRequestCount || 0)));
|
|
1465
|
+
this.lastAnnIndexSyncSuccessCount = Math.max(0, Math.floor(Number(health?.syncSuccessCount || 0)));
|
|
1466
|
+
this.lastAnnIndexSyncFailureCount = Math.max(0, Math.floor(Number(health?.syncFailureCount || 0)));
|
|
1467
|
+
this.lastAnnIndexSyncedSignature = String(health?.syncedIndexSignature || '').trim().slice(0, 200) || undefined;
|
|
1468
|
+
this.lastAnnIndexSyncedAtomCount = Number.isFinite(Number(health?.syncedAtomCount))
|
|
1469
|
+
? Math.max(0, Math.floor(Number(health?.syncedAtomCount || 0)))
|
|
1470
|
+
: undefined;
|
|
1471
|
+
const normalizedCircuitState = String(health?.circuitState || '').trim().toLowerCase();
|
|
1472
|
+
this.lastAnnAdapterCircuitState = (normalizedCircuitState === 'closed'
|
|
1473
|
+
|| normalizedCircuitState === 'open'
|
|
1474
|
+
|| normalizedCircuitState === 'half_open'
|
|
1475
|
+
|| normalizedCircuitState === 'unknown') ? normalizedCircuitState : 'unknown';
|
|
1476
|
+
this.lastAnnAdapterConsecutiveFailures = Math.max(0, Math.floor(Number(health?.consecutiveFailures || 0)));
|
|
1477
|
+
this.lastAnnAdapterRequestCount = Math.max(0, Math.floor(Number(health?.requestCount || 0)));
|
|
1478
|
+
this.lastAnnAdapterRetryCount = Math.max(0, Math.floor(Number(health?.retryCount || 0)));
|
|
1479
|
+
this.lastAnnAdapterShortCircuitCount = Math.max(0, Math.floor(Number(health?.shortCircuitCount || 0)));
|
|
1480
|
+
this.lastAnnAdapterSuccessCount = Math.max(0, Math.floor(Number(health?.successCount || 0)));
|
|
1481
|
+
this.lastAnnAdapterFailureCount = Math.max(0, Math.floor(Number(health?.failureCount || 0)));
|
|
1482
|
+
this.lastAnnAdapterHalfOpenProbeSuccessCount = Math.max(0, Math.floor(Number(health?.halfOpenProbeSuccessCount || 0)));
|
|
1483
|
+
this.lastAnnAdapterHalfOpenProbeFailureCount = Math.max(0, Math.floor(Number(health?.halfOpenProbeFailureCount || 0)));
|
|
1484
|
+
this.lastAnnAdapterLastSuccessAt = String(health?.lastSuccessAt || '').trim() || undefined;
|
|
1485
|
+
this.lastAnnAdapterLastFailureAt = String(health?.lastFailureAt || '').trim() || undefined;
|
|
1486
|
+
this.lastAnnAdapterCircuitOpenedAt = String(health?.circuitOpenedAt || '').trim() || undefined;
|
|
1487
|
+
this.lastAnnAdapterLastRequestId = String(health?.lastRequestId || '').trim() || undefined;
|
|
1488
|
+
this.lastAnnAdapterLastErrorCode = String(health?.lastErrorCode || '').trim() || undefined;
|
|
1489
|
+
this.lastAnnAdapterLastRetryAfterMs = Math.max(0, Math.floor(Number(health?.lastRetryAfterMs || 0)));
|
|
1490
|
+
this.lastAnnRepresentationVersion = String(health?.representationVersion || '').trim().slice(0, 160) || undefined;
|
|
1491
|
+
this.lastAnnEmbeddingModelId = String(health?.embeddingModelId || '').trim().slice(0, 160) || undefined;
|
|
1492
|
+
this.lastAnnEmbeddingDimension = Number.isFinite(Number(health?.embeddingDimension))
|
|
1493
|
+
? Math.max(0, Math.floor(Number(health?.embeddingDimension || 0)))
|
|
1494
|
+
: undefined;
|
|
1495
|
+
this.lastAnnIndexSignature = String(health?.indexSignature || '')
|
|
1496
|
+
.trim()
|
|
1497
|
+
.replace(/[^a-zA-Z0-9:_-]+/g, '')
|
|
1498
|
+
.slice(0, 200) || undefined;
|
|
1499
|
+
const rawRepresentationStatus = String(health?.representationStatus || '').trim().toLowerCase();
|
|
1500
|
+
this.lastAnnRepresentationStatus = (rawRepresentationStatus === 'aligned'
|
|
1501
|
+
|| rawRepresentationStatus === 'mismatch'
|
|
1502
|
+
|| rawRepresentationStatus === 'unknown') ? rawRepresentationStatus : 'unknown';
|
|
1503
|
+
this.lastAnnRepresentationStatusReason = String(health?.representationStatusReason || '')
|
|
1504
|
+
.trim()
|
|
1505
|
+
.replace(/\s+/g, ' ')
|
|
1506
|
+
.slice(0, 240) || undefined;
|
|
1507
|
+
}
|
|
1508
|
+
catch (error) {
|
|
1509
|
+
this.lastAnnAdapterHealthStatus = 'unavailable';
|
|
1510
|
+
this.lastAnnAdapterHealthMessage = String(error?.message || error).slice(0, 240);
|
|
1511
|
+
this.lastAnnAdapterHealthCheckedAt = new Date().toISOString();
|
|
1512
|
+
this.lastAnnIndexSyncStatus = 'unknown';
|
|
1513
|
+
this.lastAnnIndexSyncMessage = 'adapter_health_probe_failed';
|
|
1514
|
+
this.lastAnnIndexLastSyncAt = undefined;
|
|
1515
|
+
this.lastAnnIndexSyncRequestCount = 0;
|
|
1516
|
+
this.lastAnnIndexSyncSuccessCount = 0;
|
|
1517
|
+
this.lastAnnIndexSyncFailureCount = 0;
|
|
1518
|
+
this.lastAnnIndexSyncedSignature = undefined;
|
|
1519
|
+
this.lastAnnIndexSyncedAtomCount = undefined;
|
|
1520
|
+
this.lastAnnAdapterCircuitState = 'unknown';
|
|
1521
|
+
this.lastAnnAdapterConsecutiveFailures = 0;
|
|
1522
|
+
this.lastAnnAdapterRequestCount = 0;
|
|
1523
|
+
this.lastAnnAdapterRetryCount = 0;
|
|
1524
|
+
this.lastAnnAdapterShortCircuitCount = 0;
|
|
1525
|
+
this.lastAnnAdapterSuccessCount = 0;
|
|
1526
|
+
this.lastAnnAdapterFailureCount = 0;
|
|
1527
|
+
this.lastAnnAdapterHalfOpenProbeSuccessCount = 0;
|
|
1528
|
+
this.lastAnnAdapterHalfOpenProbeFailureCount = 0;
|
|
1529
|
+
this.lastAnnAdapterLastSuccessAt = undefined;
|
|
1530
|
+
this.lastAnnAdapterLastFailureAt = undefined;
|
|
1531
|
+
this.lastAnnAdapterCircuitOpenedAt = undefined;
|
|
1532
|
+
this.lastAnnAdapterLastRequestId = undefined;
|
|
1533
|
+
this.lastAnnAdapterLastErrorCode = undefined;
|
|
1534
|
+
this.lastAnnAdapterLastRetryAfterMs = 0;
|
|
1535
|
+
this.lastAnnRepresentationVersion = undefined;
|
|
1536
|
+
this.lastAnnEmbeddingModelId = undefined;
|
|
1537
|
+
this.lastAnnEmbeddingDimension = undefined;
|
|
1538
|
+
this.lastAnnIndexSignature = undefined;
|
|
1539
|
+
this.lastAnnRepresentationStatus = 'unknown';
|
|
1540
|
+
this.lastAnnRepresentationStatusReason = 'adapter_health_probe_failed';
|
|
1541
|
+
}
|
|
1542
|
+
}
|
|
1543
|
+
resolveAccelerationRepresentationTelemetry(index) {
|
|
1544
|
+
const expectedRepresentationVersion = LOCAL_VECTOR_REPRESENTATION_VERSION;
|
|
1545
|
+
const expectedEmbeddingModelId = LOCAL_VECTOR_EMBEDDING_MODEL_ID;
|
|
1546
|
+
const expectedEmbeddingDimension = Math.max(0, Math.floor(Number(index.documentFrequency.size || 0)));
|
|
1547
|
+
const expectedIndexSignature = String(index.signature || '').trim().slice(0, 200);
|
|
1548
|
+
const adapterId = String(this.lastAnnAdapterId || '').trim().toLowerCase();
|
|
1549
|
+
const localAdapterPath = (!adapterId
|
|
1550
|
+
|| adapterId === LOCAL_VECTOR_ACCELERATION_ADAPTER_ID
|
|
1551
|
+
|| adapterId.includes('local-vector-acceleration'));
|
|
1552
|
+
let representationVersion = this.lastAnnRepresentationVersion;
|
|
1553
|
+
let embeddingModelId = this.lastAnnEmbeddingModelId;
|
|
1554
|
+
let embeddingDimension = this.lastAnnEmbeddingDimension;
|
|
1555
|
+
let indexSignature = this.lastAnnIndexSignature;
|
|
1556
|
+
let representationStatus = this.lastAnnRepresentationStatus;
|
|
1557
|
+
let representationStatusReason = this.lastAnnRepresentationStatusReason;
|
|
1558
|
+
const mismatchReasons = [];
|
|
1559
|
+
let comparableFields = 0;
|
|
1560
|
+
if (representationVersion) {
|
|
1561
|
+
comparableFields += 1;
|
|
1562
|
+
if (representationVersion !== expectedRepresentationVersion) {
|
|
1563
|
+
mismatchReasons.push('representation_version');
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
if (embeddingModelId) {
|
|
1567
|
+
comparableFields += 1;
|
|
1568
|
+
if (embeddingModelId !== expectedEmbeddingModelId) {
|
|
1569
|
+
mismatchReasons.push('embedding_model_id');
|
|
1570
|
+
}
|
|
1571
|
+
}
|
|
1572
|
+
if (typeof embeddingDimension === 'number' && embeddingDimension > 0) {
|
|
1573
|
+
comparableFields += 1;
|
|
1574
|
+
if (embeddingDimension !== expectedEmbeddingDimension) {
|
|
1575
|
+
mismatchReasons.push('embedding_dimension');
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
if (indexSignature) {
|
|
1579
|
+
comparableFields += 1;
|
|
1580
|
+
if (indexSignature !== expectedIndexSignature) {
|
|
1581
|
+
mismatchReasons.push('index_signature');
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
if (localAdapterPath) {
|
|
1585
|
+
representationVersion = expectedRepresentationVersion;
|
|
1586
|
+
embeddingModelId = expectedEmbeddingModelId;
|
|
1587
|
+
embeddingDimension = expectedEmbeddingDimension;
|
|
1588
|
+
indexSignature = expectedIndexSignature;
|
|
1589
|
+
representationStatus = 'aligned';
|
|
1590
|
+
representationStatusReason = 'local_adapter_representation_aligned';
|
|
1591
|
+
}
|
|
1592
|
+
else if (representationStatus === 'unknown') {
|
|
1593
|
+
if (mismatchReasons.length > 0) {
|
|
1594
|
+
representationStatus = 'mismatch';
|
|
1595
|
+
representationStatusReason = `mismatch:${mismatchReasons.join(',')}`;
|
|
1596
|
+
}
|
|
1597
|
+
else if (comparableFields > 0) {
|
|
1598
|
+
representationStatus = 'aligned';
|
|
1599
|
+
representationStatusReason = representationStatusReason || 'adapter_representation_metadata_aligned';
|
|
1600
|
+
}
|
|
1601
|
+
else {
|
|
1602
|
+
representationStatus = 'unknown';
|
|
1603
|
+
representationStatusReason = representationStatusReason || 'adapter_representation_metadata_missing';
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
else if (representationStatus === 'mismatch' && !representationStatusReason) {
|
|
1607
|
+
representationStatusReason = mismatchReasons.length > 0
|
|
1608
|
+
? `mismatch:${mismatchReasons.join(',')}`
|
|
1609
|
+
: 'adapter_reported_mismatch';
|
|
1610
|
+
}
|
|
1611
|
+
else if (representationStatus === 'aligned' && !representationStatusReason) {
|
|
1612
|
+
representationStatusReason = 'adapter_representation_metadata_aligned';
|
|
1613
|
+
}
|
|
1614
|
+
this.lastAnnRepresentationVersion = representationVersion || undefined;
|
|
1615
|
+
this.lastAnnEmbeddingModelId = embeddingModelId || undefined;
|
|
1616
|
+
this.lastAnnEmbeddingDimension = (typeof embeddingDimension === 'number'
|
|
1617
|
+
&& Number.isFinite(embeddingDimension)
|
|
1618
|
+
&& embeddingDimension > 0)
|
|
1619
|
+
? Math.floor(embeddingDimension)
|
|
1620
|
+
: undefined;
|
|
1621
|
+
this.lastAnnIndexSignature = indexSignature || undefined;
|
|
1622
|
+
this.lastAnnRepresentationStatus = representationStatus;
|
|
1623
|
+
this.lastAnnRepresentationStatusReason = representationStatusReason || undefined;
|
|
1624
|
+
return {
|
|
1625
|
+
representationVersion: this.lastAnnRepresentationVersion,
|
|
1626
|
+
embeddingModelId: this.lastAnnEmbeddingModelId,
|
|
1627
|
+
embeddingDimension: this.lastAnnEmbeddingDimension,
|
|
1628
|
+
indexSignature: this.lastAnnIndexSignature,
|
|
1629
|
+
representationStatus: this.lastAnnRepresentationStatus,
|
|
1630
|
+
representationStatusReason: this.lastAnnRepresentationStatusReason,
|
|
1631
|
+
};
|
|
1632
|
+
}
|
|
1633
|
+
async ensureIndex(atoms) {
|
|
1634
|
+
const signature = computeLocalVectorSignature(atoms);
|
|
1635
|
+
if (!this.invalidated && this.activeIndex && this.activeIndex.signature === signature) {
|
|
1636
|
+
return this.activeIndex;
|
|
1637
|
+
}
|
|
1638
|
+
if (this.pendingIndexBuild && this.pendingIndexSignature === signature) {
|
|
1639
|
+
return this.pendingIndexBuild;
|
|
1640
|
+
}
|
|
1641
|
+
this.pendingIndexSignature = signature;
|
|
1642
|
+
this.pendingIndexBuild = this.loadOrBuildIndex(signature, atoms)
|
|
1643
|
+
.finally(() => {
|
|
1644
|
+
this.pendingIndexBuild = null;
|
|
1645
|
+
this.pendingIndexSignature = null;
|
|
1646
|
+
});
|
|
1647
|
+
return this.pendingIndexBuild;
|
|
1648
|
+
}
|
|
1649
|
+
async loadOrBuildIndex(signature, atoms) {
|
|
1650
|
+
const fromDisk = await this.tryLoadIndexFromDisk(signature);
|
|
1651
|
+
if (fromDisk) {
|
|
1652
|
+
this.activeIndex = fromDisk;
|
|
1653
|
+
this.invalidated = false;
|
|
1654
|
+
this.invalidationReason = undefined;
|
|
1655
|
+
this.lastError = undefined;
|
|
1656
|
+
return fromDisk;
|
|
1657
|
+
}
|
|
1658
|
+
const builtAt = new Date().toISOString();
|
|
1659
|
+
const memoryIndex = this.buildIndexFromAtoms({
|
|
1660
|
+
signature,
|
|
1661
|
+
builtAt,
|
|
1662
|
+
atoms,
|
|
1663
|
+
loadedFromDisk: false,
|
|
1664
|
+
persisted: false,
|
|
1665
|
+
});
|
|
1666
|
+
if (this.localVectorIndexPath) {
|
|
1667
|
+
const persisted = await this.tryPersistIndexToDisk(memoryIndex);
|
|
1668
|
+
memoryIndex.persisted = persisted;
|
|
1669
|
+
}
|
|
1670
|
+
this.activeIndex = memoryIndex;
|
|
1671
|
+
this.invalidated = false;
|
|
1672
|
+
this.invalidationReason = undefined;
|
|
1673
|
+
if (memoryIndex.persisted || !this.localVectorIndexPath) {
|
|
1674
|
+
this.lastError = undefined;
|
|
1675
|
+
}
|
|
1676
|
+
return memoryIndex;
|
|
1677
|
+
}
|
|
1678
|
+
buildIndexFromAtoms(params) {
|
|
1679
|
+
const atomTokensById = new Map();
|
|
1680
|
+
const documentFrequency = new Map();
|
|
1681
|
+
params.atoms.forEach((atom) => {
|
|
1682
|
+
const tokens = buildAtomSemanticTokens(atom);
|
|
1683
|
+
atomTokensById.set(atom.id, tokens);
|
|
1684
|
+
new Set(tokens).forEach((token) => {
|
|
1685
|
+
documentFrequency.set(token, (documentFrequency.get(token) || 0) + 1);
|
|
1686
|
+
});
|
|
1687
|
+
});
|
|
1688
|
+
const totalDocs = Math.max(1, params.atoms.length);
|
|
1689
|
+
const entriesByAtomId = new Map();
|
|
1690
|
+
atomTokensById.forEach((tokens, atomId) => {
|
|
1691
|
+
const frequency = buildTokenFrequency(tokens);
|
|
1692
|
+
const weights = buildTfIdfWeights(frequency, documentFrequency, totalDocs);
|
|
1693
|
+
const signaturePrefix = computeSimHashPrefix(computeWeightedSimHash(weights), LOCAL_VECTOR_ANN_SIGNATURE_PREFIX_BITS);
|
|
1694
|
+
entriesByAtomId.set(atomId, {
|
|
1695
|
+
atomId,
|
|
1696
|
+
tokens,
|
|
1697
|
+
weights,
|
|
1698
|
+
signaturePrefix,
|
|
1699
|
+
});
|
|
1700
|
+
});
|
|
1701
|
+
const annStructures = buildLocalVectorAnnStructures(entriesByAtomId);
|
|
1702
|
+
return {
|
|
1703
|
+
signature: params.signature,
|
|
1704
|
+
builtAt: params.builtAt,
|
|
1705
|
+
atomCount: params.atoms.length,
|
|
1706
|
+
documentFrequency,
|
|
1707
|
+
entriesByAtomId,
|
|
1708
|
+
tokenToAtomIds: annStructures.tokenToAtomIds,
|
|
1709
|
+
signatureBuckets: annStructures.signatureBuckets,
|
|
1710
|
+
loadedFromDisk: params.loadedFromDisk,
|
|
1711
|
+
persisted: params.persisted,
|
|
1712
|
+
};
|
|
1713
|
+
}
|
|
1714
|
+
buildSnapshot(index) {
|
|
1715
|
+
const entries = Array.from(index.entriesByAtomId.values())
|
|
1716
|
+
.map((entry) => ({
|
|
1717
|
+
atomId: entry.atomId,
|
|
1718
|
+
tokens: [...entry.tokens],
|
|
1719
|
+
weights: toSerializableWeightEntries(entry.weights),
|
|
1720
|
+
}))
|
|
1721
|
+
.sort((left, right) => left.atomId.localeCompare(right.atomId));
|
|
1722
|
+
const documentFrequency = Array.from(index.documentFrequency.entries())
|
|
1723
|
+
.map(([token, count]) => [token, Math.max(0, Math.floor(Number(count) || 0))])
|
|
1724
|
+
.filter(([token, count]) => Boolean(token) && count > 0)
|
|
1725
|
+
.sort((left, right) => left[0].localeCompare(right[0]));
|
|
1726
|
+
return {
|
|
1727
|
+
schemaVersion: LOCAL_VECTOR_INDEX_SCHEMA_VERSION,
|
|
1728
|
+
backendId: this.id,
|
|
1729
|
+
signature: index.signature,
|
|
1730
|
+
builtAt: index.builtAt,
|
|
1731
|
+
atomCount: index.atomCount,
|
|
1732
|
+
documentFrequency,
|
|
1733
|
+
entries,
|
|
1734
|
+
};
|
|
1735
|
+
}
|
|
1736
|
+
parseSnapshot(rawSnapshot) {
|
|
1737
|
+
if (!rawSnapshot || typeof rawSnapshot !== 'object') {
|
|
1738
|
+
return null;
|
|
1739
|
+
}
|
|
1740
|
+
const snapshot = rawSnapshot;
|
|
1741
|
+
if (Number(snapshot.schemaVersion) !== LOCAL_VECTOR_INDEX_SCHEMA_VERSION
|
|
1742
|
+
|| String(snapshot.backendId || '') !== this.id
|
|
1743
|
+
|| !Array.isArray(snapshot.entries)
|
|
1744
|
+
|| !Array.isArray(snapshot.documentFrequency)) {
|
|
1745
|
+
return null;
|
|
1746
|
+
}
|
|
1747
|
+
return {
|
|
1748
|
+
schemaVersion: LOCAL_VECTOR_INDEX_SCHEMA_VERSION,
|
|
1749
|
+
backendId: this.id,
|
|
1750
|
+
signature: String(snapshot.signature || '').trim(),
|
|
1751
|
+
builtAt: String(snapshot.builtAt || '').trim(),
|
|
1752
|
+
atomCount: Math.max(0, Math.floor(Number(snapshot.atomCount || 0))),
|
|
1753
|
+
documentFrequency: snapshot.documentFrequency,
|
|
1754
|
+
entries: snapshot.entries,
|
|
1755
|
+
};
|
|
1756
|
+
}
|
|
1757
|
+
snapshotToIndex(snapshot) {
|
|
1758
|
+
const documentFrequency = new Map();
|
|
1759
|
+
snapshot.documentFrequency.forEach((entry) => {
|
|
1760
|
+
if (!Array.isArray(entry) || entry.length < 2) {
|
|
1761
|
+
return;
|
|
1762
|
+
}
|
|
1763
|
+
const token = String(entry[0] || '').trim().toLowerCase();
|
|
1764
|
+
const count = Math.max(0, Math.floor(Number(entry[1]) || 0));
|
|
1765
|
+
if (!token || count <= 0) {
|
|
1766
|
+
return;
|
|
1767
|
+
}
|
|
1768
|
+
documentFrequency.set(token, count);
|
|
1769
|
+
});
|
|
1770
|
+
const entriesByAtomId = new Map();
|
|
1771
|
+
snapshot.entries.forEach((entry) => {
|
|
1772
|
+
if (!entry || typeof entry !== 'object') {
|
|
1773
|
+
return;
|
|
1774
|
+
}
|
|
1775
|
+
const atomId = String(entry.atomId || '').trim();
|
|
1776
|
+
if (!atomId) {
|
|
1777
|
+
return;
|
|
1778
|
+
}
|
|
1779
|
+
const tokens = Array.isArray(entry.tokens)
|
|
1780
|
+
? uniqueTokens(entry.tokens.map((token) => stemSemanticToken(token)).filter((token) => token.length > 0))
|
|
1781
|
+
: [];
|
|
1782
|
+
const weights = toWeightMap(entry.weights);
|
|
1783
|
+
const signaturePrefix = computeSimHashPrefix(computeWeightedSimHash(weights), LOCAL_VECTOR_ANN_SIGNATURE_PREFIX_BITS);
|
|
1784
|
+
entriesByAtomId.set(atomId, {
|
|
1785
|
+
atomId,
|
|
1786
|
+
tokens,
|
|
1787
|
+
weights,
|
|
1788
|
+
signaturePrefix,
|
|
1789
|
+
});
|
|
1790
|
+
});
|
|
1791
|
+
const annStructures = buildLocalVectorAnnStructures(entriesByAtomId);
|
|
1792
|
+
return {
|
|
1793
|
+
signature: snapshot.signature,
|
|
1794
|
+
builtAt: snapshot.builtAt || new Date().toISOString(),
|
|
1795
|
+
atomCount: Math.max(snapshot.atomCount, entriesByAtomId.size),
|
|
1796
|
+
documentFrequency,
|
|
1797
|
+
entriesByAtomId,
|
|
1798
|
+
tokenToAtomIds: annStructures.tokenToAtomIds,
|
|
1799
|
+
signatureBuckets: annStructures.signatureBuckets,
|
|
1800
|
+
loadedFromDisk: true,
|
|
1801
|
+
persisted: true,
|
|
1802
|
+
};
|
|
1803
|
+
}
|
|
1804
|
+
async tryLoadIndexFromDisk(signature) {
|
|
1805
|
+
if (!this.localVectorIndexPath) {
|
|
1806
|
+
return null;
|
|
1807
|
+
}
|
|
1808
|
+
try {
|
|
1809
|
+
const content = await fs.promises.readFile(this.localVectorIndexPath, 'utf8');
|
|
1810
|
+
const parsedSnapshot = this.parseSnapshot(JSON.parse(content));
|
|
1811
|
+
if (!parsedSnapshot || parsedSnapshot.signature !== signature) {
|
|
1812
|
+
return null;
|
|
1813
|
+
}
|
|
1814
|
+
const index = this.snapshotToIndex(parsedSnapshot);
|
|
1815
|
+
return index;
|
|
1816
|
+
}
|
|
1817
|
+
catch (error) {
|
|
1818
|
+
const code = error?.code;
|
|
1819
|
+
if (code === 'ENOENT' || code === 'ENOTDIR') {
|
|
1820
|
+
return null;
|
|
1821
|
+
}
|
|
1822
|
+
this.lastError = String(error?.message || error);
|
|
1823
|
+
return null;
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
async tryPersistIndexToDisk(index) {
|
|
1827
|
+
if (!this.localVectorIndexPath) {
|
|
1828
|
+
return false;
|
|
1829
|
+
}
|
|
1830
|
+
const resolvedPath = path.resolve(this.localVectorIndexPath);
|
|
1831
|
+
const directory = path.dirname(resolvedPath);
|
|
1832
|
+
const tempPath = `${resolvedPath}.tmp`;
|
|
1833
|
+
try {
|
|
1834
|
+
await fs.promises.mkdir(directory, { recursive: true });
|
|
1835
|
+
await fs.promises.writeFile(tempPath, JSON.stringify(this.buildSnapshot(index), null, 2), 'utf8');
|
|
1836
|
+
await fs.promises.rename(tempPath, resolvedPath);
|
|
1837
|
+
return true;
|
|
1838
|
+
}
|
|
1839
|
+
catch (error) {
|
|
1840
|
+
this.lastError = String(error?.message || error);
|
|
1841
|
+
return false;
|
|
1842
|
+
}
|
|
1843
|
+
finally {
|
|
1844
|
+
try {
|
|
1845
|
+
await fs.promises.unlink(tempPath);
|
|
1846
|
+
}
|
|
1847
|
+
catch (_cleanupError) {
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
exports.LocalVectorGraphQueryBackend = LocalVectorGraphQueryBackend;
|
|
1853
|
+
function normalizeGraphQueryBackendType(rawValue) {
|
|
1854
|
+
const normalized = String(rawValue || '').trim().toLowerCase();
|
|
1855
|
+
if (normalized === 'keyword_only' || normalized === 'keyword-only' || normalized === 'keyword') {
|
|
1856
|
+
return 'keyword_only';
|
|
1857
|
+
}
|
|
1858
|
+
if (normalized === 'local_vector'
|
|
1859
|
+
|| normalized === 'local-vector'
|
|
1860
|
+
|| normalized === 'vector'
|
|
1861
|
+
|| normalized === 'semantic_vector'
|
|
1862
|
+
|| normalized === 'semantic-vector') {
|
|
1863
|
+
return 'local_vector';
|
|
1864
|
+
}
|
|
1865
|
+
return 'local_hybrid';
|
|
1866
|
+
}
|
|
1867
|
+
function normalizeLocalVectorAccelerationFailureMode(rawValue) {
|
|
1868
|
+
const normalized = String(rawValue || '').trim().toLowerCase();
|
|
1869
|
+
if (normalized === 'fail_closed'
|
|
1870
|
+
|| normalized === 'fail-closed'
|
|
1871
|
+
|| normalized === 'strict'
|
|
1872
|
+
|| normalized === 'closed') {
|
|
1873
|
+
return 'fail_closed';
|
|
1874
|
+
}
|
|
1875
|
+
return 'fail_open';
|
|
1876
|
+
}
|
|
1877
|
+
function createGraphQueryBackend(options = {}) {
|
|
1878
|
+
const backend = normalizeGraphQueryBackendType(options.backend);
|
|
1879
|
+
if (backend === 'keyword_only') {
|
|
1880
|
+
return new KeywordOnlyGraphQueryBackend();
|
|
1881
|
+
}
|
|
1882
|
+
if (backend === 'local_vector') {
|
|
1883
|
+
return new LocalVectorGraphQueryBackend({
|
|
1884
|
+
localVectorIndexPath: options.localVectorIndexPath,
|
|
1885
|
+
localVectorAnnPrefilterEnabled: options.localVectorAnnPrefilterEnabled,
|
|
1886
|
+
localVectorAccelerationAdapter: options.localVectorAccelerationAdapter,
|
|
1887
|
+
localVectorAccelerationFailureMode: normalizeLocalVectorAccelerationFailureMode(options.localVectorAccelerationFailureMode),
|
|
1888
|
+
localVectorAccelerationRepresentationStrict: options.localVectorAccelerationRepresentationStrict === true,
|
|
1889
|
+
});
|
|
1890
|
+
}
|
|
1891
|
+
return new LocalHybridGraphQueryBackend();
|
|
1892
|
+
}
|
|
1893
|
+
function listGraphQueryBackendCatalog() {
|
|
1894
|
+
return GRAPH_QUERY_BACKEND_CATALOG.map((item) => ({
|
|
1895
|
+
...item,
|
|
1896
|
+
aliases: [...item.aliases],
|
|
1897
|
+
}));
|
|
1898
|
+
}
|