@retrivora-ai/rag-engine 2.4.3 → 2.4.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-BGslIR5l.d.mts} +65 -4
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-C3gY6kZ0.d.ts} +65 -4
- package/dist/handlers/index.js +668 -837
- package/dist/handlers/index.mjs +678 -837
- package/dist/index.css +5 -0
- package/dist/index.d.mts +4 -28
- package/dist/index.d.ts +4 -28
- package/dist/index.js +91 -43
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +91 -43
- package/dist/server.d.mts +21 -7
- package/dist/server.d.ts +21 -7
- package/dist/server.js +731 -875
- package/dist/server.mjs +741 -875
- package/package.json +11 -4
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
import axios, { AxiosInstance } from 'axios';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* ChromaDBProvider — ChromaDB implementation using its REST API.
|
|
8
|
-
*
|
|
9
|
-
* Required options: { baseUrl: string } — e.g. "http://localhost:8000"
|
|
10
|
-
* Optional: { host?: string, port?: number } — used to construct baseUrl when baseUrl not set
|
|
11
|
-
*/
|
|
12
|
-
export class ChromaDBProvider extends BaseVectorProvider {
|
|
13
|
-
private http: AxiosInstance;
|
|
14
|
-
private collectionId: string = '';
|
|
15
|
-
|
|
16
|
-
constructor(config: VectorDBConfig) {
|
|
17
|
-
super(config);
|
|
18
|
-
const opts = config.options as Record<string, unknown>;
|
|
19
|
-
|
|
20
|
-
// Accept either explicit baseUrl or host+port
|
|
21
|
-
const baseUrl =
|
|
22
|
-
(opts.baseUrl as string) ||
|
|
23
|
-
(opts.host ? `http://${opts.host}:${(opts.port as number) || 8000}` : undefined);
|
|
24
|
-
|
|
25
|
-
if (!baseUrl) throw new Error('[ChromaDBProvider] options.baseUrl is required');
|
|
26
|
-
|
|
27
|
-
this.http = axios.create({
|
|
28
|
-
baseURL: baseUrl,
|
|
29
|
-
headers: { 'Content-Type': 'application/json' },
|
|
30
|
-
});
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Get or create the ChromaDB collection.
|
|
35
|
-
*/
|
|
36
|
-
async initialize(): Promise<void> {
|
|
37
|
-
try {
|
|
38
|
-
const { data } = await this.http.get(`/api/v1/collections/${this.indexName}`);
|
|
39
|
-
this.collectionId = data.id;
|
|
40
|
-
//console.log(`[ChromaDBProvider] ✅ Collection "${this.indexName}" found (id: ${this.collectionId})`);
|
|
41
|
-
} catch (err) {
|
|
42
|
-
if (axios.isAxiosError(err) && err.response?.status === 404) {
|
|
43
|
-
// Create the collection if it doesn't exist
|
|
44
|
-
//console.log(`[ChromaDBProvider] ⏳ Collection "${this.indexName}" not found. Creating...`);
|
|
45
|
-
const { data } = await this.http.post('/api/v1/collections', {
|
|
46
|
-
name: this.indexName,
|
|
47
|
-
});
|
|
48
|
-
this.collectionId = data.id;
|
|
49
|
-
//console.log(`[ChromaDBProvider] ✅ Created collection "${this.indexName}" (id: ${this.collectionId})`);
|
|
50
|
-
} else {
|
|
51
|
-
throw err;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
57
|
-
await this.batchUpsert([doc], namespace);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
61
|
-
const payload = {
|
|
62
|
-
ids: docs.map(d => String(d.id)),
|
|
63
|
-
embeddings: docs.map(d => d.vector),
|
|
64
|
-
documents: docs.map(d => d.content),
|
|
65
|
-
metadatas: docs.map(d => ({
|
|
66
|
-
...(d.metadata || {}),
|
|
67
|
-
...(namespace ? { namespace } : {}),
|
|
68
|
-
})),
|
|
69
|
-
};
|
|
70
|
-
await this.http.post(`/api/v1/collections/${this.collectionId}/add`, payload);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
74
|
-
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
75
|
-
const whereClauses: Record<string, unknown>[] = [];
|
|
76
|
-
if (namespace) whereClauses.push({ namespace: { $eq: namespace } });
|
|
77
|
-
Object.entries(sanitizedFilter).forEach(([key, value]) => {
|
|
78
|
-
if (key === 'namespace') return;
|
|
79
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
|
|
80
|
-
whereClauses.push({ [key]: { $eq: value } });
|
|
81
|
-
}
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
const payload = {
|
|
85
|
-
query_embeddings: [vector],
|
|
86
|
-
n_results: topK,
|
|
87
|
-
where: whereClauses.length > 1
|
|
88
|
-
? { $and: whereClauses }
|
|
89
|
-
: whereClauses[0],
|
|
90
|
-
};
|
|
91
|
-
const { data } = await this.http.post(`/api/v1/collections/${this.collectionId}/query`, payload);
|
|
92
|
-
|
|
93
|
-
const matches: VectorMatch[] = [];
|
|
94
|
-
if (data.ids && data.ids[0]) {
|
|
95
|
-
for (let i = 0; i < data.ids[0].length; i++) {
|
|
96
|
-
matches.push({
|
|
97
|
-
id: data.ids[0][i],
|
|
98
|
-
score: data.distances ? 1 - data.distances[0][i] : 0,
|
|
99
|
-
content: data.documents ? data.documents[0][i] : '',
|
|
100
|
-
metadata: data.metadatas ? data.metadatas[0][i] : {},
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
return matches;
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
async delete(id: string, namespace?: string): Promise<void> {
|
|
108
|
-
await this.http.post(`/api/v1/collections/${this.collectionId}/delete`, {
|
|
109
|
-
ids: [id],
|
|
110
|
-
where: namespace ? { namespace: { $eq: namespace } } : undefined,
|
|
111
|
-
});
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
115
|
-
await this.http.post(`/api/v1/collections/${this.collectionId}/delete`, {
|
|
116
|
-
where: { namespace: { $eq: namespace } },
|
|
117
|
-
});
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
async ping(): Promise<boolean> {
|
|
121
|
-
try {
|
|
122
|
-
await this.http.get('/api/v1/heartbeat');
|
|
123
|
-
return true;
|
|
124
|
-
} catch {
|
|
125
|
-
return false;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
async disconnect(): Promise<void> { }
|
|
130
|
-
}
|
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
import axios, { AxiosInstance } from 'axios';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* MilvusProvider — Milvus implementation using its REST API v1.
|
|
8
|
-
*
|
|
9
|
-
* Required options:
|
|
10
|
-
* { baseUrl: string } — e.g. "http://localhost:19530"
|
|
11
|
-
* OR { uri: string } — alias for baseUrl
|
|
12
|
-
* OR { host: string, port: number } — auto-constructs baseUrl
|
|
13
|
-
*
|
|
14
|
-
* Optional: { apiKey?: string, headers?: Record<string, string> }
|
|
15
|
-
*/
|
|
16
|
-
export class MilvusProvider extends BaseVectorProvider {
|
|
17
|
-
private http: AxiosInstance;
|
|
18
|
-
|
|
19
|
-
constructor(config: VectorDBConfig) {
|
|
20
|
-
super(config);
|
|
21
|
-
const opts = config.options as Record<string, unknown>;
|
|
22
|
-
|
|
23
|
-
// Accept baseUrl, uri, or host+port
|
|
24
|
-
const baseUrl =
|
|
25
|
-
(opts.baseUrl as string) ||
|
|
26
|
-
(opts.uri as string) ||
|
|
27
|
-
(opts.host ? `http://${opts.host}:${(opts.port as number) || 19530}` : undefined);
|
|
28
|
-
|
|
29
|
-
if (!baseUrl) throw new Error('[MilvusProvider] options.baseUrl (or uri / host+port) is required');
|
|
30
|
-
|
|
31
|
-
this.http = axios.create({
|
|
32
|
-
baseURL: baseUrl,
|
|
33
|
-
headers: {
|
|
34
|
-
'Content-Type': 'application/json',
|
|
35
|
-
...((opts.headers as Record<string, string>) || {}),
|
|
36
|
-
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey as string}` } : {}),
|
|
37
|
-
},
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
async initialize(): Promise<void> {
|
|
42
|
-
await this.ping();
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
46
|
-
const payload = {
|
|
47
|
-
collectionName: this.indexName,
|
|
48
|
-
data: [{
|
|
49
|
-
vector: doc.vector,
|
|
50
|
-
content: doc.content,
|
|
51
|
-
metadata: doc.metadata || {},
|
|
52
|
-
...(namespace ? { namespace } : {}),
|
|
53
|
-
}],
|
|
54
|
-
};
|
|
55
|
-
await this.http.post('/v1/vector/upsert', payload);
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
59
|
-
const payload = {
|
|
60
|
-
collectionName: this.indexName,
|
|
61
|
-
data: docs.map(doc => ({
|
|
62
|
-
vector: doc.vector,
|
|
63
|
-
content: doc.content,
|
|
64
|
-
metadata: doc.metadata || {},
|
|
65
|
-
...(namespace ? { namespace } : {}),
|
|
66
|
-
})),
|
|
67
|
-
};
|
|
68
|
-
await this.http.post('/v1/vector/upsert', payload);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
72
|
-
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
73
|
-
|
|
74
|
-
// Build Milvus boolean expression from filter fields.
|
|
75
|
-
// Namespace filtering is expressed as a field equality check.
|
|
76
|
-
const filterParts: string[] = [];
|
|
77
|
-
if (namespace) {
|
|
78
|
-
filterParts.push(`namespace == "${namespace}"`);
|
|
79
|
-
}
|
|
80
|
-
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
81
|
-
if (key === 'queryText' || key === 'keywords') continue;
|
|
82
|
-
if (typeof value === 'string') {
|
|
83
|
-
filterParts.push(`metadata["${key}"] == "${value.replace(/"/g, '\\"')}"`);
|
|
84
|
-
} else if (typeof value === 'number') {
|
|
85
|
-
filterParts.push(`metadata["${key}"] == ${value}`);
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
const payload = {
|
|
90
|
-
collectionName: this.indexName,
|
|
91
|
-
vector: vector,
|
|
92
|
-
limit: topK,
|
|
93
|
-
filter: filterParts.length > 0 ? filterParts.join(' && ') : undefined,
|
|
94
|
-
outputFields: ['content', 'metadata'],
|
|
95
|
-
searchParams: {
|
|
96
|
-
nprobe: (this.config.options.nprobe as number) || 16,
|
|
97
|
-
ef: (this.config.options.efSearch as number) || Math.max(topK * 10, 64)
|
|
98
|
-
}
|
|
99
|
-
};
|
|
100
|
-
const { data } = await this.http.post('/v1/vector/search', payload);
|
|
101
|
-
return (data.data || []).map((res: Record<string, unknown>) => ({
|
|
102
|
-
id: String(res['id']),
|
|
103
|
-
score: res['distance'] as number,
|
|
104
|
-
content: res['content'] as string,
|
|
105
|
-
metadata: (res['metadata'] as Record<string, unknown>) || {},
|
|
106
|
-
}));
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
async delete(id: string | number, _namespace?: string): Promise<void> {
|
|
110
|
-
await this.http.post('/v1/vector/delete', {
|
|
111
|
-
collectionName: this.indexName,
|
|
112
|
-
id: id,
|
|
113
|
-
});
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
117
|
-
// Milvus namespace deletion requires filtering — approximate via delete-with-filter
|
|
118
|
-
await this.http.post('/v1/vector/delete', {
|
|
119
|
-
collectionName: this.indexName,
|
|
120
|
-
filter: `namespace == "${namespace}"`,
|
|
121
|
-
});
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async ping(): Promise<boolean> {
|
|
125
|
-
try {
|
|
126
|
-
await this.http.get('/v1/health');
|
|
127
|
-
return true;
|
|
128
|
-
} catch {
|
|
129
|
-
return false;
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
async disconnect(): Promise<void> {}
|
|
134
|
-
}
|
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
import { MongoClient, Collection, Db } from 'mongodb';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
6
|
-
import { ValidationError } from '../../core/ConfigValidator';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* MongoDBProvider — MongoDB Atlas Vector Search implementation.
|
|
10
|
-
*/
|
|
11
|
-
export class MongoDBProvider extends BaseVectorProvider {
|
|
12
|
-
private client: MongoClient;
|
|
13
|
-
private db?: Db;
|
|
14
|
-
private collection?: Collection<any>;
|
|
15
|
-
private dbName: string;
|
|
16
|
-
private collectionName: string;
|
|
17
|
-
private embeddingKey: string;
|
|
18
|
-
private contentKey: string;
|
|
19
|
-
private metadataKey: string;
|
|
20
|
-
|
|
21
|
-
constructor(config: VectorDBConfig) {
|
|
22
|
-
super(config);
|
|
23
|
-
const opts = config.options as Record<string, unknown>;
|
|
24
|
-
if (!opts.uri || !opts.database || !opts.collection) {
|
|
25
|
-
throw new Error('[MongoDBProvider] options uri, database, and collection are required.');
|
|
26
|
-
}
|
|
27
|
-
this.client = new MongoClient(opts.uri as string, { serverSelectionTimeoutMS: 5000 });
|
|
28
|
-
this.dbName = opts.database as string;
|
|
29
|
-
this.collectionName = opts.collection as string;
|
|
30
|
-
this.embeddingKey = (opts.embeddingKey as string) || 'embedding';
|
|
31
|
-
this.contentKey = (opts.contentKey as string) || 'content';
|
|
32
|
-
this.metadataKey = (opts.metadataKey as string) || 'metadata';
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
static getValidator(): IProviderValidator {
|
|
36
|
-
return {
|
|
37
|
-
validate(config: Record<string, unknown>): ValidationError[] {
|
|
38
|
-
const errors: ValidationError[] = [];
|
|
39
|
-
const opts = (config.options || {}) as Record<string, unknown>;
|
|
40
|
-
|
|
41
|
-
if (!opts.uri) {
|
|
42
|
-
errors.push({
|
|
43
|
-
field: 'vectorDb.options.uri',
|
|
44
|
-
message: 'MongoDB connection URI is required',
|
|
45
|
-
suggestion: 'Set MONGODB_URI environment variable',
|
|
46
|
-
severity: 'error',
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
if (!opts.database) {
|
|
50
|
-
errors.push({
|
|
51
|
-
field: 'vectorDb.options.database',
|
|
52
|
-
message: 'MongoDB database name is required',
|
|
53
|
-
severity: 'error',
|
|
54
|
-
});
|
|
55
|
-
}
|
|
56
|
-
if (!opts.collection) {
|
|
57
|
-
errors.push({
|
|
58
|
-
field: 'vectorDb.options.collection',
|
|
59
|
-
message: 'MongoDB collection name is required',
|
|
60
|
-
severity: 'error',
|
|
61
|
-
});
|
|
62
|
-
}
|
|
63
|
-
return errors;
|
|
64
|
-
}
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
static getHealthChecker(): IProviderHealthChecker {
|
|
69
|
-
return {
|
|
70
|
-
async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
|
|
71
|
-
const opts = (config.options || {}) as Record<string, string>;
|
|
72
|
-
const timestamp = Date.now();
|
|
73
|
-
try {
|
|
74
|
-
const { MongoClient } = await import('mongodb');
|
|
75
|
-
const client = new MongoClient(opts.uri);
|
|
76
|
-
await client.connect();
|
|
77
|
-
const db = client.db(opts.database);
|
|
78
|
-
await db.command({ ping: 1 });
|
|
79
|
-
const collections = await db.listCollections({ name: opts.collection }).toArray();
|
|
80
|
-
await client.close();
|
|
81
|
-
|
|
82
|
-
return {
|
|
83
|
-
healthy: true,
|
|
84
|
-
provider: 'mongodb',
|
|
85
|
-
capabilities: {
|
|
86
|
-
database: opts.database,
|
|
87
|
-
collection: opts.collection,
|
|
88
|
-
exists: collections.length > 0
|
|
89
|
-
},
|
|
90
|
-
timestamp,
|
|
91
|
-
};
|
|
92
|
-
} catch (error) {
|
|
93
|
-
return {
|
|
94
|
-
healthy: false,
|
|
95
|
-
provider: 'mongodb',
|
|
96
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
97
|
-
timestamp,
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
};
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
async initialize(): Promise<void> {
|
|
105
|
-
await this.client.connect();
|
|
106
|
-
this.db = this.client.db(this.dbName);
|
|
107
|
-
this.collection = this.db.collection(this.collectionName);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
111
|
-
const document = {
|
|
112
|
-
_id: doc.id,
|
|
113
|
-
[this.embeddingKey]: doc.vector,
|
|
114
|
-
[this.contentKey]: doc.content,
|
|
115
|
-
[this.metadataKey]: doc.metadata || {},
|
|
116
|
-
...(namespace ? { namespace } : {}),
|
|
117
|
-
};
|
|
118
|
-
await this.collection!.updateOne({ _id: doc.id }, { $set: document }, { upsert: true });
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
122
|
-
const operations = docs.map((doc) => ({
|
|
123
|
-
updateOne: {
|
|
124
|
-
filter: { _id: doc.id },
|
|
125
|
-
update: {
|
|
126
|
-
$set: {
|
|
127
|
-
[this.embeddingKey]: doc.vector,
|
|
128
|
-
[this.contentKey]: doc.content,
|
|
129
|
-
[this.metadataKey]: doc.metadata || {},
|
|
130
|
-
...(namespace ? { namespace } : {}),
|
|
131
|
-
},
|
|
132
|
-
},
|
|
133
|
-
upsert: true,
|
|
134
|
-
},
|
|
135
|
-
}));
|
|
136
|
-
await this.collection!.bulkWrite(operations);
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
140
|
-
const publicFilter = this.sanitizeFilter(filter);
|
|
141
|
-
|
|
142
|
-
// Split filters: only 'namespace' is safe for the $vectorSearch 'filter' argument
|
|
143
|
-
// unless explicitly indexed. Moving others to a subsequent $match stage.
|
|
144
|
-
const vectorSearchFilter: Record<string, unknown> = {};
|
|
145
|
-
const matchFilter: Record<string, unknown> = {};
|
|
146
|
-
|
|
147
|
-
if (namespace) {
|
|
148
|
-
vectorSearchFilter.namespace = namespace;
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
for (const [key, value] of Object.entries(publicFilter)) {
|
|
152
|
-
if (key === 'namespace') {
|
|
153
|
-
vectorSearchFilter.namespace = value;
|
|
154
|
-
} else {
|
|
155
|
-
if (typeof value === 'string') {
|
|
156
|
-
matchFilter[key] = { $regex: new RegExp(`^${value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`, 'i') };
|
|
157
|
-
} else {
|
|
158
|
-
matchFilter[key] = value;
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
const pipeline: Record<string, unknown>[] = [
|
|
164
|
-
{
|
|
165
|
-
$vectorSearch: {
|
|
166
|
-
index: this.config.indexName || 'vector_index',
|
|
167
|
-
path: this.embeddingKey,
|
|
168
|
-
queryVector: vector,
|
|
169
|
-
numCandidates: (this.config.options.numCandidates as number) || Math.max(topK * 20, 200),
|
|
170
|
-
limit: topK * 2, // Increase limit slightly to account for post-filtering
|
|
171
|
-
...(Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {}),
|
|
172
|
-
},
|
|
173
|
-
},
|
|
174
|
-
];
|
|
175
|
-
|
|
176
|
-
// Add post-filtering stage if there are additional filters
|
|
177
|
-
if (Object.keys(matchFilter).length > 0) {
|
|
178
|
-
pipeline.push({ $match: matchFilter });
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
// Final limit and projection
|
|
182
|
-
pipeline.push(
|
|
183
|
-
{ $limit: topK },
|
|
184
|
-
{
|
|
185
|
-
$project: {
|
|
186
|
-
_id: 1,
|
|
187
|
-
[this.contentKey]: 1,
|
|
188
|
-
[this.metadataKey]: 1,
|
|
189
|
-
namespace: 1,
|
|
190
|
-
score: { $meta: 'vectorSearchScore' },
|
|
191
|
-
},
|
|
192
|
-
}
|
|
193
|
-
);
|
|
194
|
-
|
|
195
|
-
const results = await this.collection!.aggregate(pipeline).toArray();
|
|
196
|
-
|
|
197
|
-
return (results as unknown as Array<{ _id: string; score: number; [key: string]: unknown }>).map((res) => {
|
|
198
|
-
// Atlas Vector Search returns cosine similarity mapped to [0, 1] via (1 + cos_sim) / 2.
|
|
199
|
-
// We reverse this mapping with (score * 2) - 1 to restore the standard cosine range
|
|
200
|
-
// of [-1, 1], so downstream scoreThreshold comparisons work consistently across
|
|
201
|
-
// all vector DB providers (which natively return values in [-1, 1]).
|
|
202
|
-
const normalizedScore = (res.score * 2) - 1;
|
|
203
|
-
|
|
204
|
-
return {
|
|
205
|
-
id: res._id,
|
|
206
|
-
content: res[this.contentKey] as string,
|
|
207
|
-
metadata: (res[this.metadataKey] as Record<string, unknown>) || {},
|
|
208
|
-
score: normalizedScore,
|
|
209
|
-
};
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
async delete(id: string | number, namespace?: string): Promise<void> {
|
|
214
|
-
await this.collection!.deleteOne({ _id: id, ...(namespace ? { namespace } : {}) });
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
218
|
-
await this.collection!.deleteMany({ namespace });
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
protected sanitizeFilter(filter?: Record<string, unknown>): Record<string, unknown> {
|
|
222
|
-
const sanitized = super.sanitizeFilter(filter);
|
|
223
|
-
const mongoFilter: Record<string, unknown> = {};
|
|
224
|
-
|
|
225
|
-
for (const [key, value] of Object.entries(sanitized)) {
|
|
226
|
-
if (key.startsWith('__') || key === 'keywords' || key === 'queryText') continue;
|
|
227
|
-
|
|
228
|
-
if (key === 'namespace') {
|
|
229
|
-
mongoFilter[key] = value;
|
|
230
|
-
} else {
|
|
231
|
-
mongoFilter[`${this.metadataKey}.${key}`] = value;
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
return mongoFilter;
|
|
236
|
-
}
|
|
237
|
-
|
|
238
|
-
async ping(): Promise<boolean> {
|
|
239
|
-
try {
|
|
240
|
-
await this.db!.command({ ping: 1 });
|
|
241
|
-
return true;
|
|
242
|
-
} catch {
|
|
243
|
-
return false;
|
|
244
|
-
}
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
async disconnect(): Promise<void> {
|
|
248
|
-
await this.client.close();
|
|
249
|
-
}
|
|
250
|
-
}
|