@retrivora-ai/rag-engine 2.4.3 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +27 -4
- package/dist/handlers/index.mjs +27 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +12 -10
- package/dist/index.mjs +12 -10
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +28 -4
- package/dist/server.mjs +28 -4
- package/package.json +1 -3
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,294 +0,0 @@
|
|
|
1
|
-
import axios, { AxiosInstance } from 'axios';
|
|
2
|
-
import crypto from 'crypto';
|
|
3
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
4
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
5
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* QdrantProvider — implementation for Qdrant using its REST API.
|
|
9
|
-
* Fully dynamic: automatically discovers and indexes schema from available data.
|
|
10
|
-
*/
|
|
11
|
-
export class QdrantProvider extends BaseVectorProvider {
|
|
12
|
-
private http: AxiosInstance;
|
|
13
|
-
private contentField: string;
|
|
14
|
-
private metadataField: string;
|
|
15
|
-
private isFlatPayload: boolean;
|
|
16
|
-
private schemaDiscovered = false;
|
|
17
|
-
|
|
18
|
-
constructor(config: VectorDBConfig) {
|
|
19
|
-
super(config);
|
|
20
|
-
const opts = config.options as Record<string, unknown>;
|
|
21
|
-
const baseUrl = opts.baseUrl as string;
|
|
22
|
-
if (!baseUrl) throw new Error('[QdrantProvider] baseUrl is required');
|
|
23
|
-
|
|
24
|
-
this.contentField = (opts.contentField as string) || 'content';
|
|
25
|
-
this.metadataField = (opts.metadataField as string) || 'metadata';
|
|
26
|
-
this.isFlatPayload = !!opts.flatPayload;
|
|
27
|
-
|
|
28
|
-
this.http = axios.create({
|
|
29
|
-
baseURL: baseUrl,
|
|
30
|
-
timeout: 15000, // 15s timeout for vector DB operations
|
|
31
|
-
headers: {
|
|
32
|
-
'Content-Type': 'application/json',
|
|
33
|
-
...(opts.apiKey ? { 'api-key': opts.apiKey as string } : {}),
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
}
|
|
37
|
-
|
|
38
|
-
async initialize(): Promise<void> {
|
|
39
|
-
if (this.schemaDiscovered) return;
|
|
40
|
-
|
|
41
|
-
await this.ping();
|
|
42
|
-
await this.ensureCollection();
|
|
43
|
-
|
|
44
|
-
// 1. Discover schema from existing data
|
|
45
|
-
//console.log(`[QdrantProvider] 🔍 Discovering schema for collection "${this.indexName}"...`);
|
|
46
|
-
const discoveredFields = await this.discoverSchema();
|
|
47
|
-
|
|
48
|
-
// 2. Add client-provided fields (if any) as fallback/override
|
|
49
|
-
const opts = this.config.options as Record<string, unknown>;
|
|
50
|
-
const configFields = (opts.filterableFields as string[]) || [];
|
|
51
|
-
|
|
52
|
-
const allFields = [...new Set([...discoveredFields, ...configFields])];
|
|
53
|
-
|
|
54
|
-
if (allFields.length > 0) {
|
|
55
|
-
//console.log(`[QdrantProvider] 🛠 Ensuring indexes for ${allFields.length} discovered fields...`);
|
|
56
|
-
// Parallelize index creation for much faster startup
|
|
57
|
-
await Promise.all(
|
|
58
|
-
allFields.map(async (fieldInfo) => {
|
|
59
|
-
const [fieldName, schemaType] = fieldInfo.split(':');
|
|
60
|
-
return this.ensureIndex(fieldName, (schemaType as string) || 'keyword');
|
|
61
|
-
})
|
|
62
|
-
);
|
|
63
|
-
} else {
|
|
64
|
-
//console.log(`[QdrantProvider] ℹ️ No fields discovered for indexing.`);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
this.schemaDiscovered = true;
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
/**
|
|
71
|
-
* Samples points from the collection to discover available payload fields.
|
|
72
|
-
*/
|
|
73
|
-
private async discoverSchema(): Promise<string[]> {
|
|
74
|
-
try {
|
|
75
|
-
// Fetch 20 points to get a representative sample of the schema
|
|
76
|
-
const { data } = await this.http.post(`/collections/${this.indexName}/points/scroll`, {
|
|
77
|
-
limit: 20,
|
|
78
|
-
with_payload: true,
|
|
79
|
-
with_vector: false
|
|
80
|
-
});
|
|
81
|
-
|
|
82
|
-
const points = data.result?.points || [];
|
|
83
|
-
const keys = new Set<string>();
|
|
84
|
-
|
|
85
|
-
for (const point of points) {
|
|
86
|
-
const payload = point.payload || {};
|
|
87
|
-
|
|
88
|
-
// Extract root keys
|
|
89
|
-
Object.keys(payload).forEach(k => {
|
|
90
|
-
if (k !== 'namespace' && k !== this.contentField && k !== this.metadataField) {
|
|
91
|
-
keys.add(k);
|
|
92
|
-
}
|
|
93
|
-
});
|
|
94
|
-
|
|
95
|
-
// Extract nested metadata keys if applicable
|
|
96
|
-
if (!this.isFlatPayload && payload[this.metadataField]) {
|
|
97
|
-
const meta = payload[this.metadataField];
|
|
98
|
-
if (typeof meta === 'object' && meta !== null) {
|
|
99
|
-
Object.keys(meta).forEach(k => keys.add(k));
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
return Array.from(keys);
|
|
105
|
-
} catch (err) {
|
|
106
|
-
console.warn(`[QdrantProvider] ⚠️ Schema discovery failed:`, err instanceof Error ? err.message : String(err));
|
|
107
|
-
return [];
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* Ensures the collection exists. Creates it if missing.
|
|
113
|
-
*/
|
|
114
|
-
private async ensureCollection(): Promise<void> {
|
|
115
|
-
try {
|
|
116
|
-
await this.http.get(`/collections/${this.indexName}`);
|
|
117
|
-
//console.log(`[QdrantProvider] ✅ Collection "${this.indexName}" already exists.`);
|
|
118
|
-
} catch (err) {
|
|
119
|
-
if (axios.isAxiosError(err) && err.response?.status === 404) {
|
|
120
|
-
const opts = this.config.options as Record<string, unknown>;
|
|
121
|
-
const dimensionsForCreate = (opts.dimensions as number) || 1536;
|
|
122
|
-
|
|
123
|
-
//console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
124
|
-
await this.http.put(`/collections/${this.indexName}`, {
|
|
125
|
-
vectors: {
|
|
126
|
-
size: dimensionsForCreate,
|
|
127
|
-
distance: 'Cosine',
|
|
128
|
-
},
|
|
129
|
-
});
|
|
130
|
-
//console.log(`[QdrantProvider] ✅ Created collection "${this.indexName}"`);
|
|
131
|
-
} else {
|
|
132
|
-
throw err;
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Ensures that a payload field has an index.
|
|
139
|
-
*/
|
|
140
|
-
private async ensureIndex(fieldName: string, schema: string = 'keyword'): Promise<void> {
|
|
141
|
-
let fullPath = fieldName;
|
|
142
|
-
if (!this.isFlatPayload && !fieldName.includes('.') && fieldName !== 'namespace' && fieldName !== this.contentField) {
|
|
143
|
-
fullPath = `${this.metadataField}.${fieldName}`;
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
try {
|
|
147
|
-
await this.http.put(`/collections/${this.indexName}/index`, {
|
|
148
|
-
field_name: fullPath,
|
|
149
|
-
field_schema: schema,
|
|
150
|
-
});
|
|
151
|
-
//console.log(`[QdrantProvider] ✅ Ensured ${schema} index for "${fullPath}"`);
|
|
152
|
-
} catch (err) {
|
|
153
|
-
let status: number | undefined;
|
|
154
|
-
if (axios.isAxiosError(err)) status = err.response?.status;
|
|
155
|
-
if (status === 409) return;
|
|
156
|
-
console.warn(`[QdrantProvider] ⚠️ Could not ensure index for "${fullPath}":`, err instanceof Error ? err.message : String(err));
|
|
157
|
-
}
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
161
|
-
await this.batchUpsert([doc], namespace);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
165
|
-
const payload = {
|
|
166
|
-
points: docs.map(doc => {
|
|
167
|
-
const pointPayload: Record<string, unknown> = {
|
|
168
|
-
[this.contentField]: doc.content,
|
|
169
|
-
...(namespace ? { namespace } : {}),
|
|
170
|
-
};
|
|
171
|
-
|
|
172
|
-
if (this.isFlatPayload) {
|
|
173
|
-
Object.assign(pointPayload, doc.metadata || {});
|
|
174
|
-
} else {
|
|
175
|
-
pointPayload[this.metadataField] = doc.metadata || {};
|
|
176
|
-
}
|
|
177
|
-
|
|
178
|
-
return {
|
|
179
|
-
id: this.normalizeId(doc.id),
|
|
180
|
-
vector: doc.vector,
|
|
181
|
-
payload: pointPayload,
|
|
182
|
-
};
|
|
183
|
-
}),
|
|
184
|
-
};
|
|
185
|
-
await this.http.put(`/collections/${this.indexName}/points`, payload);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
189
|
-
const must: Record<string, unknown>[] = [];
|
|
190
|
-
|
|
191
|
-
if (namespace) {
|
|
192
|
-
must.push({ key: 'namespace', match: { value: namespace } });
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
const sanitizedFilter = this.sanitizeFilter(_filter);
|
|
196
|
-
for (const [key, value] of Object.entries(sanitizedFilter)) {
|
|
197
|
-
let filterKey = key;
|
|
198
|
-
if (!this.isFlatPayload && !key.includes('.') && key !== 'namespace' && key !== this.contentField) {
|
|
199
|
-
filterKey = `${this.metadataField}.${key}`;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
if (Array.isArray(value)) {
|
|
203
|
-
must.push({ key: filterKey, match: { any: value } });
|
|
204
|
-
} else {
|
|
205
|
-
must.push({ key: filterKey, match: { value: value } });
|
|
206
|
-
}
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
const payload = {
|
|
210
|
-
vector: vector,
|
|
211
|
-
limit: topK,
|
|
212
|
-
with_payload: true,
|
|
213
|
-
params: {
|
|
214
|
-
hnsw_ef: ((this.config.options as Record<string, unknown>)?.efSearch as number | undefined) || Math.max(topK * 20, 128),
|
|
215
|
-
exact: false
|
|
216
|
-
},
|
|
217
|
-
filter: must.length > 0 ? { must } : undefined,
|
|
218
|
-
};
|
|
219
|
-
|
|
220
|
-
const { data } = await this.http.post(`/collections/${this.indexName}/points/search`, payload);
|
|
221
|
-
|
|
222
|
-
const results = (data.result || []).map((res: Record<string, unknown>) => {
|
|
223
|
-
const p = (res.payload as Record<string, unknown>) || {};
|
|
224
|
-
let content = p[this.contentField] || '';
|
|
225
|
-
|
|
226
|
-
if (!content) {
|
|
227
|
-
const stringFields = Object.entries(p)
|
|
228
|
-
.filter(([, v]) => typeof v === 'string')
|
|
229
|
-
.map(([k, v]) => ({ key: k, val: v as string }));
|
|
230
|
-
|
|
231
|
-
if (stringFields.length > 0) {
|
|
232
|
-
const bestMatch = stringFields.sort((a, b) => b.val.length - a.val.length)[0];
|
|
233
|
-
content = bestMatch.val;
|
|
234
|
-
} else {
|
|
235
|
-
content = JSON.stringify(p);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
let metadata: Record<string, unknown> = {};
|
|
240
|
-
if (this.isFlatPayload) {
|
|
241
|
-
metadata = { ...p };
|
|
242
|
-
delete metadata[this.contentField];
|
|
243
|
-
delete metadata['namespace'];
|
|
244
|
-
} else {
|
|
245
|
-
metadata = (p[this.metadataField] as Record<string, unknown>) || p;
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
return {
|
|
249
|
-
id: res.id as string,
|
|
250
|
-
score: res.score as number,
|
|
251
|
-
content,
|
|
252
|
-
metadata,
|
|
253
|
-
};
|
|
254
|
-
});
|
|
255
|
-
|
|
256
|
-
return results;
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
async delete(id: string | number): Promise<void> {
|
|
260
|
-
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
261
|
-
points: [this.normalizeId(id)],
|
|
262
|
-
});
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
async deleteNamespace(_namespace: string): Promise<void> {
|
|
266
|
-
await this.http.post(`/collections/${this.indexName}/points/delete`, {
|
|
267
|
-
filter: {
|
|
268
|
-
must: [{ key: 'namespace', match: { value: _namespace } }]
|
|
269
|
-
}
|
|
270
|
-
});
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
async ping(): Promise<boolean> {
|
|
274
|
-
try {
|
|
275
|
-
await this.http.get('/healthz');
|
|
276
|
-
return true;
|
|
277
|
-
} catch {
|
|
278
|
-
return false;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
|
|
282
|
-
private normalizeId(id: string | number): string | number {
|
|
283
|
-
if (typeof id === 'number') return id;
|
|
284
|
-
const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
285
|
-
if (uuidRegex.test(id)) return id.toLowerCase();
|
|
286
|
-
const hash = crypto.createHash('md5').update(id).digest('hex');
|
|
287
|
-
return `${hash.slice(0, 8)}-${hash.slice(8, 12)}-${hash.slice(12, 16)}-${hash.slice(16, 20)}-${hash.slice(20, 32)}`;
|
|
288
|
-
}
|
|
289
|
-
|
|
290
|
-
async disconnect(): Promise<void> { }
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
@@ -1,105 +0,0 @@
|
|
|
1
|
-
import axios, { AxiosInstance } from 'axios';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* RedisProvider — Redis Vector Search implementation using Upstash REST API.
|
|
8
|
-
*
|
|
9
|
-
* Required options:
|
|
10
|
-
* { baseUrl: string } — Upstash REST endpoint
|
|
11
|
-
* OR { url: string } — alias for baseUrl
|
|
12
|
-
* OR { host: string, port: number } — auto-constructs baseUrl
|
|
13
|
-
*
|
|
14
|
-
* Optional: { apiKey?: string } — bearer token (required for Upstash)
|
|
15
|
-
*
|
|
16
|
-
* Note: This provider targets the Upstash Vector REST API.
|
|
17
|
-
* For bare Redis with RediSearch, use a custom REST wrapper or the
|
|
18
|
-
* UniversalVectorProvider with custom templates.
|
|
19
|
-
*/
|
|
20
|
-
export class RedisProvider extends BaseVectorProvider {
|
|
21
|
-
private http: AxiosInstance;
|
|
22
|
-
|
|
23
|
-
constructor(config: VectorDBConfig) {
|
|
24
|
-
super(config);
|
|
25
|
-
const opts = config.options as Record<string, unknown>;
|
|
26
|
-
|
|
27
|
-
// Accept baseUrl, url, or host+port
|
|
28
|
-
const baseUrl =
|
|
29
|
-
(opts.baseUrl as string) ||
|
|
30
|
-
(opts.url as string) ||
|
|
31
|
-
(opts.host ? `http://${opts.host}:${(opts.port as number) || 6379}` : undefined);
|
|
32
|
-
|
|
33
|
-
if (!baseUrl) throw new Error('[RedisProvider] options.baseUrl is required');
|
|
34
|
-
|
|
35
|
-
this.http = axios.create({
|
|
36
|
-
baseURL: baseUrl,
|
|
37
|
-
headers: {
|
|
38
|
-
'Content-Type': 'application/json',
|
|
39
|
-
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey as string}` } : {}),
|
|
40
|
-
},
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
async initialize(): Promise<void> {
|
|
45
|
-
await this.ping();
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
49
|
-
const key = namespace ? `${namespace}:${doc.id}` : String(doc.id);
|
|
50
|
-
await this.http.post('/', ['JSON.SET', key, '$', JSON.stringify({
|
|
51
|
-
vector: doc.vector,
|
|
52
|
-
content: doc.content,
|
|
53
|
-
metadata: doc.metadata || {},
|
|
54
|
-
})]);
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
58
|
-
// Sequential upsert — Upstash REST doesn't support pipeline in a single call
|
|
59
|
-
for (const doc of docs) {
|
|
60
|
-
await this.upsert(doc, namespace);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
65
|
-
const payload = {
|
|
66
|
-
index: this.indexName,
|
|
67
|
-
vector: vector,
|
|
68
|
-
topK: topK,
|
|
69
|
-
namespace: namespace,
|
|
70
|
-
};
|
|
71
|
-
const { data } = await this.http.post('/search', payload);
|
|
72
|
-
return (data.results || []).map((res: Record<string, unknown>) => ({
|
|
73
|
-
id: res['id'] as string,
|
|
74
|
-
score: res['score'] as number,
|
|
75
|
-
content: res['content'] as string,
|
|
76
|
-
metadata: (res['metadata'] as Record<string, unknown>) || {},
|
|
77
|
-
}));
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
async delete(id: string, namespace?: string): Promise<void> {
|
|
81
|
-
const key = namespace ? `${namespace}:${id}` : id;
|
|
82
|
-
await this.http.post('/', ['DEL', key]);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
86
|
-
// Full namespace deletion requires SCAN — not trivially supported in REST API.
|
|
87
|
-
// Log a warning; callers should implement namespace-aware cleanup themselves.
|
|
88
|
-
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
/**
|
|
92
|
-
* Check reachability via a PING command.
|
|
93
|
-
* Returns false on connection failure so health checks surface real problems.
|
|
94
|
-
*/
|
|
95
|
-
async ping(): Promise<boolean> {
|
|
96
|
-
try {
|
|
97
|
-
await this.http.post('/', ['PING']);
|
|
98
|
-
return true;
|
|
99
|
-
} catch {
|
|
100
|
-
return false;
|
|
101
|
-
}
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
async disconnect(): Promise<void> {}
|
|
105
|
-
}
|
|
@@ -1,221 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* UniversalVectorProvider — Template-based REST integration for any vector database
|
|
3
|
-
*
|
|
4
|
-
* Enables connecting to any REST-based vector database with minimal configuration
|
|
5
|
-
* through request/response mapping templates.
|
|
6
|
-
*
|
|
7
|
-
* Features:
|
|
8
|
-
* - Template-based request/response mapping
|
|
9
|
-
* - Automatic JSON path extraction
|
|
10
|
-
* - Retry logic with exponential backoff
|
|
11
|
-
* - Request batching
|
|
12
|
-
* - Custom header support
|
|
13
|
-
*/
|
|
14
|
-
|
|
15
|
-
import axios, { AxiosInstance } from 'axios';
|
|
16
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
17
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
18
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
19
|
-
import { resolvePath, buildPayload } from '../../utils/templateUtils';
|
|
20
|
-
|
|
21
|
-
export interface UniversalVectorConfig extends Record<string, unknown> {
|
|
22
|
-
baseUrl: string;
|
|
23
|
-
headers?: Record<string, string>;
|
|
24
|
-
timeout?: number;
|
|
25
|
-
|
|
26
|
-
// Endpoint mappings
|
|
27
|
-
queryEndpoint?: string;
|
|
28
|
-
upsertEndpoint?: string;
|
|
29
|
-
deleteEndpoint?: string;
|
|
30
|
-
deleteNamespaceEndpoint?: string;
|
|
31
|
-
pingEndpoint?: string;
|
|
32
|
-
|
|
33
|
-
// Request templates (e.g., '{"vector": {{vector}}, "limit": {{topK}}}')
|
|
34
|
-
queryTemplate?: string;
|
|
35
|
-
upsertTemplate?: string;
|
|
36
|
-
deleteTemplate?: string;
|
|
37
|
-
|
|
38
|
-
// Response field mappings (JSONPath-like format)
|
|
39
|
-
queryResponsePath?: string; // e.g., 'data.matches' or 'results'
|
|
40
|
-
queryIdField?: string; // e.g., '_id' or 'id'
|
|
41
|
-
queryScoreField?: string; // e.g., 'score' or 'similarity'
|
|
42
|
-
queryContentField?: string; // e.g., 'text' or 'content'
|
|
43
|
-
queryMetadataField?: string; // e.g., 'metadata'
|
|
44
|
-
|
|
45
|
-
upsertSuccessStatusCode?: number; // e.g., 200, 201
|
|
46
|
-
upsertResponsePath?: string; // e.g., 'result.status'
|
|
47
|
-
|
|
48
|
-
// Retry configuration
|
|
49
|
-
maxRetries?: number;
|
|
50
|
-
retryDelayMs?: number;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
export class UniversalVectorProvider extends BaseVectorProvider {
|
|
54
|
-
private http!: AxiosInstance;
|
|
55
|
-
private readonly opts: UniversalVectorConfig;
|
|
56
|
-
|
|
57
|
-
constructor(config: VectorDBConfig) {
|
|
58
|
-
super(config);
|
|
59
|
-
this.opts = config.options as UniversalVectorConfig;
|
|
60
|
-
|
|
61
|
-
if (!this.opts.baseUrl) {
|
|
62
|
-
throw new Error('[UniversalVectorProvider] options.baseUrl is required');
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
async initialize(): Promise<void> {
|
|
67
|
-
this.http = axios.create({
|
|
68
|
-
baseURL: this.opts.baseUrl,
|
|
69
|
-
headers: {
|
|
70
|
-
'Content-Type': 'application/json',
|
|
71
|
-
...this.opts.headers,
|
|
72
|
-
},
|
|
73
|
-
timeout: this.opts.timeout ?? 30000,
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
// Test connectivity
|
|
77
|
-
if (!(await this.ping())) {
|
|
78
|
-
throw new Error(`[UniversalVectorProvider] Failed to connect to ${this.opts.baseUrl}`);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
83
|
-
const endpoint = this.opts.upsertEndpoint ?? '/upsert';
|
|
84
|
-
const template = this.opts.upsertTemplate ?? JSON.stringify({
|
|
85
|
-
id: '{{id}}',
|
|
86
|
-
vector: '{{vector}}',
|
|
87
|
-
content: '{{content}}',
|
|
88
|
-
namespace: '{{namespace}}',
|
|
89
|
-
metadata: '{{metadata}}',
|
|
90
|
-
});
|
|
91
|
-
|
|
92
|
-
const payload = buildPayload(template, {
|
|
93
|
-
id: doc.id,
|
|
94
|
-
vector: doc.vector,
|
|
95
|
-
content: doc.content,
|
|
96
|
-
namespace: namespace ?? this.indexName,
|
|
97
|
-
metadata: doc.metadata ?? {},
|
|
98
|
-
});
|
|
99
|
-
|
|
100
|
-
try {
|
|
101
|
-
await this.http.post(endpoint, payload);
|
|
102
|
-
} catch (error) {
|
|
103
|
-
throw new Error(
|
|
104
|
-
`[UniversalVectorProvider] Upsert failed: ${error instanceof Error ? error.message : String(error)}`
|
|
105
|
-
);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
110
|
-
const batchSize = 50;
|
|
111
|
-
|
|
112
|
-
// Simple batching without retry logic
|
|
113
|
-
for (let i = 0; i < docs.length; i += batchSize) {
|
|
114
|
-
const batch = docs.slice(i, i + batchSize);
|
|
115
|
-
await Promise.all(
|
|
116
|
-
batch.map((doc: UpsertDocument) => this.upsert(doc, namespace))
|
|
117
|
-
);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
|
|
121
|
-
async query(
|
|
122
|
-
vector: number[],
|
|
123
|
-
topK: number,
|
|
124
|
-
namespace?: string,
|
|
125
|
-
filter?: Record<string, unknown>
|
|
126
|
-
): Promise<VectorMatch[]> {
|
|
127
|
-
const endpoint = this.opts.queryEndpoint ?? '/query';
|
|
128
|
-
const template = this.opts.queryTemplate ?? JSON.stringify({
|
|
129
|
-
vector: '{{vector}}',
|
|
130
|
-
limit: '{{topK}}',
|
|
131
|
-
namespace: '{{namespace}}',
|
|
132
|
-
filter: '{{filter}}',
|
|
133
|
-
});
|
|
134
|
-
|
|
135
|
-
const sanitizedFilter = this.sanitizeFilter(filter);
|
|
136
|
-
const payload = buildPayload(template, {
|
|
137
|
-
vector,
|
|
138
|
-
topK,
|
|
139
|
-
namespace: namespace ?? this.indexName,
|
|
140
|
-
filter: sanitizedFilter,
|
|
141
|
-
});
|
|
142
|
-
|
|
143
|
-
try {
|
|
144
|
-
const response = await this.http.post(endpoint, payload);
|
|
145
|
-
|
|
146
|
-
// Extract results using response path
|
|
147
|
-
let results = response.data;
|
|
148
|
-
if (this.opts.queryResponsePath) {
|
|
149
|
-
results = resolvePath(response.data, this.opts.queryResponsePath);
|
|
150
|
-
}
|
|
151
|
-
|
|
152
|
-
if (!Array.isArray(results)) {
|
|
153
|
-
throw new Error(
|
|
154
|
-
`[UniversalVectorProvider] Expected array response at "${this.opts.queryResponsePath}", got ${typeof results}`
|
|
155
|
-
);
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
// Map results to VectorMatch format
|
|
159
|
-
return results.map((item: Record<string, unknown>) => ({
|
|
160
|
-
id: item[this.opts.queryIdField ?? 'id'] as string | number,
|
|
161
|
-
score: (item[this.opts.queryScoreField ?? 'score'] as number) ?? 0,
|
|
162
|
-
content: (item[this.opts.queryContentField ?? 'content'] as string) ?? '',
|
|
163
|
-
metadata: (item[this.opts.queryMetadataField ?? 'metadata'] ?? {}) as Record<
|
|
164
|
-
string,
|
|
165
|
-
unknown
|
|
166
|
-
>,
|
|
167
|
-
}));
|
|
168
|
-
} catch (error) {
|
|
169
|
-
throw new Error(
|
|
170
|
-
`[UniversalVectorProvider] Query failed: ${error instanceof Error ? error.message : String(error)}`
|
|
171
|
-
);
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
async delete(id: string | number, namespace?: string): Promise<void> {
|
|
176
|
-
const endpoint = this.opts.deleteEndpoint ?? '/delete';
|
|
177
|
-
const template = this.opts.deleteTemplate ?? JSON.stringify({
|
|
178
|
-
id: '{{id}}',
|
|
179
|
-
namespace: '{{namespace}}',
|
|
180
|
-
});
|
|
181
|
-
|
|
182
|
-
const payload = buildPayload(template, {
|
|
183
|
-
id,
|
|
184
|
-
namespace: namespace ?? this.indexName,
|
|
185
|
-
});
|
|
186
|
-
|
|
187
|
-
try {
|
|
188
|
-
await this.http.post(endpoint, payload);
|
|
189
|
-
} catch (error) {
|
|
190
|
-
throw new Error(
|
|
191
|
-
`[UniversalVectorProvider] Delete failed: ${error instanceof Error ? error.message : String(error)}`
|
|
192
|
-
);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
197
|
-
const endpoint = this.opts.deleteNamespaceEndpoint ?? '/delete-namespace';
|
|
198
|
-
|
|
199
|
-
try {
|
|
200
|
-
await this.http.post(endpoint, { namespace });
|
|
201
|
-
} catch (error) {
|
|
202
|
-
throw new Error(
|
|
203
|
-
`[UniversalVectorProvider] DeleteNamespace failed: ${error instanceof Error ? error.message : String(error)}`
|
|
204
|
-
);
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
|
|
208
|
-
async ping(): Promise<boolean> {
|
|
209
|
-
try {
|
|
210
|
-
const endpoint = this.opts.pingEndpoint ?? '/health';
|
|
211
|
-
const response = await this.http.get(endpoint, { timeout: 5000 });
|
|
212
|
-
return response.status >= 200 && response.status < 300;
|
|
213
|
-
} catch {
|
|
214
|
-
return false;
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
async disconnect(): Promise<void> {
|
|
219
|
-
// Axios doesn't require explicit cleanup
|
|
220
|
-
}
|
|
221
|
-
}
|