@retrivora-ai/rag-engine 2.4.3 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +27 -4
- package/dist/handlers/index.mjs +27 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +12 -10
- package/dist/index.mjs +12 -10
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +28 -4
- package/dist/server.mjs +28 -4
- package/package.json +1 -3
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,295 +0,0 @@
|
|
|
1
|
-
import { Pinecone, RecordMetadata } from '@pinecone-database/pinecone';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
6
|
-
import { ValidationError } from '../../core/ConfigValidator';
|
|
7
|
-
import { ConfigFetcher } from '../../core/ConfigFetcher';
|
|
8
|
-
|
|
9
|
-
/**
|
|
10
|
-
* PineconeProvider — Pinecone vector database implementation.
|
|
11
|
-
*/
|
|
12
|
-
export class PineconeProvider extends BaseVectorProvider {
|
|
13
|
-
private client!: Pinecone;
|
|
14
|
-
private apiKey: string;
|
|
15
|
-
private licenseKey?: string;
|
|
16
|
-
|
|
17
|
-
constructor(config: VectorDBConfig) {
|
|
18
|
-
super(config);
|
|
19
|
-
const opts = (config.options || {}) as Record<string, string>;
|
|
20
|
-
this.apiKey = opts.apiKey || process.env.PINECONE_API_KEY || '';
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
private getControlPlaneUrl(): string {
|
|
24
|
-
const urls = [
|
|
25
|
-
process.env.RETRIVORA_CONTROL_PLANE_URL,
|
|
26
|
-
process.env.NEXT_PUBLIC_RETRIVORA_CONTROL_PLANE_URL,
|
|
27
|
-
'https://www.retrivora.com'
|
|
28
|
-
].filter((url): url is string => Boolean(url));
|
|
29
|
-
return urls[0].replace(/\/$/, '');
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
static getValidator(): IProviderValidator {
|
|
33
|
-
return {
|
|
34
|
-
validate(config: Record<string, unknown>): ValidationError[] {
|
|
35
|
-
const errors: ValidationError[] = [];
|
|
36
|
-
const opts = (config.options || {}) as Record<string, unknown>;
|
|
37
|
-
const hasLicenseKey = Boolean(
|
|
38
|
-
config.licenseKey ||
|
|
39
|
-
process.env.RETRIVORA_LICENSE_KEY ||
|
|
40
|
-
process.env.NEXT_PUBLIC_RETRIVORA_LICENSE_KEY
|
|
41
|
-
);
|
|
42
|
-
|
|
43
|
-
if (!opts.apiKey && !hasLicenseKey) {
|
|
44
|
-
errors.push({
|
|
45
|
-
field: 'vectorDb.options.apiKey',
|
|
46
|
-
message: 'Pinecone API key is required when RETRIVORA_LICENSE_KEY is not set',
|
|
47
|
-
suggestion: 'Set PINECONE_API_KEY environment variable or supply RETRIVORA_LICENSE_KEY for keyless vector storage',
|
|
48
|
-
severity: 'error',
|
|
49
|
-
});
|
|
50
|
-
}
|
|
51
|
-
return errors;
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
static getHealthChecker(): IProviderHealthChecker {
|
|
57
|
-
return {
|
|
58
|
-
async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
|
|
59
|
-
const opts = (config.options || {}) as Record<string, string>;
|
|
60
|
-
const indexName = config.indexName as string;
|
|
61
|
-
const timestamp = Date.now();
|
|
62
|
-
|
|
63
|
-
try {
|
|
64
|
-
const { Pinecone } = await import('@pinecone-database/pinecone');
|
|
65
|
-
const client = new Pinecone({ apiKey: opts.apiKey });
|
|
66
|
-
const indexes = await client.listIndexes();
|
|
67
|
-
const indexNames = indexes.indexes?.map((i) => i.name) ?? [];
|
|
68
|
-
|
|
69
|
-
if (!indexNames.includes(indexName)) {
|
|
70
|
-
return {
|
|
71
|
-
healthy: false,
|
|
72
|
-
provider: 'pinecone',
|
|
73
|
-
error: `Index "${indexName}" not found. Available: ${indexNames.join(', ')}`,
|
|
74
|
-
timestamp,
|
|
75
|
-
};
|
|
76
|
-
}
|
|
77
|
-
return {
|
|
78
|
-
healthy: true,
|
|
79
|
-
provider: 'pinecone',
|
|
80
|
-
capabilities: { indexes: indexNames.length, targetIndex: indexName },
|
|
81
|
-
timestamp,
|
|
82
|
-
};
|
|
83
|
-
} catch (error) {
|
|
84
|
-
return {
|
|
85
|
-
healthy: false,
|
|
86
|
-
provider: 'pinecone',
|
|
87
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
88
|
-
timestamp,
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
async initialize(): Promise<void> {
|
|
96
|
-
if (this.client) return;
|
|
97
|
-
|
|
98
|
-
let key = this.apiKey || process.env.PINECONE_API_KEY || '';
|
|
99
|
-
|
|
100
|
-
if (!key) {
|
|
101
|
-
const licenseKey = process.env.RETRIVORA_LICENSE_KEY || process.env.NEXT_PUBLIC_RETRIVORA_LICENSE_KEY;
|
|
102
|
-
this.licenseKey = licenseKey;
|
|
103
|
-
const jwtProjectId = (() => {
|
|
104
|
-
if (!licenseKey) return undefined;
|
|
105
|
-
try {
|
|
106
|
-
const parts = licenseKey.split('.');
|
|
107
|
-
if (parts.length >= 2) {
|
|
108
|
-
const payload = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8'));
|
|
109
|
-
return payload?.projectId;
|
|
110
|
-
}
|
|
111
|
-
} catch { /* ignore */ }
|
|
112
|
-
return undefined;
|
|
113
|
-
})();
|
|
114
|
-
|
|
115
|
-
const projectId = process.env.RAG_PROJECT_ID || process.env.NEXT_PUBLIC_RAG_PROJECT_ID || jwtProjectId || this.projectId || 'default';
|
|
116
|
-
this.projectId = projectId;
|
|
117
|
-
|
|
118
|
-
const remoteConfig = await ConfigFetcher.fetchRemoteVectorConfig(projectId, licenseKey);
|
|
119
|
-
|
|
120
|
-
// The config API does NOT return the Pinecone API key (server-only secret).
|
|
121
|
-
// We still use the remote config for indexName and namespace/workspace info.
|
|
122
|
-
if (remoteConfig) {
|
|
123
|
-
if (remoteConfig.apiKey) {
|
|
124
|
-
key = remoteConfig.apiKey;
|
|
125
|
-
this.apiKey = key;
|
|
126
|
-
}
|
|
127
|
-
if (remoteConfig.indexName) {
|
|
128
|
-
this.indexName = remoteConfig.indexName;
|
|
129
|
-
}
|
|
130
|
-
if (remoteConfig.projectId) {
|
|
131
|
-
this.projectId = remoteConfig.projectId;
|
|
132
|
-
}
|
|
133
|
-
if (remoteConfig.namespace) {
|
|
134
|
-
// Store namespace for use in queries — ensures workspace isolation
|
|
135
|
-
(this as any).namespace = remoteConfig.namespace;
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
if (!key) {
|
|
141
|
-
console.warn(
|
|
142
|
-
'[PineconeProvider] Warning: No Pinecone API key available. ' +
|
|
143
|
-
'Set PINECONE_API_KEY in your environment, or ensure the Retrivora Control Plane ' +
|
|
144
|
-
'(https://www.retrivora.com) is reachable for proxy-based vector operations.'
|
|
145
|
-
);
|
|
146
|
-
return;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
this.client = new Pinecone({ apiKey: key });
|
|
150
|
-
try {
|
|
151
|
-
const indexes = await this.client.listIndexes();
|
|
152
|
-
const names = indexes.indexes?.map((i) => i.name) ?? [];
|
|
153
|
-
if (!names.includes(this.indexName)) {
|
|
154
|
-
console.warn(`[PineconeProvider] Target index "${this.indexName}" not listed. Proceeding with configured index name.`);
|
|
155
|
-
}
|
|
156
|
-
} catch (err) {
|
|
157
|
-
console.warn('[PineconeProvider] List index verification warning:', err instanceof Error ? err.message : String(err));
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
|
|
161
|
-
private async getActiveIndex(namespace?: string) {
|
|
162
|
-
if (!this.client) {
|
|
163
|
-
await this.initialize();
|
|
164
|
-
}
|
|
165
|
-
if (!this.client) {
|
|
166
|
-
throw new Error('[PineconeProvider] Cannot execute vector operation: Pinecone client failed to initialize.');
|
|
167
|
-
}
|
|
168
|
-
const idx = this.client.index(this.indexName);
|
|
169
|
-
return namespace ? idx.namespace(namespace) : idx.namespace('');
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
173
|
-
if (!this.client) {
|
|
174
|
-
if (!this.licenseKey) throw new Error('[PineconeProvider] Cannot execute vector operation: Missing API key and License key.');
|
|
175
|
-
const res = await fetch(`${this.getControlPlaneUrl()}/api/v1/vectors/upsert`, {
|
|
176
|
-
method: 'POST',
|
|
177
|
-
headers: { 'Content-Type': 'application/json', 'x-license-key': this.licenseKey },
|
|
178
|
-
body: JSON.stringify({
|
|
179
|
-
projectId: this.projectId,
|
|
180
|
-
indexName: this.indexName,
|
|
181
|
-
namespace: namespace || this.projectId,
|
|
182
|
-
records: [{ id: String(doc.id), values: doc.vector, metadata: { content: doc.content, ...(doc.metadata ?? {}) } }]
|
|
183
|
-
})
|
|
184
|
-
});
|
|
185
|
-
if (!res.ok) throw new Error(`[PineconeProvider] Proxy upsert failed: ${await res.text()}`);
|
|
186
|
-
return;
|
|
187
|
-
}
|
|
188
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
189
|
-
await targetIdx.upsert({
|
|
190
|
-
records: [{
|
|
191
|
-
id: String(doc.id),
|
|
192
|
-
values: doc.vector,
|
|
193
|
-
metadata: { content: doc.content, ...(doc.metadata ?? {}) } as RecordMetadata,
|
|
194
|
-
}],
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
199
|
-
const BATCH = 100;
|
|
200
|
-
if (!this.client) {
|
|
201
|
-
if (!this.licenseKey) throw new Error('[PineconeProvider] Cannot execute vector operation: Missing API key and License key.');
|
|
202
|
-
for (let i = 0; i < docs.length; i += BATCH) {
|
|
203
|
-
const records = docs.slice(i, i + BATCH).map((d) => ({
|
|
204
|
-
id: String(d.id), values: d.vector, metadata: { content: d.content, ...(d.metadata ?? {}) }
|
|
205
|
-
}));
|
|
206
|
-
const res = await fetch(`${this.getControlPlaneUrl()}/api/v1/vectors/upsert`, {
|
|
207
|
-
method: 'POST',
|
|
208
|
-
headers: { 'Content-Type': 'application/json', 'x-license-key': this.licenseKey },
|
|
209
|
-
body: JSON.stringify({
|
|
210
|
-
projectId: this.projectId, indexName: this.indexName, namespace: namespace || this.projectId, records
|
|
211
|
-
})
|
|
212
|
-
});
|
|
213
|
-
if (!res.ok) throw new Error(`[PineconeProvider] Proxy batch upsert failed: ${await res.text()}`);
|
|
214
|
-
}
|
|
215
|
-
return;
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
219
|
-
for (let i = 0; i < docs.length; i += BATCH) {
|
|
220
|
-
const records = docs.slice(i, i + BATCH).map((d) => ({
|
|
221
|
-
id: String(d.id),
|
|
222
|
-
values: d.vector,
|
|
223
|
-
metadata: { content: d.content, ...(d.metadata ?? {}) } as RecordMetadata,
|
|
224
|
-
}));
|
|
225
|
-
await targetIdx.upsert({ records });
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
async query(
|
|
230
|
-
vector: number[],
|
|
231
|
-
topK: number,
|
|
232
|
-
namespace?: string,
|
|
233
|
-
filter?: Record<string, unknown>
|
|
234
|
-
): Promise<VectorMatch[]> {
|
|
235
|
-
const pineconeFilter = this.sanitizeFilter(filter);
|
|
236
|
-
|
|
237
|
-
if (!this.client) {
|
|
238
|
-
if (!this.licenseKey) throw new Error('[PineconeProvider] Cannot execute vector operation: Missing API key and License key.');
|
|
239
|
-
const res = await fetch(`${this.getControlPlaneUrl()}/api/v1/vectors/query`, {
|
|
240
|
-
method: 'POST',
|
|
241
|
-
headers: { 'Content-Type': 'application/json', 'x-license-key': this.licenseKey },
|
|
242
|
-
body: JSON.stringify({
|
|
243
|
-
projectId: this.projectId,
|
|
244
|
-
indexName: this.indexName,
|
|
245
|
-
namespace: namespace || this.projectId,
|
|
246
|
-
vector,
|
|
247
|
-
topK,
|
|
248
|
-
filter: pineconeFilter
|
|
249
|
-
})
|
|
250
|
-
});
|
|
251
|
-
if (!res.ok) throw new Error(`[PineconeProvider] Proxy query failed: ${await res.text()}`);
|
|
252
|
-
const data = await res.json();
|
|
253
|
-
return (data.matches ?? []).map((m: any) => ({
|
|
254
|
-
id: m.id, score: m.score ?? 0, content: (m.metadata?.content || '') as string, metadata: m.metadata as Record<string, unknown>
|
|
255
|
-
}));
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
259
|
-
const result = await targetIdx.query({
|
|
260
|
-
vector,
|
|
261
|
-
topK,
|
|
262
|
-
includeMetadata: true,
|
|
263
|
-
...(Object.keys(pineconeFilter).length > 0 ? { filter: pineconeFilter as Record<string, string | number | boolean | string[] | number[]> } : {}),
|
|
264
|
-
});
|
|
265
|
-
return (result.matches ?? []).map((m) => ({
|
|
266
|
-
id: m.id,
|
|
267
|
-
score: m.score ?? 0,
|
|
268
|
-
content: (m.metadata?.content || '') as string,
|
|
269
|
-
metadata: m.metadata as Record<string, unknown>,
|
|
270
|
-
}));
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
async delete(id: string | number, namespace?: string): Promise<void> {
|
|
274
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
275
|
-
await targetIdx.deleteOne({ id: String(id) });
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
279
|
-
const targetIdx = await this.getActiveIndex(namespace);
|
|
280
|
-
await targetIdx.deleteAll();
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
async ping(): Promise<boolean> {
|
|
284
|
-
try {
|
|
285
|
-
if (!this.client) await this.initialize();
|
|
286
|
-
if (!this.client) return false;
|
|
287
|
-
await this.client.listIndexes();
|
|
288
|
-
return true;
|
|
289
|
-
} catch {
|
|
290
|
-
return false;
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
async disconnect(): Promise<void> {}
|
|
295
|
-
}
|
|
@@ -1,227 +0,0 @@
|
|
|
1
|
-
import { Pool, PoolClient } from 'pg';
|
|
2
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
3
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
|
-
import { IProviderValidator, IProviderHealthChecker, HealthCheckResult } from '../../core/ProviderInterfaces';
|
|
6
|
-
import { ValidationError } from '../../core/ConfigValidator';
|
|
7
|
-
|
|
8
|
-
/**
|
|
9
|
-
* PostgreSQLProvider — PostgreSQL implementation using the pgvector extension.
|
|
10
|
-
*/
|
|
11
|
-
export class PostgreSQLProvider extends BaseVectorProvider {
|
|
12
|
-
private pool!: Pool;
|
|
13
|
-
private readonly dimensions: number;
|
|
14
|
-
private readonly connectionString: string;
|
|
15
|
-
private readonly tableName: string;
|
|
16
|
-
|
|
17
|
-
constructor(config: VectorDBConfig) {
|
|
18
|
-
super(config);
|
|
19
|
-
this.tableName = this.indexName.replace(/[^a-z0-9_]/gi, '_');
|
|
20
|
-
const opts = config.options as Record<string, unknown>;
|
|
21
|
-
if (!opts.connectionString) throw new Error('[PostgreSQLProvider] options.connectionString is required');
|
|
22
|
-
this.connectionString = opts.connectionString as string;
|
|
23
|
-
this.dimensions = (opts.dimensions as number) ?? 1536;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
static getValidator(): IProviderValidator {
|
|
27
|
-
return {
|
|
28
|
-
validate(config: Record<string, unknown>): ValidationError[] {
|
|
29
|
-
const errors: ValidationError[] = [];
|
|
30
|
-
const opts = (config.options || {}) as Record<string, unknown>;
|
|
31
|
-
|
|
32
|
-
if (!opts.connectionString) {
|
|
33
|
-
errors.push({
|
|
34
|
-
field: 'vectorDb.options.connectionString',
|
|
35
|
-
message: 'PostgreSQL connection string is required',
|
|
36
|
-
suggestion: 'Set PGVECTOR_CONNECTION_STRING environment variable',
|
|
37
|
-
severity: 'error',
|
|
38
|
-
});
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
if (opts.tables && typeof opts.tables !== 'string' && !Array.isArray(opts.tables)) {
|
|
42
|
-
errors.push({
|
|
43
|
-
field: 'vectorDb.options.tables',
|
|
44
|
-
message: 'PostgreSQL tables must be a string or a string array',
|
|
45
|
-
severity: 'error',
|
|
46
|
-
});
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
return errors;
|
|
50
|
-
}
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
static getHealthChecker(): IProviderHealthChecker {
|
|
55
|
-
return {
|
|
56
|
-
async check(config: Record<string, unknown>): Promise<HealthCheckResult> {
|
|
57
|
-
const opts = (config.options || {}) as Record<string, string>;
|
|
58
|
-
const timestamp = Date.now();
|
|
59
|
-
try {
|
|
60
|
-
const { Client } = await import('pg');
|
|
61
|
-
const client = new Client({ connectionString: opts.connectionString });
|
|
62
|
-
await client.connect();
|
|
63
|
-
const result = await client.query(`
|
|
64
|
-
SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'vector');
|
|
65
|
-
`);
|
|
66
|
-
const hasVector = result.rows[0].exists;
|
|
67
|
-
await client.end();
|
|
68
|
-
return {
|
|
69
|
-
healthy: true,
|
|
70
|
-
provider: 'postgresql',
|
|
71
|
-
capabilities: { pgvectorInstalled: hasVector },
|
|
72
|
-
timestamp,
|
|
73
|
-
};
|
|
74
|
-
} catch (error) {
|
|
75
|
-
return {
|
|
76
|
-
healthy: false,
|
|
77
|
-
provider: 'postgresql',
|
|
78
|
-
error: `Connection failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
79
|
-
timestamp,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
};
|
|
84
|
-
}
|
|
85
|
-
|
|
86
|
-
async initialize(): Promise<void> {
|
|
87
|
-
this.pool = new Pool({ connectionString: this.connectionString });
|
|
88
|
-
const client: PoolClient = await this.pool.connect();
|
|
89
|
-
try {
|
|
90
|
-
await client.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
91
|
-
await client.query(`
|
|
92
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
93
|
-
id TEXT PRIMARY KEY,
|
|
94
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
95
|
-
content TEXT NOT NULL,
|
|
96
|
-
metadata JSONB,
|
|
97
|
-
embedding VECTOR(${this.dimensions})
|
|
98
|
-
)
|
|
99
|
-
`);
|
|
100
|
-
await client.query(`
|
|
101
|
-
CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
|
|
102
|
-
ON ${this.tableName}
|
|
103
|
-
USING hnsw (embedding vector_cosine_ops)
|
|
104
|
-
`);
|
|
105
|
-
} finally {
|
|
106
|
-
client.release();
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
async upsert(doc: UpsertDocument, namespace = ''): Promise<void> {
|
|
111
|
-
const vectorLiteral = `[${doc.vector.join(',')}]`;
|
|
112
|
-
await this.pool.query(
|
|
113
|
-
`INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
114
|
-
VALUES ($1, $2, $3, $4, $5::vector)
|
|
115
|
-
ON CONFLICT (id) DO UPDATE
|
|
116
|
-
SET namespace = EXCLUDED.namespace,
|
|
117
|
-
content = EXCLUDED.content,
|
|
118
|
-
metadata = EXCLUDED.metadata,
|
|
119
|
-
embedding = EXCLUDED.embedding`,
|
|
120
|
-
[doc.id, namespace, doc.content, JSON.stringify(doc.metadata ?? {}), vectorLiteral]
|
|
121
|
-
);
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
|
|
125
|
-
if (docs.length === 0) return;
|
|
126
|
-
|
|
127
|
-
const client = await this.pool.connect();
|
|
128
|
-
try {
|
|
129
|
-
await client.query('BEGIN');
|
|
130
|
-
|
|
131
|
-
const BATCH_SIZE = 50;
|
|
132
|
-
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
133
|
-
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
134
|
-
const values: unknown[] = [];
|
|
135
|
-
const valuePlaceholders = batch.map((doc, idx) => {
|
|
136
|
-
const offset = idx * 5;
|
|
137
|
-
values.push(doc.id, namespace, doc.content, JSON.stringify(doc.metadata ?? {}), `[${doc.vector.join(',')}]`);
|
|
138
|
-
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
139
|
-
}).join(', ');
|
|
140
|
-
|
|
141
|
-
const query = `
|
|
142
|
-
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
143
|
-
VALUES ${valuePlaceholders}
|
|
144
|
-
ON CONFLICT (id) DO UPDATE
|
|
145
|
-
SET namespace = EXCLUDED.namespace,
|
|
146
|
-
content = EXCLUDED.content,
|
|
147
|
-
metadata = EXCLUDED.metadata,
|
|
148
|
-
embedding = EXCLUDED.embedding
|
|
149
|
-
`;
|
|
150
|
-
await client.query(query, values);
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
await client.query('COMMIT');
|
|
154
|
-
} catch (error) {
|
|
155
|
-
await client.query('ROLLBACK');
|
|
156
|
-
throw error;
|
|
157
|
-
} finally {
|
|
158
|
-
client.release();
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
163
|
-
const vectorLiteral = `[${vector.join(',')}]`;
|
|
164
|
-
let whereClause = namespace ? `WHERE namespace = $3` : '';
|
|
165
|
-
const params: unknown[] = [vectorLiteral, topK];
|
|
166
|
-
if (namespace) params.push(namespace);
|
|
167
|
-
|
|
168
|
-
const publicFilter = this.sanitizeFilter(filter);
|
|
169
|
-
if (Object.keys(publicFilter).length > 0) {
|
|
170
|
-
const filterConditions = Object.entries(publicFilter)
|
|
171
|
-
.map(([key, val]) => {
|
|
172
|
-
const paramIdx = params.length + 1;
|
|
173
|
-
params.push(String(val));
|
|
174
|
-
return `metadata->>'${key}' = $${paramIdx}`;
|
|
175
|
-
})
|
|
176
|
-
.join(' AND ');
|
|
177
|
-
whereClause = whereClause ? `${whereClause} AND ${filterConditions}` : `WHERE ${filterConditions}`;
|
|
178
|
-
}
|
|
179
|
-
|
|
180
|
-
const client = await this.pool.connect();
|
|
181
|
-
try {
|
|
182
|
-
const efSearch = (this.config.options.efSearch as number) || Math.max(topK * 10, 40);
|
|
183
|
-
await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
|
|
184
|
-
|
|
185
|
-
const result = await client.query(
|
|
186
|
-
`SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
|
|
187
|
-
FROM ${this.tableName}
|
|
188
|
-
${whereClause}
|
|
189
|
-
ORDER BY embedding <=> $1::vector
|
|
190
|
-
LIMIT $2`,
|
|
191
|
-
params
|
|
192
|
-
);
|
|
193
|
-
|
|
194
|
-
return result.rows.map((row: Record<string, unknown>) => ({
|
|
195
|
-
id: String(row['id']),
|
|
196
|
-
score: parseFloat(String(row['score'])),
|
|
197
|
-
content: String(row['content']),
|
|
198
|
-
metadata: row['metadata'] as Record<string, unknown>,
|
|
199
|
-
}));
|
|
200
|
-
} finally {
|
|
201
|
-
client.release();
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
async delete(id: string | number, namespace?: string): Promise<void> {
|
|
206
|
-
const where = namespace ? 'WHERE id = $1 AND namespace = $2' : 'WHERE id = $1';
|
|
207
|
-
const params = namespace ? [id, namespace] : [id];
|
|
208
|
-
await this.pool.query(`DELETE FROM ${this.tableName} ${where}`, params);
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
async deleteNamespace(namespace: string): Promise<void> {
|
|
212
|
-
await this.pool.query(`DELETE FROM ${this.tableName} WHERE namespace = $1`, [namespace]);
|
|
213
|
-
}
|
|
214
|
-
|
|
215
|
-
async ping(): Promise<boolean> {
|
|
216
|
-
try {
|
|
217
|
-
await this.pool.query('SELECT 1');
|
|
218
|
-
return true;
|
|
219
|
-
} catch {
|
|
220
|
-
return false;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
|
|
224
|
-
async disconnect(): Promise<void> {
|
|
225
|
-
await this.pool.end();
|
|
226
|
-
}
|
|
227
|
-
}
|