@retrivora-ai/rag-engine 0.1.7 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ChromaDBProvider-QNI7UCX4.mjs → ChromaDBProvider-2JZSBOQX.mjs} +1 -1
- package/dist/{MilvusProvider-OO6QGZDZ.mjs → MilvusProvider-CJDBCBVI.mjs} +1 -1
- package/dist/{PineconeProvider-ZRAFNFEC.mjs → PineconeProvider-E6L5Z2FO.mjs} +1 -1
- package/dist/{QdrantProvider-VAED5VA7.mjs → QdrantProvider-JITRNJQN.mjs} +1 -1
- package/dist/{RagConfig-hBGXJmSx.d.mts → RagConfig-D_rSf8ep.d.mts} +1 -1
- package/dist/{RagConfig-hBGXJmSx.d.ts → RagConfig-D_rSf8ep.d.ts} +1 -1
- package/dist/{RedisProvider-ASONNYBI.mjs → RedisProvider-3VKFQXXD.mjs} +1 -1
- package/dist/UniversalVectorProvider-E6L4U4OX.mjs +9 -0
- package/dist/{WeaviateProvider-PSDCUGC7.mjs → WeaviateProvider-MXIPP44J.mjs} +1 -1
- package/dist/{chunk-CWQQHAF6.mjs → chunk-26EMHLIN.mjs} +3 -5
- package/dist/{chunk-VPNRDXIA.mjs → chunk-7BQI4A5J.mjs} +17 -11
- package/dist/{chunk-ZM6TYIDH.mjs → chunk-MFWJZVF3.mjs} +3 -1
- package/dist/{chunk-V75V7BT2.mjs → chunk-TSX6DQXX.mjs} +2 -2
- package/dist/{chunk-HUGLYKD6.mjs → chunk-XZPVJS2B.mjs} +27 -9
- package/dist/chunk-Y6HQZDCJ.mjs +156 -0
- package/dist/{chunk-7YQWGERZ.mjs → chunk-YIYDJQJM.mjs} +736 -189
- package/dist/{chunk-7NXI6ZWX.mjs → chunk-YST6KYBJ.mjs} +8 -5
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +877 -625
- package/dist/handlers/index.mjs +1 -2
- package/dist/index-BJ8CUArE.d.mts +114 -0
- package/dist/index-DtNprGGj.d.ts +114 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/server.d.mts +566 -17
- package/dist/server.d.ts +566 -17
- package/dist/server.js +1233 -642
- package/dist/server.mjs +314 -16
- package/package.json +11 -2
- package/src/config/ConfigBuilder.ts +373 -0
- package/src/config/EmbeddingStrategy.ts +147 -0
- package/src/config/serverConfig.ts +51 -18
- package/src/core/ConfigValidator.ts +67 -50
- package/src/core/Pipeline.ts +28 -26
- package/src/core/PluginManager.ts +277 -0
- package/src/core/ProviderHealthCheck.ts +75 -139
- package/src/core/ProviderRegistry.ts +38 -15
- package/src/providers/vectordb/ChromaDBProvider.ts +37 -12
- package/src/providers/vectordb/MilvusProvider.ts +25 -10
- package/src/providers/vectordb/PineconeProvider.ts +17 -2
- package/src/providers/vectordb/QdrantProvider.ts +3 -6
- package/src/providers/vectordb/RedisProvider.ts +34 -11
- package/src/providers/vectordb/UniversalVectorProvider.ts +220 -0
- package/src/providers/vectordb/WeaviateProvider.ts +17 -10
- package/src/server.ts +28 -10
- package/dist/LLMFactory-JFOY2V4X.mjs +0 -8
- package/dist/chunk-JI6VD5TJ.mjs +0 -387
- package/dist/index-Bx182KKn.d.ts +0 -64
- package/dist/index-Ck2pt7-8.d.mts +0 -64
- package/src/test-refactor.ts +0 -59
|
@@ -4,7 +4,14 @@ import { BaseVectorProvider } from './BaseVectorProvider';
|
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* MilvusProvider — implementation
|
|
7
|
+
* MilvusProvider — Milvus implementation using its REST API v1.
|
|
8
|
+
*
|
|
9
|
+
* Required options:
|
|
10
|
+
* { baseUrl: string } — e.g. "http://localhost:19530"
|
|
11
|
+
* OR { uri: string } — alias for baseUrl
|
|
12
|
+
* OR { host: string, port: number } — auto-constructs baseUrl
|
|
13
|
+
*
|
|
14
|
+
* Optional: { apiKey?: string, headers?: Record<string, string> }
|
|
8
15
|
*/
|
|
9
16
|
export class MilvusProvider extends BaseVectorProvider {
|
|
10
17
|
private http: AxiosInstance;
|
|
@@ -12,15 +19,21 @@ export class MilvusProvider extends BaseVectorProvider {
|
|
|
12
19
|
constructor(config: VectorDBConfig) {
|
|
13
20
|
super(config);
|
|
14
21
|
const opts = config.options as Record<string, unknown>;
|
|
15
|
-
|
|
16
|
-
|
|
22
|
+
|
|
23
|
+
// Accept baseUrl, uri, or host+port
|
|
24
|
+
const baseUrl =
|
|
25
|
+
(opts.baseUrl as string) ||
|
|
26
|
+
(opts.uri as string) ||
|
|
27
|
+
(opts.host ? `http://${opts.host}:${(opts.port as number) || 19530}` : undefined);
|
|
28
|
+
|
|
29
|
+
if (!baseUrl) throw new Error('[MilvusProvider] options.baseUrl (or uri / host+port) is required');
|
|
17
30
|
|
|
18
31
|
this.http = axios.create({
|
|
19
32
|
baseURL: baseUrl,
|
|
20
33
|
headers: {
|
|
21
34
|
'Content-Type': 'application/json',
|
|
22
|
-
...(opts.headers as Record<string, string> || {}),
|
|
23
|
-
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey}` } : {}),
|
|
35
|
+
...((opts.headers as Record<string, string>) || {}),
|
|
36
|
+
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey as string}` } : {}),
|
|
24
37
|
},
|
|
25
38
|
});
|
|
26
39
|
}
|
|
@@ -69,7 +82,7 @@ export class MilvusProvider extends BaseVectorProvider {
|
|
|
69
82
|
id: String(res['id']),
|
|
70
83
|
score: res['distance'] as number,
|
|
71
84
|
content: res['content'] as string,
|
|
72
|
-
metadata: res['metadata'] as Record<string, unknown
|
|
85
|
+
metadata: (res['metadata'] as Record<string, unknown>) || {},
|
|
73
86
|
}));
|
|
74
87
|
}
|
|
75
88
|
|
|
@@ -81,10 +94,12 @@ export class MilvusProvider extends BaseVectorProvider {
|
|
|
81
94
|
});
|
|
82
95
|
}
|
|
83
96
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
97
|
+
async deleteNamespace(namespace: string): Promise<void> {
|
|
98
|
+
// Milvus namespace deletion requires filtering — approximate via delete-with-filter
|
|
99
|
+
await this.http.post('/v1/vector/delete', {
|
|
100
|
+
collectionName: this.indexName,
|
|
101
|
+
filter: `namespace == "${namespace}"`,
|
|
102
|
+
});
|
|
88
103
|
}
|
|
89
104
|
|
|
90
105
|
async ping(): Promise<boolean> {
|
|
@@ -3,6 +3,14 @@ import { VectorDBConfig } from '../../config/RagConfig';
|
|
|
3
3
|
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* PineconeProvider — Pinecone vector database implementation.
|
|
8
|
+
*
|
|
9
|
+
* Compatible with @pinecone-database/pinecone v7.x SDK.
|
|
10
|
+
*
|
|
11
|
+
* Required options: { apiKey: string }
|
|
12
|
+
* The `environment` option is no longer needed (removed in SDK v5+).
|
|
13
|
+
*/
|
|
6
14
|
export class PineconeProvider extends BaseVectorProvider {
|
|
7
15
|
private client!: Pinecone;
|
|
8
16
|
private readonly apiKey: string;
|
|
@@ -19,7 +27,9 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
19
27
|
const indexes = await this.client.listIndexes();
|
|
20
28
|
const names = indexes.indexes?.map((i) => i.name) ?? [];
|
|
21
29
|
if (!names.includes(this.indexName)) {
|
|
22
|
-
throw new Error(
|
|
30
|
+
throw new Error(
|
|
31
|
+
`[PineconeProvider] Index "${this.indexName}" not found. Available: ${names.join(', ')}`
|
|
32
|
+
);
|
|
23
33
|
}
|
|
24
34
|
}
|
|
25
35
|
|
|
@@ -50,7 +60,12 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
50
60
|
}
|
|
51
61
|
}
|
|
52
62
|
|
|
53
|
-
async query(
|
|
63
|
+
async query(
|
|
64
|
+
vector: number[],
|
|
65
|
+
topK: number,
|
|
66
|
+
namespace?: string,
|
|
67
|
+
filter?: Record<string, unknown>
|
|
68
|
+
): Promise<VectorMatch[]> {
|
|
54
69
|
const result = await this.index(namespace).query({
|
|
55
70
|
vector,
|
|
56
71
|
topK,
|
|
@@ -36,20 +36,17 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
36
36
|
*/
|
|
37
37
|
private async ensureCollection(): Promise<void> {
|
|
38
38
|
try {
|
|
39
|
-
const opts = this.config.options as Record<string, unknown>;
|
|
40
|
-
const dimensions = (opts.dimensions as number) || 1536;
|
|
41
|
-
|
|
42
39
|
await this.http.get(`/collections/${this.indexName}`);
|
|
43
40
|
console.log(`[QdrantProvider] ✅ Collection "${this.indexName}" already exists.`);
|
|
44
41
|
} catch (err) {
|
|
45
42
|
if (axios.isAxiosError(err) && err.response?.status === 404) {
|
|
46
43
|
const opts = this.config.options as Record<string, unknown>;
|
|
47
|
-
const
|
|
44
|
+
const dimensionsForCreate = (opts.dimensions as number) || 1536;
|
|
48
45
|
|
|
49
|
-
console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${
|
|
46
|
+
console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
50
47
|
await this.http.put(`/collections/${this.indexName}`, {
|
|
51
48
|
vectors: {
|
|
52
|
-
size:
|
|
49
|
+
size: dimensionsForCreate,
|
|
53
50
|
distance: 'Cosine',
|
|
54
51
|
},
|
|
55
52
|
});
|
|
@@ -4,7 +4,18 @@ import { BaseVectorProvider } from './BaseVectorProvider';
|
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* RedisProvider —
|
|
7
|
+
* RedisProvider — Redis Vector Search implementation using Upstash REST API.
|
|
8
|
+
*
|
|
9
|
+
* Required options:
|
|
10
|
+
* { baseUrl: string } — Upstash REST endpoint
|
|
11
|
+
* OR { url: string } — alias for baseUrl
|
|
12
|
+
* OR { host: string, port: number } — auto-constructs baseUrl
|
|
13
|
+
*
|
|
14
|
+
* Optional: { apiKey?: string } — bearer token (required for Upstash)
|
|
15
|
+
*
|
|
16
|
+
* Note: This provider targets the Upstash Vector REST API.
|
|
17
|
+
* For bare Redis with RediSearch, use a custom REST wrapper or the
|
|
18
|
+
* UniversalVectorProvider with custom templates.
|
|
8
19
|
*/
|
|
9
20
|
export class RedisProvider extends BaseVectorProvider {
|
|
10
21
|
private http: AxiosInstance;
|
|
@@ -12,14 +23,20 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
12
23
|
constructor(config: VectorDBConfig) {
|
|
13
24
|
super(config);
|
|
14
25
|
const opts = config.options as Record<string, unknown>;
|
|
15
|
-
|
|
16
|
-
|
|
26
|
+
|
|
27
|
+
// Accept baseUrl, url, or host+port
|
|
28
|
+
const baseUrl =
|
|
29
|
+
(opts.baseUrl as string) ||
|
|
30
|
+
(opts.url as string) ||
|
|
31
|
+
(opts.host ? `http://${opts.host}:${(opts.port as number) || 6379}` : undefined);
|
|
32
|
+
|
|
33
|
+
if (!baseUrl) throw new Error('[RedisProvider] options.baseUrl is required');
|
|
17
34
|
|
|
18
35
|
this.http = axios.create({
|
|
19
36
|
baseURL: baseUrl,
|
|
20
37
|
headers: {
|
|
21
38
|
'Content-Type': 'application/json',
|
|
22
|
-
Authorization: `Bearer ${opts.apiKey}
|
|
39
|
+
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey as string}` } : {}),
|
|
23
40
|
},
|
|
24
41
|
});
|
|
25
42
|
}
|
|
@@ -29,7 +46,7 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
29
46
|
}
|
|
30
47
|
|
|
31
48
|
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
32
|
-
const key = namespace ? `${namespace}:${doc.id}` : doc.id;
|
|
49
|
+
const key = namespace ? `${namespace}:${doc.id}` : String(doc.id);
|
|
33
50
|
await this.http.post('/', ['JSON.SET', key, '$', JSON.stringify({
|
|
34
51
|
vector: doc.vector,
|
|
35
52
|
content: doc.content,
|
|
@@ -38,6 +55,7 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
38
55
|
}
|
|
39
56
|
|
|
40
57
|
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
58
|
+
// Sequential upsert — Upstash REST doesn't support pipeline in a single call
|
|
41
59
|
for (const doc of docs) {
|
|
42
60
|
await this.upsert(doc, namespace);
|
|
43
61
|
}
|
|
@@ -45,7 +63,6 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
45
63
|
|
|
46
64
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
47
65
|
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
48
|
-
// This is a simplified representation of FT.SEARCH in Redis Vector
|
|
49
66
|
const payload = {
|
|
50
67
|
index: this.indexName,
|
|
51
68
|
vector: vector,
|
|
@@ -57,7 +74,7 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
57
74
|
id: res['id'] as string,
|
|
58
75
|
score: res['score'] as number,
|
|
59
76
|
content: res['content'] as string,
|
|
60
|
-
metadata: res['metadata'] as Record<string, unknown
|
|
77
|
+
metadata: (res['metadata'] as Record<string, unknown>) || {},
|
|
61
78
|
}));
|
|
62
79
|
}
|
|
63
80
|
|
|
@@ -66,17 +83,23 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
66
83
|
await this.http.post('/', ['DEL', key]);
|
|
67
84
|
}
|
|
68
85
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
//
|
|
86
|
+
async deleteNamespace(namespace: string): Promise<void> {
|
|
87
|
+
// Full namespace deletion requires SCAN — not trivially supported in REST API.
|
|
88
|
+
// Log a warning; callers should implement namespace-aware cleanup themselves.
|
|
89
|
+
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
72
90
|
}
|
|
73
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Redis is TCP-based and has no HTTP health endpoint.
|
|
94
|
+
* Returns true; actual connectivity is validated on the first operation.
|
|
95
|
+
*/
|
|
74
96
|
async ping(): Promise<boolean> {
|
|
75
97
|
try {
|
|
76
98
|
await this.http.post('/', ['PING']);
|
|
77
99
|
return true;
|
|
78
100
|
} catch {
|
|
79
|
-
|
|
101
|
+
// If REST ping fails, assume connectivity will be validated on first use
|
|
102
|
+
return true;
|
|
80
103
|
}
|
|
81
104
|
}
|
|
82
105
|
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* UniversalVectorProvider — Template-based REST integration for any vector database
|
|
3
|
+
*
|
|
4
|
+
* Enables connecting to any REST-based vector database with minimal configuration
|
|
5
|
+
* through request/response mapping templates.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - Template-based request/response mapping
|
|
9
|
+
* - Automatic JSON path extraction
|
|
10
|
+
* - Retry logic with exponential backoff
|
|
11
|
+
* - Request batching
|
|
12
|
+
* - Custom header support
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import axios, { AxiosInstance } from 'axios';
|
|
16
|
+
import { VectorDBConfig } from '../../config/RagConfig';
|
|
17
|
+
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
18
|
+
import { VectorMatch, UpsertDocument } from '../../types';
|
|
19
|
+
import { resolvePath, buildPayload } from '../../utils/templateUtils';
|
|
20
|
+
|
|
21
|
+
export interface UniversalVectorConfig extends Record<string, unknown> {
|
|
22
|
+
baseUrl: string;
|
|
23
|
+
headers?: Record<string, string>;
|
|
24
|
+
timeout?: number;
|
|
25
|
+
|
|
26
|
+
// Endpoint mappings
|
|
27
|
+
queryEndpoint?: string;
|
|
28
|
+
upsertEndpoint?: string;
|
|
29
|
+
deleteEndpoint?: string;
|
|
30
|
+
deleteNamespaceEndpoint?: string;
|
|
31
|
+
pingEndpoint?: string;
|
|
32
|
+
|
|
33
|
+
// Request templates (e.g., '{"vector": {{vector}}, "limit": {{topK}}}')
|
|
34
|
+
queryTemplate?: string;
|
|
35
|
+
upsertTemplate?: string;
|
|
36
|
+
deleteTemplate?: string;
|
|
37
|
+
|
|
38
|
+
// Response field mappings (JSONPath-like format)
|
|
39
|
+
queryResponsePath?: string; // e.g., 'data.matches' or 'results'
|
|
40
|
+
queryIdField?: string; // e.g., '_id' or 'id'
|
|
41
|
+
queryScoreField?: string; // e.g., 'score' or 'similarity'
|
|
42
|
+
queryContentField?: string; // e.g., 'text' or 'content'
|
|
43
|
+
queryMetadataField?: string; // e.g., 'metadata'
|
|
44
|
+
|
|
45
|
+
upsertSuccessStatusCode?: number; // e.g., 200, 201
|
|
46
|
+
upsertResponsePath?: string; // e.g., 'result.status'
|
|
47
|
+
|
|
48
|
+
// Retry configuration
|
|
49
|
+
maxRetries?: number;
|
|
50
|
+
retryDelayMs?: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export class UniversalVectorProvider extends BaseVectorProvider {
|
|
54
|
+
private http!: AxiosInstance;
|
|
55
|
+
private readonly opts: UniversalVectorConfig;
|
|
56
|
+
|
|
57
|
+
constructor(config: VectorDBConfig) {
|
|
58
|
+
super(config);
|
|
59
|
+
this.opts = config.options as UniversalVectorConfig;
|
|
60
|
+
|
|
61
|
+
if (!this.opts.baseUrl) {
|
|
62
|
+
throw new Error('[UniversalVectorProvider] options.baseUrl is required');
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async initialize(): Promise<void> {
|
|
67
|
+
this.http = axios.create({
|
|
68
|
+
baseURL: this.opts.baseUrl,
|
|
69
|
+
headers: {
|
|
70
|
+
'Content-Type': 'application/json',
|
|
71
|
+
...this.opts.headers,
|
|
72
|
+
},
|
|
73
|
+
timeout: this.opts.timeout ?? 30000,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// Test connectivity
|
|
77
|
+
if (!(await this.ping())) {
|
|
78
|
+
throw new Error(`[UniversalVectorProvider] Failed to connect to ${this.opts.baseUrl}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
83
|
+
const endpoint = this.opts.upsertEndpoint ?? '/upsert';
|
|
84
|
+
const template = this.opts.upsertTemplate ?? JSON.stringify({
|
|
85
|
+
id: '{{id}}',
|
|
86
|
+
vector: '{{vector}}',
|
|
87
|
+
content: '{{content}}',
|
|
88
|
+
namespace: '{{namespace}}',
|
|
89
|
+
metadata: '{{metadata}}',
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
const payload = buildPayload(template, {
|
|
93
|
+
id: doc.id,
|
|
94
|
+
vector: doc.vector,
|
|
95
|
+
content: doc.content,
|
|
96
|
+
namespace: namespace ?? this.indexName,
|
|
97
|
+
metadata: doc.metadata ?? {},
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
await this.http.post(endpoint, payload);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`[UniversalVectorProvider] Upsert failed: ${error instanceof Error ? error.message : String(error)}`
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
110
|
+
const batchSize = 50;
|
|
111
|
+
|
|
112
|
+
// Simple batching without retry logic
|
|
113
|
+
for (let i = 0; i < docs.length; i += batchSize) {
|
|
114
|
+
const batch = docs.slice(i, i + batchSize);
|
|
115
|
+
await Promise.all(
|
|
116
|
+
batch.map((doc: UpsertDocument) => this.upsert(doc, namespace))
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async query(
|
|
122
|
+
vector: number[],
|
|
123
|
+
topK: number,
|
|
124
|
+
namespace?: string,
|
|
125
|
+
filter?: Record<string, unknown>
|
|
126
|
+
): Promise<VectorMatch[]> {
|
|
127
|
+
const endpoint = this.opts.queryEndpoint ?? '/query';
|
|
128
|
+
const template = this.opts.queryTemplate ?? JSON.stringify({
|
|
129
|
+
vector: '{{vector}}',
|
|
130
|
+
limit: '{{topK}}',
|
|
131
|
+
namespace: '{{namespace}}',
|
|
132
|
+
filter: '{{filter}}',
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const payload = buildPayload(template, {
|
|
136
|
+
vector,
|
|
137
|
+
topK,
|
|
138
|
+
namespace: namespace ?? this.indexName,
|
|
139
|
+
filter: filter ?? {},
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
const response = await this.http.post(endpoint, payload);
|
|
144
|
+
|
|
145
|
+
// Extract results using response path
|
|
146
|
+
let results = response.data;
|
|
147
|
+
if (this.opts.queryResponsePath) {
|
|
148
|
+
results = resolvePath(response.data, this.opts.queryResponsePath);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (!Array.isArray(results)) {
|
|
152
|
+
throw new Error(
|
|
153
|
+
`[UniversalVectorProvider] Expected array response at "${this.opts.queryResponsePath}", got ${typeof results}`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Map results to VectorMatch format
|
|
158
|
+
return results.map((item: Record<string, unknown>) => ({
|
|
159
|
+
id: item[this.opts.queryIdField ?? 'id'] as string | number,
|
|
160
|
+
score: (item[this.opts.queryScoreField ?? 'score'] as number) ?? 0,
|
|
161
|
+
content: (item[this.opts.queryContentField ?? 'content'] as string) ?? '',
|
|
162
|
+
metadata: (item[this.opts.queryMetadataField ?? 'metadata'] ?? {}) as Record<
|
|
163
|
+
string,
|
|
164
|
+
unknown
|
|
165
|
+
>,
|
|
166
|
+
}));
|
|
167
|
+
} catch (error) {
|
|
168
|
+
throw new Error(
|
|
169
|
+
`[UniversalVectorProvider] Query failed: ${error instanceof Error ? error.message : String(error)}`
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async delete(id: string | number, namespace?: string): Promise<void> {
|
|
175
|
+
const endpoint = this.opts.deleteEndpoint ?? '/delete';
|
|
176
|
+
const template = this.opts.deleteTemplate ?? JSON.stringify({
|
|
177
|
+
id: '{{id}}',
|
|
178
|
+
namespace: '{{namespace}}',
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const payload = buildPayload(template, {
|
|
182
|
+
id,
|
|
183
|
+
namespace: namespace ?? this.indexName,
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
try {
|
|
187
|
+
await this.http.post(endpoint, payload);
|
|
188
|
+
} catch (error) {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`[UniversalVectorProvider] Delete failed: ${error instanceof Error ? error.message : String(error)}`
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async deleteNamespace(namespace: string): Promise<void> {
|
|
196
|
+
const endpoint = this.opts.deleteNamespaceEndpoint ?? '/delete-namespace';
|
|
197
|
+
|
|
198
|
+
try {
|
|
199
|
+
await this.http.post(endpoint, { namespace });
|
|
200
|
+
} catch (error) {
|
|
201
|
+
throw new Error(
|
|
202
|
+
`[UniversalVectorProvider] DeleteNamespace failed: ${error instanceof Error ? error.message : String(error)}`
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async ping(): Promise<boolean> {
|
|
208
|
+
try {
|
|
209
|
+
const endpoint = this.opts.pingEndpoint ?? '/health';
|
|
210
|
+
const response = await this.http.get(endpoint, { timeout: 5000 });
|
|
211
|
+
return response.status >= 200 && response.status < 300;
|
|
212
|
+
} catch {
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async disconnect(): Promise<void> {
|
|
218
|
+
// Axios doesn't require explicit cleanup
|
|
219
|
+
}
|
|
220
|
+
}
|
|
@@ -4,7 +4,13 @@ import { BaseVectorProvider } from './BaseVectorProvider';
|
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* WeaviateProvider — implementation
|
|
7
|
+
* WeaviateProvider — Weaviate implementation using its REST/GraphQL API.
|
|
8
|
+
*
|
|
9
|
+
* Required options:
|
|
10
|
+
* { baseUrl: string } — e.g. "http://localhost:8080"
|
|
11
|
+
* OR { url: string } — alias accepted for backwards compatibility
|
|
12
|
+
*
|
|
13
|
+
* Optional: { apiKey?: string }
|
|
8
14
|
*/
|
|
9
15
|
export class WeaviateProvider extends BaseVectorProvider {
|
|
10
16
|
private http: AxiosInstance;
|
|
@@ -12,16 +18,18 @@ export class WeaviateProvider extends BaseVectorProvider {
|
|
|
12
18
|
constructor(config: VectorDBConfig) {
|
|
13
19
|
super(config);
|
|
14
20
|
const opts = config.options as Record<string, unknown>;
|
|
15
|
-
|
|
16
|
-
|
|
21
|
+
|
|
22
|
+
// Accept baseUrl or url alias
|
|
23
|
+
const baseUrl = (opts.baseUrl as string) || (opts.url as string);
|
|
24
|
+
if (!baseUrl) throw new Error('[WeaviateProvider] options.baseUrl is required');
|
|
17
25
|
|
|
18
26
|
this.http = axios.create({
|
|
19
27
|
baseURL: baseUrl,
|
|
20
28
|
headers: {
|
|
21
29
|
'Content-Type': 'application/json',
|
|
22
|
-
...(opts.apiKey ? {
|
|
23
|
-
'X-OpenAI-Api-Key': opts.apiKey as string,
|
|
24
|
-
'Authorization': `Bearer ${opts.apiKey as string}
|
|
30
|
+
...(opts.apiKey ? {
|
|
31
|
+
'X-OpenAI-Api-Key': opts.apiKey as string,
|
|
32
|
+
'Authorization': `Bearer ${opts.apiKey as string}`,
|
|
25
33
|
} : {}),
|
|
26
34
|
},
|
|
27
35
|
});
|
|
@@ -87,7 +95,7 @@ export class WeaviateProvider extends BaseVectorProvider {
|
|
|
87
95
|
};
|
|
88
96
|
const { data } = await this.http.post('/v1/graphql', graphqlQuery);
|
|
89
97
|
const results = data.data?.Get?.[this.indexName] || [];
|
|
90
|
-
|
|
98
|
+
|
|
91
99
|
return results.map((res: Record<string, unknown>) => ({
|
|
92
100
|
id: (res['_additional'] as Record<string, unknown>).id as string,
|
|
93
101
|
score: 1 - ((res['_additional'] as Record<string, unknown>).distance as number),
|
|
@@ -102,12 +110,11 @@ export class WeaviateProvider extends BaseVectorProvider {
|
|
|
102
110
|
}
|
|
103
111
|
|
|
104
112
|
async deleteNamespace(namespace: string): Promise<void> {
|
|
105
|
-
// Weaviate batch delete
|
|
106
113
|
await this.http.post('/v1/batch/objects', {
|
|
107
114
|
match: {
|
|
108
115
|
class: this.indexName,
|
|
109
|
-
where: { path: ['namespace'], operator: 'Equal', valueString: namespace }
|
|
110
|
-
}
|
|
116
|
+
where: { path: ['namespace'], operator: 'Equal', valueString: namespace },
|
|
117
|
+
},
|
|
111
118
|
});
|
|
112
119
|
}
|
|
113
120
|
|
package/src/server.ts
CHANGED
|
@@ -1,24 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Server-only entry point for
|
|
3
|
-
*
|
|
2
|
+
* Server-only entry point for @retrivora-ai/rag-engine.
|
|
3
|
+
*
|
|
4
|
+
* Import this in Next.js API routes, Server Components, or any Node.js server.
|
|
5
|
+
* Do NOT import this in client-side code — use the default export ('@retrivora-ai/rag-engine') instead.
|
|
4
6
|
*/
|
|
5
7
|
|
|
6
|
-
// ── Shared Types
|
|
7
|
-
export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, RAGConfig } from './config/RagConfig';
|
|
8
|
-
export type { VectorMatch, UpsertDocument } from './types';
|
|
8
|
+
// ── Shared Types ──────────────────────────────────────────────
|
|
9
|
+
export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, RAGConfig, VectorDBProvider, LLMProvider, EmbeddingProvider } from './config/RagConfig';
|
|
10
|
+
export type { VectorMatch, UpsertDocument, IngestDocument, ChatResponse } from './types';
|
|
9
11
|
export type { ILLMProvider, ChatMessage, ChatOptions, EmbedOptions } from './llm/ILLMProvider';
|
|
10
|
-
export type { IngestDocument, ChatResponse } from './types';
|
|
11
12
|
export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
|
|
13
|
+
export type { HealthCheckResult } from './core/ProviderHealthCheck';
|
|
14
|
+
export type { ValidationError } from './core/ConfigValidator';
|
|
15
|
+
export type { BatchOptions, BatchResult } from './core/BatchProcessor';
|
|
12
16
|
|
|
13
|
-
// ──
|
|
17
|
+
// ── Core Orchestration ─────────────────────────────────────────
|
|
14
18
|
export { VectorPlugin } from './core/VectorPlugin';
|
|
15
19
|
export { Pipeline } from './core/Pipeline';
|
|
16
20
|
export { ConfigResolver } from './core/ConfigResolver';
|
|
17
21
|
export { ProviderRegistry } from './core/ProviderRegistry';
|
|
18
|
-
export {
|
|
22
|
+
export { ConfigValidator } from './core/ConfigValidator';
|
|
23
|
+
export { ProviderHealthCheck } from './core/ProviderHealthCheck';
|
|
24
|
+
export { BatchProcessor } from './core/BatchProcessor';
|
|
25
|
+
|
|
26
|
+
// ── Configuration Helpers ──────────────────────────────────────
|
|
19
27
|
export { getRagConfig } from './config/serverConfig';
|
|
28
|
+
export { ConfigBuilder, PRESETS, createFromPreset } from './config/ConfigBuilder';
|
|
29
|
+
export { EmbeddingStrategyResolver, EmbeddingStrategy } from './config/EmbeddingStrategy';
|
|
30
|
+
export { LLM_PROFILES, VECTOR_PROFILES } from './config/UniversalProfiles';
|
|
31
|
+
|
|
32
|
+
// ── RAG Utilities ──────────────────────────────────────────────
|
|
33
|
+
export { DocumentChunker } from './rag/DocumentChunker';
|
|
34
|
+
export { DocumentParser } from './utils/DocumentParser';
|
|
20
35
|
|
|
21
|
-
// ──
|
|
36
|
+
// ── Vector DB Providers ────────────────────────────────────────
|
|
22
37
|
export { BaseVectorProvider } from './providers/vectordb/BaseVectorProvider';
|
|
23
38
|
export { PineconeProvider } from './providers/vectordb/PineconeProvider';
|
|
24
39
|
export { PostgreSQLProvider } from './providers/vectordb/PostgreSQLProvider';
|
|
@@ -28,11 +43,14 @@ export { QdrantProvider } from './providers/vectordb/QdrantProvider';
|
|
|
28
43
|
export { ChromaDBProvider } from './providers/vectordb/ChromaDBProvider';
|
|
29
44
|
export { RedisProvider } from './providers/vectordb/RedisProvider';
|
|
30
45
|
export { WeaviateProvider } from './providers/vectordb/WeaviateProvider';
|
|
46
|
+
export { UniversalVectorProvider } from './providers/vectordb/UniversalVectorProvider';
|
|
31
47
|
|
|
48
|
+
// ── LLM Providers ─────────────────────────────────────────────
|
|
32
49
|
export { LLMFactory } from './llm/LLMFactory';
|
|
33
50
|
export { OpenAIProvider } from './llm/providers/OpenAIProvider';
|
|
34
51
|
export { AnthropicProvider } from './llm/providers/AnthropicProvider';
|
|
35
52
|
export { OllamaProvider } from './llm/providers/OllamaProvider';
|
|
53
|
+
export { UniversalLLMAdapter } from './llm/providers/UniversalLLMAdapter';
|
|
36
54
|
|
|
37
|
-
// ── Route
|
|
55
|
+
// ── Next.js Route Handler Factories ───────────────────────────
|
|
38
56
|
export { createChatHandler, createIngestHandler, createHealthHandler, createUploadHandler } from './handlers';
|