@retrivora-ai/rag-engine 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ChromaDBProvider-QNI7UCX4.mjs → ChromaDBProvider-T7TK3ONZ.mjs} +2 -2
- package/dist/{MilvusProvider-OO6QGZDZ.mjs → MilvusProvider-Y5FV5EAE.mjs} +2 -2
- package/dist/{MongoDBProvider-WWVJG3WT.mjs → MongoDBProvider-QHMGD2LZ.mjs} +2 -2
- package/dist/{PineconeProvider-ZRAFNFEC.mjs → PineconeProvider-A47MRRYJ.mjs} +2 -2
- package/dist/{PostgreSQLProvider-ZNXA67IM.mjs → PostgreSQLProvider-PJ5ER5Z4.mjs} +1 -1
- package/dist/{QdrantProvider-VAED5VA7.mjs → QdrantProvider-OLPJK7CY.mjs} +2 -2
- package/dist/{RagConfig-hBGXJmSx.d.mts → RagConfig-D_rSf8ep.d.mts} +1 -1
- package/dist/{RagConfig-hBGXJmSx.d.ts → RagConfig-D_rSf8ep.d.ts} +1 -1
- package/dist/{RedisProvider-ASONNYBI.mjs → RedisProvider-ANEJ3BHR.mjs} +2 -2
- package/dist/UniversalVectorProvider-QJIV2AJJ.mjs +9 -0
- package/dist/{WeaviateProvider-PSDCUGC7.mjs → WeaviateProvider-WIK2QN23.mjs} +2 -2
- package/dist/{chunk-7YQWGERZ.mjs → chunk-2VR5ZMXV.mjs} +740 -193
- package/dist/{chunk-QEYVWVT5.mjs → chunk-5HXNKSCR.mjs} +1 -1
- package/dist/{chunk-ZM6TYIDH.mjs → chunk-BMHJTWSU.mjs} +4 -2
- package/dist/{chunk-UKDXCXW7.mjs → chunk-EDLTMSNY.mjs} +1 -1
- package/dist/{chunk-I4E63NIC.mjs → chunk-FWCSY2DS.mjs} +14 -1
- package/dist/{chunk-VPNRDXIA.mjs → chunk-HOMXEE3M.mjs} +17 -11
- package/dist/{chunk-V75V7BT2.mjs → chunk-RUKZC3ON.mjs} +3 -3
- package/dist/{chunk-7NXI6ZWX.mjs → chunk-VEJNRS4B.mjs} +9 -6
- package/dist/{chunk-HUGLYKD6.mjs → chunk-VKE5ZW7Y.mjs} +28 -10
- package/dist/chunk-VV2ML6TM.mjs +156 -0
- package/dist/{chunk-CWQQHAF6.mjs → chunk-W2PQR3UK.mjs} +4 -6
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +877 -625
- package/dist/handlers/index.mjs +3 -4
- package/dist/index-BJ8CUArE.d.mts +114 -0
- package/dist/index-DtNprGGj.d.ts +114 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +67 -58
- package/dist/index.mjs +74 -47
- package/dist/server.d.mts +601 -17
- package/dist/server.d.ts +601 -17
- package/dist/server.js +1426 -708
- package/dist/server.mjs +429 -18
- package/package.json +11 -2
- package/src/app/constants.tsx +220 -0
- package/src/app/page.tsx +193 -363
- package/src/app/types.ts +30 -0
- package/src/components/ChatWindow.tsx +3 -11
- package/src/config/ConfigBuilder.ts +373 -0
- package/src/config/EmbeddingStrategy.ts +147 -0
- package/src/config/serverConfig.ts +51 -18
- package/src/core/ConfigValidator.ts +67 -50
- package/src/core/Pipeline.ts +28 -26
- package/src/core/PluginManager.ts +277 -0
- package/src/core/ProviderHealthCheck.ts +75 -139
- package/src/core/ProviderRegistry.ts +38 -15
- package/src/providers/vectordb/ChromaDBProvider.ts +37 -12
- package/src/providers/vectordb/MilvusProvider.ts +25 -10
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +164 -0
- package/src/providers/vectordb/PineconeProvider.ts +17 -2
- package/src/providers/vectordb/QdrantProvider.ts +3 -6
- package/src/providers/vectordb/RedisProvider.ts +34 -11
- package/src/providers/vectordb/UniversalVectorProvider.ts +220 -0
- package/src/providers/vectordb/WeaviateProvider.ts +17 -10
- package/src/server.ts +29 -10
- package/dist/LLMFactory-JFOY2V4X.mjs +0 -8
- package/dist/chunk-JI6VD5TJ.mjs +0 -387
- package/dist/index-Bx182KKn.d.ts +0 -64
- package/dist/index-Ck2pt7-8.d.mts +0 -64
- package/src/test-refactor.ts +0 -59
|
@@ -4,7 +4,14 @@ import { BaseVectorProvider } from './BaseVectorProvider';
|
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* MilvusProvider — implementation
|
|
7
|
+
* MilvusProvider — Milvus implementation using its REST API v1.
|
|
8
|
+
*
|
|
9
|
+
* Required options:
|
|
10
|
+
* { baseUrl: string } — e.g. "http://localhost:19530"
|
|
11
|
+
* OR { uri: string } — alias for baseUrl
|
|
12
|
+
* OR { host: string, port: number } — auto-constructs baseUrl
|
|
13
|
+
*
|
|
14
|
+
* Optional: { apiKey?: string, headers?: Record<string, string> }
|
|
8
15
|
*/
|
|
9
16
|
export class MilvusProvider extends BaseVectorProvider {
|
|
10
17
|
private http: AxiosInstance;
|
|
@@ -12,15 +19,21 @@ export class MilvusProvider extends BaseVectorProvider {
|
|
|
12
19
|
constructor(config: VectorDBConfig) {
|
|
13
20
|
super(config);
|
|
14
21
|
const opts = config.options as Record<string, unknown>;
|
|
15
|
-
|
|
16
|
-
|
|
22
|
+
|
|
23
|
+
// Accept baseUrl, uri, or host+port
|
|
24
|
+
const baseUrl =
|
|
25
|
+
(opts.baseUrl as string) ||
|
|
26
|
+
(opts.uri as string) ||
|
|
27
|
+
(opts.host ? `http://${opts.host}:${(opts.port as number) || 19530}` : undefined);
|
|
28
|
+
|
|
29
|
+
if (!baseUrl) throw new Error('[MilvusProvider] options.baseUrl (or uri / host+port) is required');
|
|
17
30
|
|
|
18
31
|
this.http = axios.create({
|
|
19
32
|
baseURL: baseUrl,
|
|
20
33
|
headers: {
|
|
21
34
|
'Content-Type': 'application/json',
|
|
22
|
-
...(opts.headers as Record<string, string> || {}),
|
|
23
|
-
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey}` } : {}),
|
|
35
|
+
...((opts.headers as Record<string, string>) || {}),
|
|
36
|
+
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey as string}` } : {}),
|
|
24
37
|
},
|
|
25
38
|
});
|
|
26
39
|
}
|
|
@@ -69,7 +82,7 @@ export class MilvusProvider extends BaseVectorProvider {
|
|
|
69
82
|
id: String(res['id']),
|
|
70
83
|
score: res['distance'] as number,
|
|
71
84
|
content: res['content'] as string,
|
|
72
|
-
metadata: res['metadata'] as Record<string, unknown
|
|
85
|
+
metadata: (res['metadata'] as Record<string, unknown>) || {},
|
|
73
86
|
}));
|
|
74
87
|
}
|
|
75
88
|
|
|
@@ -81,10 +94,12 @@ export class MilvusProvider extends BaseVectorProvider {
|
|
|
81
94
|
});
|
|
82
95
|
}
|
|
83
96
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
97
|
+
async deleteNamespace(namespace: string): Promise<void> {
|
|
98
|
+
// Milvus namespace deletion requires filtering — approximate via delete-with-filter
|
|
99
|
+
await this.http.post('/v1/vector/delete', {
|
|
100
|
+
collectionName: this.indexName,
|
|
101
|
+
filter: `namespace == "${namespace}"`,
|
|
102
|
+
});
|
|
88
103
|
}
|
|
89
104
|
|
|
90
105
|
async ping(): Promise<boolean> {
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
import { Pool, PoolClient } from 'pg';
|
|
2
|
+
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
3
|
+
import { VectorMatch, UpsertDocument } from '../../types';
|
|
4
|
+
import { VectorDBConfig } from '../../config/RagConfig';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* MultiTablePostgresProvider — PostgreSQL implementation that searches across
|
|
8
|
+
* multiple existing tables with pre-existing embeddings.
|
|
9
|
+
*
|
|
10
|
+
* Extends BaseVectorProvider so it can be registered with ProviderRegistry.
|
|
11
|
+
* Upsert operations are not supported — data is assumed to be managed externally
|
|
12
|
+
* or via custom ingestion scripts that write directly to each table.
|
|
13
|
+
*/
|
|
14
|
+
export class MultiTablePostgresProvider extends BaseVectorProvider {
|
|
15
|
+
private pool!: Pool;
|
|
16
|
+
private readonly dimensions: number;
|
|
17
|
+
private readonly connectionString: string;
|
|
18
|
+
private readonly tables: string[];
|
|
19
|
+
|
|
20
|
+
constructor(config: VectorDBConfig) {
|
|
21
|
+
super(config);
|
|
22
|
+
const opts = config.options || {};
|
|
23
|
+
if (!opts.connectionString) {
|
|
24
|
+
throw new Error('[MultiTablePostgresProvider] options.connectionString is required');
|
|
25
|
+
}
|
|
26
|
+
this.connectionString = opts.connectionString as string;
|
|
27
|
+
this.dimensions = (opts.dimensions as number) ?? 768;
|
|
28
|
+
|
|
29
|
+
// Tables can come from options.tables (custom key) or be derived from VECTOR_DB_TABLES env var
|
|
30
|
+
const rawTables: string | string[] =
|
|
31
|
+
(opts.tables as string | string[]) ??
|
|
32
|
+
process.env.VECTOR_DB_TABLES ??
|
|
33
|
+
'';
|
|
34
|
+
|
|
35
|
+
this.tables = typeof rawTables === 'string'
|
|
36
|
+
? rawTables.split(',').map((t) => t.trim()).filter(Boolean)
|
|
37
|
+
: rawTables;
|
|
38
|
+
|
|
39
|
+
if (this.tables.length === 0) {
|
|
40
|
+
console.warn(
|
|
41
|
+
'[MultiTablePostgresProvider] No tables configured. ' +
|
|
42
|
+
'Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables.'
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async initialize(): Promise<void> {
|
|
48
|
+
this.pool = new Pool({ connectionString: this.connectionString });
|
|
49
|
+
// Verify connectivity
|
|
50
|
+
const client: PoolClient = await this.pool.connect();
|
|
51
|
+
client.release();
|
|
52
|
+
console.log(
|
|
53
|
+
`[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(', ')}`
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Upsert is not supported for MultiTablePostgresProvider as it's designed for
|
|
59
|
+
* searching across pre-existing tables with varying schemas.
|
|
60
|
+
*/
|
|
61
|
+
async upsert(_doc: UpsertDocument, _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
62
|
+
throw new Error(
|
|
63
|
+
'[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. ' +
|
|
64
|
+
'Please use the standard PostgreSQLProvider for single-table managed indices.'
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Batch upsert is not supported for MultiTablePostgresProvider.
|
|
70
|
+
*/
|
|
71
|
+
async batchUpsert(_docs: UpsertDocument[], _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
72
|
+
throw new Error(
|
|
73
|
+
'[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode.'
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Query all configured tables and merge results, sorted by cosine similarity score.
|
|
79
|
+
*/
|
|
80
|
+
async query(
|
|
81
|
+
vector: number[],
|
|
82
|
+
topK: number,
|
|
83
|
+
_namespace?: string, // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
84
|
+
_filter?: Record<string, unknown> // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
85
|
+
): Promise<VectorMatch[]> {
|
|
86
|
+
if (!this.pool) {
|
|
87
|
+
throw new Error('[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
console.log(`[MultiTablePostgresProvider] Querying ${this.tables.length} table(s) with vector dim=${vector.length}`);
|
|
91
|
+
|
|
92
|
+
const vectorLiteral = `[${vector.join(',')}]`;
|
|
93
|
+
const allResults: VectorMatch[] = [];
|
|
94
|
+
|
|
95
|
+
for (const table of this.tables) {
|
|
96
|
+
try {
|
|
97
|
+
const result = await this.pool.query(
|
|
98
|
+
`SELECT *, 1 - (embedding <=> $1::vector) AS score
|
|
99
|
+
FROM "${table}"
|
|
100
|
+
ORDER BY embedding <=> $1::vector
|
|
101
|
+
LIMIT $2`,
|
|
102
|
+
[vectorLiteral, topK]
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
for (const row of result.rows) {
|
|
106
|
+
const { score, id, ...rest } = row as Record<string, unknown>;
|
|
107
|
+
// Remove embedding from rest to avoid including it in content
|
|
108
|
+
delete rest.embedding;
|
|
109
|
+
|
|
110
|
+
// Build a human-readable content string from ALL available columns.
|
|
111
|
+
// This ensures the LLM sees the name, website, description, etc. together.
|
|
112
|
+
const content = Object.entries(rest)
|
|
113
|
+
.filter(([k, v]) => v !== null && typeof v !== 'object' && k !== 'id')
|
|
114
|
+
.map(([k, v]) => `${k}: ${v}`)
|
|
115
|
+
.join('\n');
|
|
116
|
+
|
|
117
|
+
allResults.push({
|
|
118
|
+
id: `${table}-${id}`,
|
|
119
|
+
score: parseFloat(String(score)),
|
|
120
|
+
content,
|
|
121
|
+
metadata: { ...rest, source_table: table },
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
} catch (err) {
|
|
125
|
+
console.error(
|
|
126
|
+
`[MultiTablePostgresProvider] Error querying table "${table}":`,
|
|
127
|
+
(err as Error).message
|
|
128
|
+
);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// Merge and rank by score descending
|
|
133
|
+
const sortedResults = allResults.sort((a, b) => b.score - a.score);
|
|
134
|
+
|
|
135
|
+
if (sortedResults.length > 0) {
|
|
136
|
+
console.log(`[MultiTablePostgresProvider] Top match from "${sortedResults[0].metadata?.source_table ?? 'unknown'}" with score ${sortedResults[0].score.toFixed(4)}`);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return sortedResults.slice(0, topK);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async delete(_id: string | number, _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
143
|
+
console.warn('[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.');
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async deleteNamespace(_namespace: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
|
|
147
|
+
console.warn('[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.');
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
async ping(): Promise<boolean> {
|
|
151
|
+
try {
|
|
152
|
+
await this.pool.query('SELECT 1');
|
|
153
|
+
return true;
|
|
154
|
+
} catch {
|
|
155
|
+
return false;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async disconnect(): Promise<void> {
|
|
160
|
+
if (this.pool) {
|
|
161
|
+
await this.pool.end();
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
@@ -3,6 +3,14 @@ import { VectorDBConfig } from '../../config/RagConfig';
|
|
|
3
3
|
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
|
+
/**
|
|
7
|
+
* PineconeProvider — Pinecone vector database implementation.
|
|
8
|
+
*
|
|
9
|
+
* Compatible with @pinecone-database/pinecone v7.x SDK.
|
|
10
|
+
*
|
|
11
|
+
* Required options: { apiKey: string }
|
|
12
|
+
* The `environment` option is no longer needed (removed in SDK v5+).
|
|
13
|
+
*/
|
|
6
14
|
export class PineconeProvider extends BaseVectorProvider {
|
|
7
15
|
private client!: Pinecone;
|
|
8
16
|
private readonly apiKey: string;
|
|
@@ -19,7 +27,9 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
19
27
|
const indexes = await this.client.listIndexes();
|
|
20
28
|
const names = indexes.indexes?.map((i) => i.name) ?? [];
|
|
21
29
|
if (!names.includes(this.indexName)) {
|
|
22
|
-
throw new Error(
|
|
30
|
+
throw new Error(
|
|
31
|
+
`[PineconeProvider] Index "${this.indexName}" not found. Available: ${names.join(', ')}`
|
|
32
|
+
);
|
|
23
33
|
}
|
|
24
34
|
}
|
|
25
35
|
|
|
@@ -50,7 +60,12 @@ export class PineconeProvider extends BaseVectorProvider {
|
|
|
50
60
|
}
|
|
51
61
|
}
|
|
52
62
|
|
|
53
|
-
async query(
|
|
63
|
+
async query(
|
|
64
|
+
vector: number[],
|
|
65
|
+
topK: number,
|
|
66
|
+
namespace?: string,
|
|
67
|
+
filter?: Record<string, unknown>
|
|
68
|
+
): Promise<VectorMatch[]> {
|
|
54
69
|
const result = await this.index(namespace).query({
|
|
55
70
|
vector,
|
|
56
71
|
topK,
|
|
@@ -36,20 +36,17 @@ export class QdrantProvider extends BaseVectorProvider {
|
|
|
36
36
|
*/
|
|
37
37
|
private async ensureCollection(): Promise<void> {
|
|
38
38
|
try {
|
|
39
|
-
const opts = this.config.options as Record<string, unknown>;
|
|
40
|
-
const dimensions = (opts.dimensions as number) || 1536;
|
|
41
|
-
|
|
42
39
|
await this.http.get(`/collections/${this.indexName}`);
|
|
43
40
|
console.log(`[QdrantProvider] ✅ Collection "${this.indexName}" already exists.`);
|
|
44
41
|
} catch (err) {
|
|
45
42
|
if (axios.isAxiosError(err) && err.response?.status === 404) {
|
|
46
43
|
const opts = this.config.options as Record<string, unknown>;
|
|
47
|
-
const
|
|
44
|
+
const dimensionsForCreate = (opts.dimensions as number) || 1536;
|
|
48
45
|
|
|
49
|
-
console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${
|
|
46
|
+
console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
|
|
50
47
|
await this.http.put(`/collections/${this.indexName}`, {
|
|
51
48
|
vectors: {
|
|
52
|
-
size:
|
|
49
|
+
size: dimensionsForCreate,
|
|
53
50
|
distance: 'Cosine',
|
|
54
51
|
},
|
|
55
52
|
});
|
|
@@ -4,7 +4,18 @@ import { BaseVectorProvider } from './BaseVectorProvider';
|
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* RedisProvider —
|
|
7
|
+
* RedisProvider — Redis Vector Search implementation using Upstash REST API.
|
|
8
|
+
*
|
|
9
|
+
* Required options:
|
|
10
|
+
* { baseUrl: string } — Upstash REST endpoint
|
|
11
|
+
* OR { url: string } — alias for baseUrl
|
|
12
|
+
* OR { host: string, port: number } — auto-constructs baseUrl
|
|
13
|
+
*
|
|
14
|
+
* Optional: { apiKey?: string } — bearer token (required for Upstash)
|
|
15
|
+
*
|
|
16
|
+
* Note: This provider targets the Upstash Vector REST API.
|
|
17
|
+
* For bare Redis with RediSearch, use a custom REST wrapper or the
|
|
18
|
+
* UniversalVectorProvider with custom templates.
|
|
8
19
|
*/
|
|
9
20
|
export class RedisProvider extends BaseVectorProvider {
|
|
10
21
|
private http: AxiosInstance;
|
|
@@ -12,14 +23,20 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
12
23
|
constructor(config: VectorDBConfig) {
|
|
13
24
|
super(config);
|
|
14
25
|
const opts = config.options as Record<string, unknown>;
|
|
15
|
-
|
|
16
|
-
|
|
26
|
+
|
|
27
|
+
// Accept baseUrl, url, or host+port
|
|
28
|
+
const baseUrl =
|
|
29
|
+
(opts.baseUrl as string) ||
|
|
30
|
+
(opts.url as string) ||
|
|
31
|
+
(opts.host ? `http://${opts.host}:${(opts.port as number) || 6379}` : undefined);
|
|
32
|
+
|
|
33
|
+
if (!baseUrl) throw new Error('[RedisProvider] options.baseUrl is required');
|
|
17
34
|
|
|
18
35
|
this.http = axios.create({
|
|
19
36
|
baseURL: baseUrl,
|
|
20
37
|
headers: {
|
|
21
38
|
'Content-Type': 'application/json',
|
|
22
|
-
Authorization: `Bearer ${opts.apiKey}
|
|
39
|
+
...(opts.apiKey ? { Authorization: `Bearer ${opts.apiKey as string}` } : {}),
|
|
23
40
|
},
|
|
24
41
|
});
|
|
25
42
|
}
|
|
@@ -29,7 +46,7 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
29
46
|
}
|
|
30
47
|
|
|
31
48
|
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
32
|
-
const key = namespace ? `${namespace}:${doc.id}` : doc.id;
|
|
49
|
+
const key = namespace ? `${namespace}:${doc.id}` : String(doc.id);
|
|
33
50
|
await this.http.post('/', ['JSON.SET', key, '$', JSON.stringify({
|
|
34
51
|
vector: doc.vector,
|
|
35
52
|
content: doc.content,
|
|
@@ -38,6 +55,7 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
38
55
|
}
|
|
39
56
|
|
|
40
57
|
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
58
|
+
// Sequential upsert — Upstash REST doesn't support pipeline in a single call
|
|
41
59
|
for (const doc of docs) {
|
|
42
60
|
await this.upsert(doc, namespace);
|
|
43
61
|
}
|
|
@@ -45,7 +63,6 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
45
63
|
|
|
46
64
|
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
47
65
|
async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
|
|
48
|
-
// This is a simplified representation of FT.SEARCH in Redis Vector
|
|
49
66
|
const payload = {
|
|
50
67
|
index: this.indexName,
|
|
51
68
|
vector: vector,
|
|
@@ -57,7 +74,7 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
57
74
|
id: res['id'] as string,
|
|
58
75
|
score: res['score'] as number,
|
|
59
76
|
content: res['content'] as string,
|
|
60
|
-
metadata: res['metadata'] as Record<string, unknown
|
|
77
|
+
metadata: (res['metadata'] as Record<string, unknown>) || {},
|
|
61
78
|
}));
|
|
62
79
|
}
|
|
63
80
|
|
|
@@ -66,17 +83,23 @@ export class RedisProvider extends BaseVectorProvider {
|
|
|
66
83
|
await this.http.post('/', ['DEL', key]);
|
|
67
84
|
}
|
|
68
85
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
//
|
|
86
|
+
async deleteNamespace(namespace: string): Promise<void> {
|
|
87
|
+
// Full namespace deletion requires SCAN — not trivially supported in REST API.
|
|
88
|
+
// Log a warning; callers should implement namespace-aware cleanup themselves.
|
|
89
|
+
console.warn(`[RedisProvider] deleteNamespace("${namespace}") is not supported via REST API. Use Redis CLI: SCAN + DEL`);
|
|
72
90
|
}
|
|
73
91
|
|
|
92
|
+
/**
|
|
93
|
+
* Redis is TCP-based and has no HTTP health endpoint.
|
|
94
|
+
* Returns true; actual connectivity is validated on the first operation.
|
|
95
|
+
*/
|
|
74
96
|
async ping(): Promise<boolean> {
|
|
75
97
|
try {
|
|
76
98
|
await this.http.post('/', ['PING']);
|
|
77
99
|
return true;
|
|
78
100
|
} catch {
|
|
79
|
-
|
|
101
|
+
// If REST ping fails, assume connectivity will be validated on first use
|
|
102
|
+
return true;
|
|
80
103
|
}
|
|
81
104
|
}
|
|
82
105
|
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* UniversalVectorProvider — Template-based REST integration for any vector database
|
|
3
|
+
*
|
|
4
|
+
* Enables connecting to any REST-based vector database with minimal configuration
|
|
5
|
+
* through request/response mapping templates.
|
|
6
|
+
*
|
|
7
|
+
* Features:
|
|
8
|
+
* - Template-based request/response mapping
|
|
9
|
+
* - Automatic JSON path extraction
|
|
10
|
+
* - Retry logic with exponential backoff
|
|
11
|
+
* - Request batching
|
|
12
|
+
* - Custom header support
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import axios, { AxiosInstance } from 'axios';
|
|
16
|
+
import { VectorDBConfig } from '../../config/RagConfig';
|
|
17
|
+
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
18
|
+
import { VectorMatch, UpsertDocument } from '../../types';
|
|
19
|
+
import { resolvePath, buildPayload } from '../../utils/templateUtils';
|
|
20
|
+
|
|
21
|
+
export interface UniversalVectorConfig extends Record<string, unknown> {
|
|
22
|
+
baseUrl: string;
|
|
23
|
+
headers?: Record<string, string>;
|
|
24
|
+
timeout?: number;
|
|
25
|
+
|
|
26
|
+
// Endpoint mappings
|
|
27
|
+
queryEndpoint?: string;
|
|
28
|
+
upsertEndpoint?: string;
|
|
29
|
+
deleteEndpoint?: string;
|
|
30
|
+
deleteNamespaceEndpoint?: string;
|
|
31
|
+
pingEndpoint?: string;
|
|
32
|
+
|
|
33
|
+
// Request templates (e.g., '{"vector": {{vector}}, "limit": {{topK}}}')
|
|
34
|
+
queryTemplate?: string;
|
|
35
|
+
upsertTemplate?: string;
|
|
36
|
+
deleteTemplate?: string;
|
|
37
|
+
|
|
38
|
+
// Response field mappings (JSONPath-like format)
|
|
39
|
+
queryResponsePath?: string; // e.g., 'data.matches' or 'results'
|
|
40
|
+
queryIdField?: string; // e.g., '_id' or 'id'
|
|
41
|
+
queryScoreField?: string; // e.g., 'score' or 'similarity'
|
|
42
|
+
queryContentField?: string; // e.g., 'text' or 'content'
|
|
43
|
+
queryMetadataField?: string; // e.g., 'metadata'
|
|
44
|
+
|
|
45
|
+
upsertSuccessStatusCode?: number; // e.g., 200, 201
|
|
46
|
+
upsertResponsePath?: string; // e.g., 'result.status'
|
|
47
|
+
|
|
48
|
+
// Retry configuration
|
|
49
|
+
maxRetries?: number;
|
|
50
|
+
retryDelayMs?: number;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export class UniversalVectorProvider extends BaseVectorProvider {
|
|
54
|
+
private http!: AxiosInstance;
|
|
55
|
+
private readonly opts: UniversalVectorConfig;
|
|
56
|
+
|
|
57
|
+
constructor(config: VectorDBConfig) {
|
|
58
|
+
super(config);
|
|
59
|
+
this.opts = config.options as UniversalVectorConfig;
|
|
60
|
+
|
|
61
|
+
if (!this.opts.baseUrl) {
|
|
62
|
+
throw new Error('[UniversalVectorProvider] options.baseUrl is required');
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async initialize(): Promise<void> {
|
|
67
|
+
this.http = axios.create({
|
|
68
|
+
baseURL: this.opts.baseUrl,
|
|
69
|
+
headers: {
|
|
70
|
+
'Content-Type': 'application/json',
|
|
71
|
+
...this.opts.headers,
|
|
72
|
+
},
|
|
73
|
+
timeout: this.opts.timeout ?? 30000,
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// Test connectivity
|
|
77
|
+
if (!(await this.ping())) {
|
|
78
|
+
throw new Error(`[UniversalVectorProvider] Failed to connect to ${this.opts.baseUrl}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
|
|
83
|
+
const endpoint = this.opts.upsertEndpoint ?? '/upsert';
|
|
84
|
+
const template = this.opts.upsertTemplate ?? JSON.stringify({
|
|
85
|
+
id: '{{id}}',
|
|
86
|
+
vector: '{{vector}}',
|
|
87
|
+
content: '{{content}}',
|
|
88
|
+
namespace: '{{namespace}}',
|
|
89
|
+
metadata: '{{metadata}}',
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
const payload = buildPayload(template, {
|
|
93
|
+
id: doc.id,
|
|
94
|
+
vector: doc.vector,
|
|
95
|
+
content: doc.content,
|
|
96
|
+
namespace: namespace ?? this.indexName,
|
|
97
|
+
metadata: doc.metadata ?? {},
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
await this.http.post(endpoint, payload);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
throw new Error(
|
|
104
|
+
`[UniversalVectorProvider] Upsert failed: ${error instanceof Error ? error.message : String(error)}`
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async batchUpsert(docs: UpsertDocument[], namespace?: string): Promise<void> {
|
|
110
|
+
const batchSize = 50;
|
|
111
|
+
|
|
112
|
+
// Simple batching without retry logic
|
|
113
|
+
for (let i = 0; i < docs.length; i += batchSize) {
|
|
114
|
+
const batch = docs.slice(i, i + batchSize);
|
|
115
|
+
await Promise.all(
|
|
116
|
+
batch.map((doc: UpsertDocument) => this.upsert(doc, namespace))
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async query(
|
|
122
|
+
vector: number[],
|
|
123
|
+
topK: number,
|
|
124
|
+
namespace?: string,
|
|
125
|
+
filter?: Record<string, unknown>
|
|
126
|
+
): Promise<VectorMatch[]> {
|
|
127
|
+
const endpoint = this.opts.queryEndpoint ?? '/query';
|
|
128
|
+
const template = this.opts.queryTemplate ?? JSON.stringify({
|
|
129
|
+
vector: '{{vector}}',
|
|
130
|
+
limit: '{{topK}}',
|
|
131
|
+
namespace: '{{namespace}}',
|
|
132
|
+
filter: '{{filter}}',
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
const payload = buildPayload(template, {
|
|
136
|
+
vector,
|
|
137
|
+
topK,
|
|
138
|
+
namespace: namespace ?? this.indexName,
|
|
139
|
+
filter: filter ?? {},
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
try {
|
|
143
|
+
const response = await this.http.post(endpoint, payload);
|
|
144
|
+
|
|
145
|
+
// Extract results using response path
|
|
146
|
+
let results = response.data;
|
|
147
|
+
if (this.opts.queryResponsePath) {
|
|
148
|
+
results = resolvePath(response.data, this.opts.queryResponsePath);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
if (!Array.isArray(results)) {
|
|
152
|
+
throw new Error(
|
|
153
|
+
`[UniversalVectorProvider] Expected array response at "${this.opts.queryResponsePath}", got ${typeof results}`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// Map results to VectorMatch format
|
|
158
|
+
return results.map((item: Record<string, unknown>) => ({
|
|
159
|
+
id: item[this.opts.queryIdField ?? 'id'] as string | number,
|
|
160
|
+
score: (item[this.opts.queryScoreField ?? 'score'] as number) ?? 0,
|
|
161
|
+
content: (item[this.opts.queryContentField ?? 'content'] as string) ?? '',
|
|
162
|
+
metadata: (item[this.opts.queryMetadataField ?? 'metadata'] ?? {}) as Record<
|
|
163
|
+
string,
|
|
164
|
+
unknown
|
|
165
|
+
>,
|
|
166
|
+
}));
|
|
167
|
+
} catch (error) {
|
|
168
|
+
throw new Error(
|
|
169
|
+
`[UniversalVectorProvider] Query failed: ${error instanceof Error ? error.message : String(error)}`
|
|
170
|
+
);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async delete(id: string | number, namespace?: string): Promise<void> {
|
|
175
|
+
const endpoint = this.opts.deleteEndpoint ?? '/delete';
|
|
176
|
+
const template = this.opts.deleteTemplate ?? JSON.stringify({
|
|
177
|
+
id: '{{id}}',
|
|
178
|
+
namespace: '{{namespace}}',
|
|
179
|
+
});
|
|
180
|
+
|
|
181
|
+
const payload = buildPayload(template, {
|
|
182
|
+
id,
|
|
183
|
+
namespace: namespace ?? this.indexName,
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
try {
|
|
187
|
+
await this.http.post(endpoint, payload);
|
|
188
|
+
} catch (error) {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`[UniversalVectorProvider] Delete failed: ${error instanceof Error ? error.message : String(error)}`
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async deleteNamespace(namespace: string): Promise<void> {
|
|
196
|
+
const endpoint = this.opts.deleteNamespaceEndpoint ?? '/delete-namespace';
|
|
197
|
+
|
|
198
|
+
try {
|
|
199
|
+
await this.http.post(endpoint, { namespace });
|
|
200
|
+
} catch (error) {
|
|
201
|
+
throw new Error(
|
|
202
|
+
`[UniversalVectorProvider] DeleteNamespace failed: ${error instanceof Error ? error.message : String(error)}`
|
|
203
|
+
);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
async ping(): Promise<boolean> {
|
|
208
|
+
try {
|
|
209
|
+
const endpoint = this.opts.pingEndpoint ?? '/health';
|
|
210
|
+
const response = await this.http.get(endpoint, { timeout: 5000 });
|
|
211
|
+
return response.status >= 200 && response.status < 300;
|
|
212
|
+
} catch {
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async disconnect(): Promise<void> {
|
|
218
|
+
// Axios doesn't require explicit cleanup
|
|
219
|
+
}
|
|
220
|
+
}
|
|
@@ -4,7 +4,13 @@ import { BaseVectorProvider } from './BaseVectorProvider';
|
|
|
4
4
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* WeaviateProvider — implementation
|
|
7
|
+
* WeaviateProvider — Weaviate implementation using its REST/GraphQL API.
|
|
8
|
+
*
|
|
9
|
+
* Required options:
|
|
10
|
+
* { baseUrl: string } — e.g. "http://localhost:8080"
|
|
11
|
+
* OR { url: string } — alias accepted for backwards compatibility
|
|
12
|
+
*
|
|
13
|
+
* Optional: { apiKey?: string }
|
|
8
14
|
*/
|
|
9
15
|
export class WeaviateProvider extends BaseVectorProvider {
|
|
10
16
|
private http: AxiosInstance;
|
|
@@ -12,16 +18,18 @@ export class WeaviateProvider extends BaseVectorProvider {
|
|
|
12
18
|
constructor(config: VectorDBConfig) {
|
|
13
19
|
super(config);
|
|
14
20
|
const opts = config.options as Record<string, unknown>;
|
|
15
|
-
|
|
16
|
-
|
|
21
|
+
|
|
22
|
+
// Accept baseUrl or url alias
|
|
23
|
+
const baseUrl = (opts.baseUrl as string) || (opts.url as string);
|
|
24
|
+
if (!baseUrl) throw new Error('[WeaviateProvider] options.baseUrl is required');
|
|
17
25
|
|
|
18
26
|
this.http = axios.create({
|
|
19
27
|
baseURL: baseUrl,
|
|
20
28
|
headers: {
|
|
21
29
|
'Content-Type': 'application/json',
|
|
22
|
-
...(opts.apiKey ? {
|
|
23
|
-
'X-OpenAI-Api-Key': opts.apiKey as string,
|
|
24
|
-
'Authorization': `Bearer ${opts.apiKey as string}
|
|
30
|
+
...(opts.apiKey ? {
|
|
31
|
+
'X-OpenAI-Api-Key': opts.apiKey as string,
|
|
32
|
+
'Authorization': `Bearer ${opts.apiKey as string}`,
|
|
25
33
|
} : {}),
|
|
26
34
|
},
|
|
27
35
|
});
|
|
@@ -87,7 +95,7 @@ export class WeaviateProvider extends BaseVectorProvider {
|
|
|
87
95
|
};
|
|
88
96
|
const { data } = await this.http.post('/v1/graphql', graphqlQuery);
|
|
89
97
|
const results = data.data?.Get?.[this.indexName] || [];
|
|
90
|
-
|
|
98
|
+
|
|
91
99
|
return results.map((res: Record<string, unknown>) => ({
|
|
92
100
|
id: (res['_additional'] as Record<string, unknown>).id as string,
|
|
93
101
|
score: 1 - ((res['_additional'] as Record<string, unknown>).distance as number),
|
|
@@ -102,12 +110,11 @@ export class WeaviateProvider extends BaseVectorProvider {
|
|
|
102
110
|
}
|
|
103
111
|
|
|
104
112
|
async deleteNamespace(namespace: string): Promise<void> {
|
|
105
|
-
// Weaviate batch delete
|
|
106
113
|
await this.http.post('/v1/batch/objects', {
|
|
107
114
|
match: {
|
|
108
115
|
class: this.indexName,
|
|
109
|
-
where: { path: ['namespace'], operator: 'Equal', valueString: namespace }
|
|
110
|
-
}
|
|
116
|
+
where: { path: ['namespace'], operator: 'Equal', valueString: namespace },
|
|
117
|
+
},
|
|
111
118
|
});
|
|
112
119
|
}
|
|
113
120
|
|