@retrivora-ai/rag-engine 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/DocumentChunker-BEyzadsv.d.mts +93 -0
- package/dist/DocumentChunker-BEyzadsv.d.ts +93 -0
- package/dist/{LLMFactory-XC55FTD2.mjs → LLMFactory-JFOY2V4X.mjs} +2 -1
- package/dist/{PineconeProvider-NJ675H7U.mjs → PineconeProvider-ZRAFNFEC.mjs} +1 -1
- package/dist/{PostgreSQLProvider-ISNMD3BE.mjs → PostgreSQLProvider-ZNXA67IM.mjs} +1 -1
- package/dist/{QdrantProvider-LJWOIGES.mjs → QdrantProvider-NYGHAA7C.mjs} +1 -1
- package/dist/{Pipeline-Bo6CUCox.d.mts → RagConfig-hBGXJmSx.d.mts} +23 -93
- package/dist/{Pipeline-Bo6CUCox.d.ts → RagConfig-hBGXJmSx.d.ts} +23 -93
- package/dist/{chunk-6FODXNUF.mjs → chunk-5K23H7JL.mjs} +27 -1
- package/dist/chunk-HSBXE2WV.mjs +1758 -0
- package/dist/{chunk-S5DRHETN.mjs → chunk-IUTAZ7QR.mjs} +31 -2
- package/dist/{chunk-GD3QJFNN.mjs → chunk-JI6VD5TJ.mjs} +4 -41
- package/dist/chunk-UKDXCXW7.mjs +49 -0
- package/dist/{chunk-AALIF3AL.mjs → chunk-ZM6TYIDH.mjs} +3 -3
- package/dist/handlers/index.d.mts +3 -44
- package/dist/handlers/index.d.ts +3 -44
- package/dist/handlers/index.js +1322 -57
- package/dist/handlers/index.mjs +3 -3
- package/dist/index-Bx182KKn.d.ts +64 -0
- package/dist/index-Ck2pt7-8.d.mts +64 -0
- package/dist/index.d.mts +4 -3
- package/dist/index.d.ts +4 -3
- package/dist/index.js +19 -1646
- package/dist/index.mjs +1 -11
- package/dist/server.d.mts +188 -11
- package/dist/server.d.ts +188 -11
- package/dist/server.js +1322 -57
- package/dist/server.mjs +10 -11
- package/package.json +20 -2
- package/src/config/serverConfig.ts +3 -0
- package/src/core/BatchProcessor.ts +347 -0
- package/src/core/ConfigValidator.ts +560 -0
- package/src/core/Pipeline.ts +91 -49
- package/src/core/ProviderHealthCheck.ts +565 -0
- package/src/core/VectorPlugin.ts +51 -9
- package/src/handlers/index.ts +2 -2
- package/src/index.ts +1 -7
- package/src/providers/vectordb/BaseVectorProvider.ts +1 -1
- package/src/providers/vectordb/MilvusProvider.ts +1 -1
- package/src/providers/vectordb/MongoDBProvider.ts +1 -1
- package/src/providers/vectordb/PineconeProvider.ts +4 -4
- package/src/providers/vectordb/PostgreSQLProvider.ts +35 -3
- package/src/providers/vectordb/QdrantProvider.ts +32 -2
- package/src/rag/DocumentChunker.ts +2 -2
- package/src/server.ts +1 -1
- package/src/types/index.ts +13 -2
- package/dist/DocumentChunker-BQ5kQD7B.d.ts +0 -155
- package/dist/DocumentChunker-D9-fObJp.d.mts +0 -155
- package/dist/chunk-AIAB2IEE.mjs +0 -394
- package/dist/chunk-FGGSVVSY.mjs +0 -162
|
@@ -52,8 +52,37 @@ var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
|
52
52
|
);
|
|
53
53
|
}
|
|
54
54
|
async batchUpsert(docs, namespace = "") {
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
if (docs.length === 0) return;
|
|
56
|
+
const client = await this.pool.connect();
|
|
57
|
+
try {
|
|
58
|
+
await client.query("BEGIN");
|
|
59
|
+
const BATCH_SIZE = 50;
|
|
60
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
61
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
62
|
+
const values = [];
|
|
63
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
64
|
+
var _a;
|
|
65
|
+
const offset = idx * 5;
|
|
66
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
67
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
68
|
+
}).join(", ");
|
|
69
|
+
const query = `
|
|
70
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
71
|
+
VALUES ${valuePlaceholders}
|
|
72
|
+
ON CONFLICT (id) DO UPDATE
|
|
73
|
+
SET namespace = EXCLUDED.namespace,
|
|
74
|
+
content = EXCLUDED.content,
|
|
75
|
+
metadata = EXCLUDED.metadata,
|
|
76
|
+
embedding = EXCLUDED.embedding
|
|
77
|
+
`;
|
|
78
|
+
await client.query(query, values);
|
|
79
|
+
}
|
|
80
|
+
await client.query("COMMIT");
|
|
81
|
+
} catch (error) {
|
|
82
|
+
await client.query("ROLLBACK");
|
|
83
|
+
throw error;
|
|
84
|
+
} finally {
|
|
85
|
+
client.release();
|
|
57
86
|
}
|
|
58
87
|
}
|
|
59
88
|
async query(vector, topK, namespace, filter) {
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import {
|
|
2
|
+
buildPayload,
|
|
3
|
+
resolvePath
|
|
4
|
+
} from "./chunk-UKDXCXW7.mjs";
|
|
1
5
|
import {
|
|
2
6
|
__spreadProps,
|
|
3
7
|
__spreadValues
|
|
@@ -192,46 +196,6 @@ ${context}`;
|
|
|
192
196
|
// src/llm/providers/UniversalLLMAdapter.ts
|
|
193
197
|
import axios2 from "axios";
|
|
194
198
|
|
|
195
|
-
// src/utils/templateUtils.ts
|
|
196
|
-
function isRecord(value) {
|
|
197
|
-
return typeof value === "object" && value !== null;
|
|
198
|
-
}
|
|
199
|
-
function resolvePath(obj, path) {
|
|
200
|
-
if (!path || !obj) return void 0;
|
|
201
|
-
return path.replace(/\[(\w+)\]/g, ".$1").replace(/^\./, "").split(".").reduce((current, segment) => {
|
|
202
|
-
if (!isRecord(current) && !Array.isArray(current)) {
|
|
203
|
-
return void 0;
|
|
204
|
-
}
|
|
205
|
-
return current[segment];
|
|
206
|
-
}, obj);
|
|
207
|
-
}
|
|
208
|
-
function buildPayload(template, vars) {
|
|
209
|
-
let raw = template;
|
|
210
|
-
for (const [key, val] of Object.entries(vars)) {
|
|
211
|
-
const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
|
|
212
|
-
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
|
|
213
|
-
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
|
|
214
|
-
}
|
|
215
|
-
try {
|
|
216
|
-
return JSON.parse(raw);
|
|
217
|
-
} catch (err) {
|
|
218
|
-
throw new Error(`Failed to parse payload template: ${err.message}
|
|
219
|
-
Raw payload: ${raw}`);
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
function mergeDefined(base, override) {
|
|
223
|
-
const merged = __spreadValues({}, base || {});
|
|
224
|
-
if (!override) {
|
|
225
|
-
return merged;
|
|
226
|
-
}
|
|
227
|
-
for (const [key, value] of Object.entries(override)) {
|
|
228
|
-
if (value !== void 0 && value !== null && value !== "") {
|
|
229
|
-
merged[key] = value;
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
return merged;
|
|
233
|
-
}
|
|
234
|
-
|
|
235
199
|
// src/config/UniversalProfiles.ts
|
|
236
200
|
var OPENAI_BASE = {
|
|
237
201
|
chatPath: "/chat/completions",
|
|
@@ -416,7 +380,6 @@ var LLMFactory = class _LLMFactory {
|
|
|
416
380
|
};
|
|
417
381
|
|
|
418
382
|
export {
|
|
419
|
-
mergeDefined,
|
|
420
383
|
OpenAIProvider,
|
|
421
384
|
AnthropicProvider,
|
|
422
385
|
OllamaProvider,
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import {
|
|
2
|
+
__spreadValues
|
|
3
|
+
} from "./chunk-I4E63NIC.mjs";
|
|
4
|
+
|
|
5
|
+
// src/utils/templateUtils.ts
|
|
6
|
+
function isRecord(value) {
|
|
7
|
+
return typeof value === "object" && value !== null;
|
|
8
|
+
}
|
|
9
|
+
function resolvePath(obj, path) {
|
|
10
|
+
if (!path || !obj) return void 0;
|
|
11
|
+
return path.replace(/\[(\w+)\]/g, ".$1").replace(/^\./, "").split(".").reduce((current, segment) => {
|
|
12
|
+
if (!isRecord(current) && !Array.isArray(current)) {
|
|
13
|
+
return void 0;
|
|
14
|
+
}
|
|
15
|
+
return current[segment];
|
|
16
|
+
}, obj);
|
|
17
|
+
}
|
|
18
|
+
function buildPayload(template, vars) {
|
|
19
|
+
let raw = template;
|
|
20
|
+
for (const [key, val] of Object.entries(vars)) {
|
|
21
|
+
const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
|
|
22
|
+
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
|
|
23
|
+
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
|
|
24
|
+
}
|
|
25
|
+
try {
|
|
26
|
+
return JSON.parse(raw);
|
|
27
|
+
} catch (err) {
|
|
28
|
+
throw new Error(`Failed to parse payload template: ${err.message}
|
|
29
|
+
Raw payload: ${raw}`);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
function mergeDefined(base, override) {
|
|
33
|
+
const merged = __spreadValues({}, base || {});
|
|
34
|
+
if (!override) {
|
|
35
|
+
return merged;
|
|
36
|
+
}
|
|
37
|
+
for (const [key, value] of Object.entries(override)) {
|
|
38
|
+
if (value !== void 0 && value !== null && value !== "") {
|
|
39
|
+
merged[key] = value;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return merged;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export {
|
|
46
|
+
resolvePath,
|
|
47
|
+
buildPayload,
|
|
48
|
+
mergeDefined
|
|
49
|
+
};
|
|
@@ -31,7 +31,7 @@ var PineconeProvider = class extends BaseVectorProvider {
|
|
|
31
31
|
var _a;
|
|
32
32
|
await this.index(namespace).upsert({
|
|
33
33
|
records: [{
|
|
34
|
-
id: doc.id,
|
|
34
|
+
id: String(doc.id),
|
|
35
35
|
values: doc.vector,
|
|
36
36
|
metadata: __spreadValues({ content: doc.content }, (_a = doc.metadata) != null ? _a : {})
|
|
37
37
|
}]
|
|
@@ -43,7 +43,7 @@ var PineconeProvider = class extends BaseVectorProvider {
|
|
|
43
43
|
const records = docs.slice(i, i + BATCH).map((d) => {
|
|
44
44
|
var _a;
|
|
45
45
|
return {
|
|
46
|
-
id: d.id,
|
|
46
|
+
id: String(d.id),
|
|
47
47
|
values: d.vector,
|
|
48
48
|
metadata: __spreadValues({ content: d.content }, (_a = d.metadata) != null ? _a : {})
|
|
49
49
|
};
|
|
@@ -69,7 +69,7 @@ var PineconeProvider = class extends BaseVectorProvider {
|
|
|
69
69
|
});
|
|
70
70
|
}
|
|
71
71
|
async delete(id, namespace) {
|
|
72
|
-
await this.index(namespace).deleteOne({ id });
|
|
72
|
+
await this.index(namespace).deleteOne({ id: String(id) });
|
|
73
73
|
}
|
|
74
74
|
async deleteNamespace(namespace) {
|
|
75
75
|
await this.client.index(this.indexName).namespace(namespace).deleteAll();
|
|
@@ -1,44 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* createChatHandler — factory that returns a Next.js App Router POST handler
|
|
6
|
-
*/
|
|
7
|
-
declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
8
|
-
error: string;
|
|
9
|
-
}> | NextResponse<ChatResponse>>;
|
|
10
|
-
/**
|
|
11
|
-
* createIngestHandler — factory for the document ingestion endpoint.
|
|
12
|
-
*/
|
|
13
|
-
declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
14
|
-
error: string;
|
|
15
|
-
}> | NextResponse<{
|
|
16
|
-
results: {
|
|
17
|
-
docId: string;
|
|
18
|
-
chunksIngested: number;
|
|
19
|
-
}[];
|
|
20
|
-
}>>;
|
|
21
|
-
/**
|
|
22
|
-
* createHealthHandler — factory for the health-check endpoint.
|
|
23
|
-
*/
|
|
24
|
-
declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
|
|
25
|
-
timestamp: string;
|
|
26
|
-
status: string;
|
|
27
|
-
}> | NextResponse<{
|
|
28
|
-
status: string;
|
|
29
|
-
error: string;
|
|
30
|
-
}>>;
|
|
31
|
-
/**
|
|
32
|
-
* createUploadHandler — factory for the file upload ingestion endpoint.
|
|
33
|
-
*/
|
|
34
|
-
declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
35
|
-
error: string;
|
|
36
|
-
}> | NextResponse<{
|
|
37
|
-
message: string;
|
|
38
|
-
results: {
|
|
39
|
-
docId: string;
|
|
40
|
-
chunksIngested: number;
|
|
41
|
-
}[];
|
|
42
|
-
}>>;
|
|
43
|
-
|
|
44
|
-
export { createChatHandler, createHealthHandler, createIngestHandler, createUploadHandler };
|
|
1
|
+
export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-Ck2pt7-8.mjs';
|
|
2
|
+
import '../RagConfig-hBGXJmSx.mjs';
|
|
3
|
+
import 'next/server';
|
package/dist/handlers/index.d.ts
CHANGED
|
@@ -1,44 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* createChatHandler — factory that returns a Next.js App Router POST handler
|
|
6
|
-
*/
|
|
7
|
-
declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
8
|
-
error: string;
|
|
9
|
-
}> | NextResponse<ChatResponse>>;
|
|
10
|
-
/**
|
|
11
|
-
* createIngestHandler — factory for the document ingestion endpoint.
|
|
12
|
-
*/
|
|
13
|
-
declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
14
|
-
error: string;
|
|
15
|
-
}> | NextResponse<{
|
|
16
|
-
results: {
|
|
17
|
-
docId: string;
|
|
18
|
-
chunksIngested: number;
|
|
19
|
-
}[];
|
|
20
|
-
}>>;
|
|
21
|
-
/**
|
|
22
|
-
* createHealthHandler — factory for the health-check endpoint.
|
|
23
|
-
*/
|
|
24
|
-
declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
|
|
25
|
-
timestamp: string;
|
|
26
|
-
status: string;
|
|
27
|
-
}> | NextResponse<{
|
|
28
|
-
status: string;
|
|
29
|
-
error: string;
|
|
30
|
-
}>>;
|
|
31
|
-
/**
|
|
32
|
-
* createUploadHandler — factory for the file upload ingestion endpoint.
|
|
33
|
-
*/
|
|
34
|
-
declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
35
|
-
error: string;
|
|
36
|
-
}> | NextResponse<{
|
|
37
|
-
message: string;
|
|
38
|
-
results: {
|
|
39
|
-
docId: string;
|
|
40
|
-
chunksIngested: number;
|
|
41
|
-
}[];
|
|
42
|
-
}>>;
|
|
43
|
-
|
|
44
|
-
export { createChatHandler, createHealthHandler, createIngestHandler, createUploadHandler };
|
|
1
|
+
export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-Bx182KKn.js';
|
|
2
|
+
import '../RagConfig-hBGXJmSx.js';
|
|
3
|
+
import 'next/server';
|