@retrivora-ai/rag-engine 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-cfaMidtA.d.mts → DocumentChunker-BEyzadsv.d.mts} +2 -2
- package/dist/{DocumentChunker-cfaMidtA.d.ts → DocumentChunker-BEyzadsv.d.ts} +2 -2
- package/dist/{PineconeProvider-NJ675H7U.mjs → PineconeProvider-ZRAFNFEC.mjs} +1 -1
- package/dist/{PostgreSQLProvider-ISNMD3BE.mjs → PostgreSQLProvider-ZNXA67IM.mjs} +1 -1
- package/dist/{QdrantProvider-LJWOIGES.mjs → QdrantProvider-NYGHAA7C.mjs} +1 -1
- package/dist/{RagConfig-DG_0f8ka.d.mts → RagConfig-hBGXJmSx.d.mts} +3 -3
- package/dist/{RagConfig-DG_0f8ka.d.ts → RagConfig-hBGXJmSx.d.ts} +3 -3
- package/dist/{chunk-6FODXNUF.mjs → chunk-5K23H7JL.mjs} +27 -1
- package/dist/chunk-HSBXE2WV.mjs +1758 -0
- package/dist/{chunk-S5DRHETN.mjs → chunk-IUTAZ7QR.mjs} +31 -2
- package/dist/{chunk-AALIF3AL.mjs → chunk-ZM6TYIDH.mjs} +3 -3
- package/dist/handlers/index.d.mts +3 -44
- package/dist/handlers/index.d.ts +3 -44
- package/dist/handlers/index.js +1322 -57
- package/dist/handlers/index.mjs +1 -1
- package/dist/index-Bx182KKn.d.ts +64 -0
- package/dist/index-Ck2pt7-8.d.mts +64 -0
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/server.d.mts +65 -18
- package/dist/server.d.ts +65 -18
- package/dist/server.js +1322 -57
- package/dist/server.mjs +4 -4
- package/package.json +2 -1
- package/src/config/serverConfig.ts +3 -0
- package/src/core/BatchProcessor.ts +347 -0
- package/src/core/ConfigValidator.ts +560 -0
- package/src/core/Pipeline.ts +89 -39
- package/src/core/ProviderHealthCheck.ts +565 -0
- package/src/core/VectorPlugin.ts +49 -8
- package/src/handlers/index.ts +2 -2
- package/src/providers/vectordb/BaseVectorProvider.ts +1 -1
- package/src/providers/vectordb/MilvusProvider.ts +1 -1
- package/src/providers/vectordb/MongoDBProvider.ts +1 -1
- package/src/providers/vectordb/PineconeProvider.ts +4 -4
- package/src/providers/vectordb/PostgreSQLProvider.ts +35 -3
- package/src/providers/vectordb/QdrantProvider.ts +32 -2
- package/src/rag/DocumentChunker.ts +2 -2
- package/src/types/index.ts +3 -3
- package/dist/chunk-BP4U4TT5.mjs +0 -548
|
@@ -52,8 +52,37 @@ var PostgreSQLProvider = class extends BaseVectorProvider {
|
|
|
52
52
|
);
|
|
53
53
|
}
|
|
54
54
|
async batchUpsert(docs, namespace = "") {
|
|
55
|
-
|
|
56
|
-
|
|
55
|
+
if (docs.length === 0) return;
|
|
56
|
+
const client = await this.pool.connect();
|
|
57
|
+
try {
|
|
58
|
+
await client.query("BEGIN");
|
|
59
|
+
const BATCH_SIZE = 50;
|
|
60
|
+
for (let i = 0; i < docs.length; i += BATCH_SIZE) {
|
|
61
|
+
const batch = docs.slice(i, i + BATCH_SIZE);
|
|
62
|
+
const values = [];
|
|
63
|
+
const valuePlaceholders = batch.map((doc, idx) => {
|
|
64
|
+
var _a;
|
|
65
|
+
const offset = idx * 5;
|
|
66
|
+
values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
|
|
67
|
+
return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
|
|
68
|
+
}).join(", ");
|
|
69
|
+
const query = `
|
|
70
|
+
INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
|
|
71
|
+
VALUES ${valuePlaceholders}
|
|
72
|
+
ON CONFLICT (id) DO UPDATE
|
|
73
|
+
SET namespace = EXCLUDED.namespace,
|
|
74
|
+
content = EXCLUDED.content,
|
|
75
|
+
metadata = EXCLUDED.metadata,
|
|
76
|
+
embedding = EXCLUDED.embedding
|
|
77
|
+
`;
|
|
78
|
+
await client.query(query, values);
|
|
79
|
+
}
|
|
80
|
+
await client.query("COMMIT");
|
|
81
|
+
} catch (error) {
|
|
82
|
+
await client.query("ROLLBACK");
|
|
83
|
+
throw error;
|
|
84
|
+
} finally {
|
|
85
|
+
client.release();
|
|
57
86
|
}
|
|
58
87
|
}
|
|
59
88
|
async query(vector, topK, namespace, filter) {
|
|
@@ -31,7 +31,7 @@ var PineconeProvider = class extends BaseVectorProvider {
|
|
|
31
31
|
var _a;
|
|
32
32
|
await this.index(namespace).upsert({
|
|
33
33
|
records: [{
|
|
34
|
-
id: doc.id,
|
|
34
|
+
id: String(doc.id),
|
|
35
35
|
values: doc.vector,
|
|
36
36
|
metadata: __spreadValues({ content: doc.content }, (_a = doc.metadata) != null ? _a : {})
|
|
37
37
|
}]
|
|
@@ -43,7 +43,7 @@ var PineconeProvider = class extends BaseVectorProvider {
|
|
|
43
43
|
const records = docs.slice(i, i + BATCH).map((d) => {
|
|
44
44
|
var _a;
|
|
45
45
|
return {
|
|
46
|
-
id: d.id,
|
|
46
|
+
id: String(d.id),
|
|
47
47
|
values: d.vector,
|
|
48
48
|
metadata: __spreadValues({ content: d.content }, (_a = d.metadata) != null ? _a : {})
|
|
49
49
|
};
|
|
@@ -69,7 +69,7 @@ var PineconeProvider = class extends BaseVectorProvider {
|
|
|
69
69
|
});
|
|
70
70
|
}
|
|
71
71
|
async delete(id, namespace) {
|
|
72
|
-
await this.index(namespace).deleteOne({ id });
|
|
72
|
+
await this.index(namespace).deleteOne({ id: String(id) });
|
|
73
73
|
}
|
|
74
74
|
async deleteNamespace(namespace) {
|
|
75
75
|
await this.client.index(this.indexName).namespace(namespace).deleteAll();
|
|
@@ -1,44 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* createChatHandler — factory that returns a Next.js App Router POST handler
|
|
6
|
-
*/
|
|
7
|
-
declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
8
|
-
error: string;
|
|
9
|
-
}> | NextResponse<ChatResponse>>;
|
|
10
|
-
/**
|
|
11
|
-
* createIngestHandler — factory for the document ingestion endpoint.
|
|
12
|
-
*/
|
|
13
|
-
declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
14
|
-
error: string;
|
|
15
|
-
}> | NextResponse<{
|
|
16
|
-
results: {
|
|
17
|
-
docId: string;
|
|
18
|
-
chunksIngested: number;
|
|
19
|
-
}[];
|
|
20
|
-
}>>;
|
|
21
|
-
/**
|
|
22
|
-
* createHealthHandler — factory for the health-check endpoint.
|
|
23
|
-
*/
|
|
24
|
-
declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
|
|
25
|
-
timestamp: string;
|
|
26
|
-
status: string;
|
|
27
|
-
}> | NextResponse<{
|
|
28
|
-
status: string;
|
|
29
|
-
error: string;
|
|
30
|
-
}>>;
|
|
31
|
-
/**
|
|
32
|
-
* createUploadHandler — factory for the file upload ingestion endpoint.
|
|
33
|
-
*/
|
|
34
|
-
declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
35
|
-
error: string;
|
|
36
|
-
}> | NextResponse<{
|
|
37
|
-
message: string;
|
|
38
|
-
results: {
|
|
39
|
-
docId: string;
|
|
40
|
-
chunksIngested: number;
|
|
41
|
-
}[];
|
|
42
|
-
}>>;
|
|
43
|
-
|
|
44
|
-
export { createChatHandler, createHealthHandler, createIngestHandler, createUploadHandler };
|
|
1
|
+
export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-Ck2pt7-8.mjs';
|
|
2
|
+
import '../RagConfig-hBGXJmSx.mjs';
|
|
3
|
+
import 'next/server';
|
package/dist/handlers/index.d.ts
CHANGED
|
@@ -1,44 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
import
|
|
3
|
-
|
|
4
|
-
/**
|
|
5
|
-
* createChatHandler — factory that returns a Next.js App Router POST handler
|
|
6
|
-
*/
|
|
7
|
-
declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
8
|
-
error: string;
|
|
9
|
-
}> | NextResponse<ChatResponse>>;
|
|
10
|
-
/**
|
|
11
|
-
* createIngestHandler — factory for the document ingestion endpoint.
|
|
12
|
-
*/
|
|
13
|
-
declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
14
|
-
error: string;
|
|
15
|
-
}> | NextResponse<{
|
|
16
|
-
results: {
|
|
17
|
-
docId: string;
|
|
18
|
-
chunksIngested: number;
|
|
19
|
-
}[];
|
|
20
|
-
}>>;
|
|
21
|
-
/**
|
|
22
|
-
* createHealthHandler — factory for the health-check endpoint.
|
|
23
|
-
*/
|
|
24
|
-
declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
|
|
25
|
-
timestamp: string;
|
|
26
|
-
status: string;
|
|
27
|
-
}> | NextResponse<{
|
|
28
|
-
status: string;
|
|
29
|
-
error: string;
|
|
30
|
-
}>>;
|
|
31
|
-
/**
|
|
32
|
-
* createUploadHandler — factory for the file upload ingestion endpoint.
|
|
33
|
-
*/
|
|
34
|
-
declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
|
|
35
|
-
error: string;
|
|
36
|
-
}> | NextResponse<{
|
|
37
|
-
message: string;
|
|
38
|
-
results: {
|
|
39
|
-
docId: string;
|
|
40
|
-
chunksIngested: number;
|
|
41
|
-
}[];
|
|
42
|
-
}>>;
|
|
43
|
-
|
|
44
|
-
export { createChatHandler, createHealthHandler, createIngestHandler, createUploadHandler };
|
|
1
|
+
export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-Bx182KKn.js';
|
|
2
|
+
import '../RagConfig-hBGXJmSx.js';
|
|
3
|
+
import 'next/server';
|