@retrivora-ai/rag-engine 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/dist/DocumentChunker-BEyzadsv.d.mts +93 -0
  2. package/dist/DocumentChunker-BEyzadsv.d.ts +93 -0
  3. package/dist/{LLMFactory-XC55FTD2.mjs → LLMFactory-JFOY2V4X.mjs} +2 -1
  4. package/dist/{PineconeProvider-NJ675H7U.mjs → PineconeProvider-ZRAFNFEC.mjs} +1 -1
  5. package/dist/{PostgreSQLProvider-ISNMD3BE.mjs → PostgreSQLProvider-ZNXA67IM.mjs} +1 -1
  6. package/dist/{QdrantProvider-LJWOIGES.mjs → QdrantProvider-NYGHAA7C.mjs} +1 -1
  7. package/dist/{Pipeline-Bo6CUCox.d.mts → RagConfig-hBGXJmSx.d.mts} +23 -93
  8. package/dist/{Pipeline-Bo6CUCox.d.ts → RagConfig-hBGXJmSx.d.ts} +23 -93
  9. package/dist/{chunk-6FODXNUF.mjs → chunk-5K23H7JL.mjs} +27 -1
  10. package/dist/chunk-HSBXE2WV.mjs +1758 -0
  11. package/dist/{chunk-S5DRHETN.mjs → chunk-IUTAZ7QR.mjs} +31 -2
  12. package/dist/{chunk-GD3QJFNN.mjs → chunk-JI6VD5TJ.mjs} +4 -41
  13. package/dist/chunk-UKDXCXW7.mjs +49 -0
  14. package/dist/{chunk-AALIF3AL.mjs → chunk-ZM6TYIDH.mjs} +3 -3
  15. package/dist/handlers/index.d.mts +3 -44
  16. package/dist/handlers/index.d.ts +3 -44
  17. package/dist/handlers/index.js +1322 -57
  18. package/dist/handlers/index.mjs +3 -3
  19. package/dist/index-Bx182KKn.d.ts +64 -0
  20. package/dist/index-Ck2pt7-8.d.mts +64 -0
  21. package/dist/index.d.mts +4 -3
  22. package/dist/index.d.ts +4 -3
  23. package/dist/index.js +19 -1646
  24. package/dist/index.mjs +1 -11
  25. package/dist/server.d.mts +188 -11
  26. package/dist/server.d.ts +188 -11
  27. package/dist/server.js +1322 -57
  28. package/dist/server.mjs +10 -11
  29. package/package.json +20 -2
  30. package/src/config/serverConfig.ts +3 -0
  31. package/src/core/BatchProcessor.ts +347 -0
  32. package/src/core/ConfigValidator.ts +560 -0
  33. package/src/core/Pipeline.ts +91 -49
  34. package/src/core/ProviderHealthCheck.ts +565 -0
  35. package/src/core/VectorPlugin.ts +51 -9
  36. package/src/handlers/index.ts +2 -2
  37. package/src/index.ts +1 -7
  38. package/src/providers/vectordb/BaseVectorProvider.ts +1 -1
  39. package/src/providers/vectordb/MilvusProvider.ts +1 -1
  40. package/src/providers/vectordb/MongoDBProvider.ts +1 -1
  41. package/src/providers/vectordb/PineconeProvider.ts +4 -4
  42. package/src/providers/vectordb/PostgreSQLProvider.ts +35 -3
  43. package/src/providers/vectordb/QdrantProvider.ts +32 -2
  44. package/src/rag/DocumentChunker.ts +2 -2
  45. package/src/server.ts +1 -1
  46. package/src/types/index.ts +13 -2
  47. package/dist/DocumentChunker-BQ5kQD7B.d.ts +0 -155
  48. package/dist/DocumentChunker-D9-fObJp.d.mts +0 -155
  49. package/dist/chunk-AIAB2IEE.mjs +0 -394
  50. package/dist/chunk-FGGSVVSY.mjs +0 -162
@@ -52,8 +52,37 @@ var PostgreSQLProvider = class extends BaseVectorProvider {
52
52
  );
53
53
  }
54
54
  async batchUpsert(docs, namespace = "") {
55
- for (const doc of docs) {
56
- await this.upsert(doc, namespace);
55
+ if (docs.length === 0) return;
56
+ const client = await this.pool.connect();
57
+ try {
58
+ await client.query("BEGIN");
59
+ const BATCH_SIZE = 50;
60
+ for (let i = 0; i < docs.length; i += BATCH_SIZE) {
61
+ const batch = docs.slice(i, i + BATCH_SIZE);
62
+ const values = [];
63
+ const valuePlaceholders = batch.map((doc, idx) => {
64
+ var _a;
65
+ const offset = idx * 5;
66
+ values.push(doc.id, namespace, doc.content, JSON.stringify((_a = doc.metadata) != null ? _a : {}), `[${doc.vector.join(",")}]`);
67
+ return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}::vector)`;
68
+ }).join(", ");
69
+ const query = `
70
+ INSERT INTO ${this.tableName} (id, namespace, content, metadata, embedding)
71
+ VALUES ${valuePlaceholders}
72
+ ON CONFLICT (id) DO UPDATE
73
+ SET namespace = EXCLUDED.namespace,
74
+ content = EXCLUDED.content,
75
+ metadata = EXCLUDED.metadata,
76
+ embedding = EXCLUDED.embedding
77
+ `;
78
+ await client.query(query, values);
79
+ }
80
+ await client.query("COMMIT");
81
+ } catch (error) {
82
+ await client.query("ROLLBACK");
83
+ throw error;
84
+ } finally {
85
+ client.release();
57
86
  }
58
87
  }
59
88
  async query(vector, topK, namespace, filter) {
@@ -1,3 +1,7 @@
1
+ import {
2
+ buildPayload,
3
+ resolvePath
4
+ } from "./chunk-UKDXCXW7.mjs";
1
5
  import {
2
6
  __spreadProps,
3
7
  __spreadValues
@@ -192,46 +196,6 @@ ${context}`;
192
196
  // src/llm/providers/UniversalLLMAdapter.ts
193
197
  import axios2 from "axios";
194
198
 
195
- // src/utils/templateUtils.ts
196
- function isRecord(value) {
197
- return typeof value === "object" && value !== null;
198
- }
199
- function resolvePath(obj, path) {
200
- if (!path || !obj) return void 0;
201
- return path.replace(/\[(\w+)\]/g, ".$1").replace(/^\./, "").split(".").reduce((current, segment) => {
202
- if (!isRecord(current) && !Array.isArray(current)) {
203
- return void 0;
204
- }
205
- return current[segment];
206
- }, obj);
207
- }
208
- function buildPayload(template, vars) {
209
- let raw = template;
210
- for (const [key, val] of Object.entries(vars)) {
211
- const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
212
- raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
213
- raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
214
- }
215
- try {
216
- return JSON.parse(raw);
217
- } catch (err) {
218
- throw new Error(`Failed to parse payload template: ${err.message}
219
- Raw payload: ${raw}`);
220
- }
221
- }
222
- function mergeDefined(base, override) {
223
- const merged = __spreadValues({}, base || {});
224
- if (!override) {
225
- return merged;
226
- }
227
- for (const [key, value] of Object.entries(override)) {
228
- if (value !== void 0 && value !== null && value !== "") {
229
- merged[key] = value;
230
- }
231
- }
232
- return merged;
233
- }
234
-
235
199
  // src/config/UniversalProfiles.ts
236
200
  var OPENAI_BASE = {
237
201
  chatPath: "/chat/completions",
@@ -416,7 +380,6 @@ var LLMFactory = class _LLMFactory {
416
380
  };
417
381
 
418
382
  export {
419
- mergeDefined,
420
383
  OpenAIProvider,
421
384
  AnthropicProvider,
422
385
  OllamaProvider,
@@ -0,0 +1,49 @@
1
+ import {
2
+ __spreadValues
3
+ } from "./chunk-I4E63NIC.mjs";
4
+
5
+ // src/utils/templateUtils.ts
6
+ function isRecord(value) {
7
+ return typeof value === "object" && value !== null;
8
+ }
9
+ function resolvePath(obj, path) {
10
+ if (!path || !obj) return void 0;
11
+ return path.replace(/\[(\w+)\]/g, ".$1").replace(/^\./, "").split(".").reduce((current, segment) => {
12
+ if (!isRecord(current) && !Array.isArray(current)) {
13
+ return void 0;
14
+ }
15
+ return current[segment];
16
+ }, obj);
17
+ }
18
+ function buildPayload(template, vars) {
19
+ let raw = template;
20
+ for (const [key, val] of Object.entries(vars)) {
21
+ const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
22
+ raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
23
+ raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
24
+ }
25
+ try {
26
+ return JSON.parse(raw);
27
+ } catch (err) {
28
+ throw new Error(`Failed to parse payload template: ${err.message}
29
+ Raw payload: ${raw}`);
30
+ }
31
+ }
32
+ function mergeDefined(base, override) {
33
+ const merged = __spreadValues({}, base || {});
34
+ if (!override) {
35
+ return merged;
36
+ }
37
+ for (const [key, value] of Object.entries(override)) {
38
+ if (value !== void 0 && value !== null && value !== "") {
39
+ merged[key] = value;
40
+ }
41
+ }
42
+ return merged;
43
+ }
44
+
45
+ export {
46
+ resolvePath,
47
+ buildPayload,
48
+ mergeDefined
49
+ };
@@ -31,7 +31,7 @@ var PineconeProvider = class extends BaseVectorProvider {
31
31
  var _a;
32
32
  await this.index(namespace).upsert({
33
33
  records: [{
34
- id: doc.id,
34
+ id: String(doc.id),
35
35
  values: doc.vector,
36
36
  metadata: __spreadValues({ content: doc.content }, (_a = doc.metadata) != null ? _a : {})
37
37
  }]
@@ -43,7 +43,7 @@ var PineconeProvider = class extends BaseVectorProvider {
43
43
  const records = docs.slice(i, i + BATCH).map((d) => {
44
44
  var _a;
45
45
  return {
46
- id: d.id,
46
+ id: String(d.id),
47
47
  values: d.vector,
48
48
  metadata: __spreadValues({ content: d.content }, (_a = d.metadata) != null ? _a : {})
49
49
  };
@@ -69,7 +69,7 @@ var PineconeProvider = class extends BaseVectorProvider {
69
69
  });
70
70
  }
71
71
  async delete(id, namespace) {
72
- await this.index(namespace).deleteOne({ id });
72
+ await this.index(namespace).deleteOne({ id: String(id) });
73
73
  }
74
74
  async deleteNamespace(namespace) {
75
75
  await this.client.index(this.indexName).namespace(namespace).deleteAll();
@@ -1,44 +1,3 @@
1
- import { e as RagConfig, b as ChatResponse } from '../Pipeline-Bo6CUCox.mjs';
2
- import { NextRequest, NextResponse } from 'next/server';
3
-
4
- /**
5
- * createChatHandler — factory that returns a Next.js App Router POST handler
6
- */
7
- declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
8
- error: string;
9
- }> | NextResponse<ChatResponse>>;
10
- /**
11
- * createIngestHandler — factory for the document ingestion endpoint.
12
- */
13
- declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
14
- error: string;
15
- }> | NextResponse<{
16
- results: {
17
- docId: string;
18
- chunksIngested: number;
19
- }[];
20
- }>>;
21
- /**
22
- * createHealthHandler — factory for the health-check endpoint.
23
- */
24
- declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
25
- timestamp: string;
26
- status: string;
27
- }> | NextResponse<{
28
- status: string;
29
- error: string;
30
- }>>;
31
- /**
32
- * createUploadHandler — factory for the file upload ingestion endpoint.
33
- */
34
- declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
35
- error: string;
36
- }> | NextResponse<{
37
- message: string;
38
- results: {
39
- docId: string;
40
- chunksIngested: number;
41
- }[];
42
- }>>;
43
-
44
- export { createChatHandler, createHealthHandler, createIngestHandler, createUploadHandler };
1
+ export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-Ck2pt7-8.mjs';
2
+ import '../RagConfig-hBGXJmSx.mjs';
3
+ import 'next/server';
@@ -1,44 +1,3 @@
1
- import { e as RagConfig, b as ChatResponse } from '../Pipeline-Bo6CUCox.js';
2
- import { NextRequest, NextResponse } from 'next/server';
3
-
4
- /**
5
- * createChatHandler — factory that returns a Next.js App Router POST handler
6
- */
7
- declare function createChatHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
8
- error: string;
9
- }> | NextResponse<ChatResponse>>;
10
- /**
11
- * createIngestHandler — factory for the document ingestion endpoint.
12
- */
13
- declare function createIngestHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
14
- error: string;
15
- }> | NextResponse<{
16
- results: {
17
- docId: string;
18
- chunksIngested: number;
19
- }[];
20
- }>>;
21
- /**
22
- * createHealthHandler — factory for the health-check endpoint.
23
- */
24
- declare function createHealthHandler(config?: Partial<RagConfig>): () => Promise<NextResponse<{
25
- timestamp: string;
26
- status: string;
27
- }> | NextResponse<{
28
- status: string;
29
- error: string;
30
- }>>;
31
- /**
32
- * createUploadHandler — factory for the file upload ingestion endpoint.
33
- */
34
- declare function createUploadHandler(config?: Partial<RagConfig>): (req: NextRequest) => Promise<NextResponse<{
35
- error: string;
36
- }> | NextResponse<{
37
- message: string;
38
- results: {
39
- docId: string;
40
- chunksIngested: number;
41
- }[];
42
- }>>;
43
-
44
- export { createChatHandler, createHealthHandler, createIngestHandler, createUploadHandler };
1
+ export { c as createChatHandler, a as createHealthHandler, b as createIngestHandler, d as createUploadHandler } from '../index-Bx182KKn.js';
2
+ import '../RagConfig-hBGXJmSx.js';
3
+ import 'next/server';