@retrivora-ai/rag-engine 0.4.4 → 0.4.5

package/dist/{MongoDBProvider-ZKW34AEL.mjs → MongoDBProvider-RE3Q5S5B.mjs}

@@ -1,6 +1,6 @@
 import {
   MongoDBProvider
-} from "./chunk-IWHCAQEA.mjs";
+} from "./chunk-PQKTC73Y.mjs";
 import "./chunk-IMP6FUCY.mjs";
 import "./chunk-FWCSY2DS.mjs";
 export {

package/dist/{chunk-IWHCAQEA.mjs → chunk-PQKTC73Y.mjs}

@@ -77,7 +77,7 @@ var MongoDBProvider = class extends BaseVectorProvider {
     return results.map((res) => ({
       id: res._id,
       content: res[this.contentKey],
-      metadata: res[this.metadataKey],
+      metadata: res[this.metadataKey] || {},
       score: res.score
     }));
   }

package/dist/{chunk-OKY5P6RA.mjs → chunk-PRC5CZIZ.mjs}

@@ -864,7 +864,7 @@ Output JSON:
     try {
       const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
       return JSON.parse(cleanJson);
-    } catch (error) {
+    } catch (e) {
       console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
       return { nodes: [], edges: [] };
     }
@@ -1443,7 +1443,7 @@ var ProviderRegistry = class {
         return new PostgreSQLProvider(config);
       }
       case "mongodb": {
-        const { MongoDBProvider } = await import("./MongoDBProvider-ZKW34AEL.mjs");
+        const { MongoDBProvider } = await import("./MongoDBProvider-RE3Q5S5B.mjs");
         return new MongoDBProvider(config);
       }
       case "milvus": {
@@ -1998,12 +1998,28 @@ var Pipeline = class {
           chunksIngested: upsertResult.totalProcessed
         });
         if (this.graphDB && this.entityExtractor) {
-          console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
-          for (const chunk of chunks) {
-            const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
-            if (nodes.length > 0) await this.graphDB.addNodes(nodes);
-            if (edges.length > 0) await this.graphDB.addEdges(edges);
-          }
+          console.log(`[Pipeline] Extracting entities for doc ${doc.docId} (${chunks.length} chunks)...`);
+          const extractionOptions = {
+            batchSize: 2,
+            // Low concurrency for LLM extraction
+            maxRetries: 1,
+            initialDelayMs: 500
+          };
+          await BatchProcessor.processBatch(
+            chunks,
+            async (batch) => {
+              for (const chunk of batch) {
+                try {
+                  const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
+                  if (nodes.length > 0) await this.graphDB.addNodes(nodes);
+                  if (edges.length > 0) await this.graphDB.addEdges(edges);
+                } catch (err) {
+                  console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+                }
+              }
+            },
+            extractionOptions
+          );
         }
       } catch (error) {
         console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
@@ -2599,7 +2615,7 @@ var DocumentParser = class {
     }
     if (extension === "pdf" || mimeType === "application/pdf") {
       try {
-        const pdf = await import("pdf-parse/lib/pdf-parse.js");
+        const pdf = await import("pdf-parse");
         const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
         const data = await pdf.default(buffer);
         return data.text;

package/dist/handlers/index.js

@@ -445,7 +445,7 @@ var init_MongoDBProvider = __esm({
         return results.map((res) => ({
           id: res._id,
           content: res[this.contentKey],
-          metadata: res[this.metadataKey],
+          metadata: res[this.metadataKey] || {},
           score: res.score
         }));
       }
@@ -2135,7 +2135,7 @@ Output JSON:
     try {
       const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
       return JSON.parse(cleanJson);
-    } catch (error) {
+    } catch (e) {
       console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
       return { nodes: [], edges: [] };
     }
@@ -3225,12 +3225,28 @@ var Pipeline = class {
           chunksIngested: upsertResult.totalProcessed
         });
         if (this.graphDB && this.entityExtractor) {
-          console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
-          for (const chunk of chunks) {
-            const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
-            if (nodes.length > 0) await this.graphDB.addNodes(nodes);
-            if (edges.length > 0) await this.graphDB.addEdges(edges);
-          }
+          console.log(`[Pipeline] Extracting entities for doc ${doc.docId} (${chunks.length} chunks)...`);
+          const extractionOptions = {
+            batchSize: 2,
+            // Low concurrency for LLM extraction
+            maxRetries: 1,
+            initialDelayMs: 500
+          };
+          await BatchProcessor.processBatch(
+            chunks,
+            async (batch) => {
+              for (const chunk of batch) {
+                try {
+                  const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
+                  if (nodes.length > 0) await this.graphDB.addNodes(nodes);
+                  if (edges.length > 0) await this.graphDB.addEdges(edges);
+                } catch (err) {
+                  console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+                }
+              }
+            },
+            extractionOptions
+          );
         }
       } catch (error) {
         console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
@@ -3826,7 +3842,7 @@ var DocumentParser = class {
     }
     if (extension === "pdf" || mimeType === "application/pdf") {
       try {
-        const pdf = await import("pdf-parse/lib/pdf-parse.js");
+        const pdf = await import("pdf-parse");
         const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
         const data = await pdf.default(buffer);
         return data.text;

package/dist/handlers/index.mjs

@@ -3,7 +3,7 @@ import {
   createHealthHandler,
   createIngestHandler,
   createUploadHandler
-} from "../chunk-OKY5P6RA.mjs";
+} from "../chunk-PRC5CZIZ.mjs";
 import "../chunk-EDLTMSNY.mjs";
 import "../chunk-FWCSY2DS.mjs";
 export {

package/dist/index.d.mts

@@ -27,7 +27,7 @@ interface DocumentUploadProps {
     /** Optional namespace for the upload */
     namespace?: string;
     /** Callback when upload completes */
-    onUploadComplete?: (results: any) => void;
+    onUploadComplete?: (results: unknown) => void;
     /** Additional className */
     className?: string;
 }

package/dist/index.d.ts

@@ -27,7 +27,7 @@ interface DocumentUploadProps {
     /** Optional namespace for the upload */
     namespace?: string;
     /** Callback when upload completes */
-    onUploadComplete?: (results: any) => void;
+    onUploadComplete?: (results: unknown) => void;
     /** Additional className */
     className?: string;
 }

package/dist/server.js

@@ -457,7 +457,7 @@ var init_MongoDBProvider = __esm({
         return results.map((res) => ({
           id: res._id,
           content: res[this.contentKey],
-          metadata: res[this.metadataKey],
+          metadata: res[this.metadataKey] || {},
           score: res.score
         }));
       }
@@ -2179,7 +2179,7 @@ Output JSON:
     try {
       const cleanJson = response.replace(/```json\n?|\n?```/g, "").trim();
       return JSON.parse(cleanJson);
-    } catch (error) {
+    } catch (e) {
       console.warn("[EntityExtractor] Failed to parse LLM response as JSON:", response);
       return { nodes: [], edges: [] };
     }
@@ -3314,12 +3314,28 @@ var Pipeline = class {
           chunksIngested: upsertResult.totalProcessed
         });
         if (this.graphDB && this.entityExtractor) {
-          console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
-          for (const chunk of chunks) {
-            const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
-            if (nodes.length > 0) await this.graphDB.addNodes(nodes);
-            if (edges.length > 0) await this.graphDB.addEdges(edges);
-          }
+          console.log(`[Pipeline] Extracting entities for doc ${doc.docId} (${chunks.length} chunks)...`);
+          const extractionOptions = {
+            batchSize: 2,
+            // Low concurrency for LLM extraction
+            maxRetries: 1,
+            initialDelayMs: 500
+          };
+          await BatchProcessor.processBatch(
+            chunks,
+            async (batch) => {
+              for (const chunk of batch) {
+                try {
+                  const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
+                  if (nodes.length > 0) await this.graphDB.addNodes(nodes);
+                  if (edges.length > 0) await this.graphDB.addEdges(edges);
+                } catch (err) {
+                  console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+                }
+              }
+            },
+            extractionOptions
+          );
         }
       } catch (error) {
         console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
@@ -4188,7 +4204,7 @@ var DocumentParser = class {
     }
     if (extension === "pdf" || mimeType === "application/pdf") {
       try {
-        const pdf = await import("pdf-parse/lib/pdf-parse.js");
+        const pdf = await import("pdf-parse");
         const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
         const data = await pdf.default(buffer);
         return data.text;

package/dist/server.mjs

@@ -34,7 +34,7 @@ import {
   createIngestHandler,
   createUploadHandler,
   getRagConfig
-} from "./chunk-OKY5P6RA.mjs";
+} from "./chunk-PRC5CZIZ.mjs";
 import "./chunk-EDLTMSNY.mjs";
 import {
   PineconeProvider
@@ -44,7 +44,7 @@ import {
 } from "./chunk-LJWWPTWE.mjs";
 import {
   MongoDBProvider
-} from "./chunk-IWHCAQEA.mjs";
+} from "./chunk-PQKTC73Y.mjs";
 import {
   MilvusProvider
 } from "./chunk-3QWAK3RZ.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "0.4.4",
+  "version": "0.4.5",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",

package/src/components/DocumentUpload.tsx

@@ -8,7 +8,7 @@ interface DocumentUploadProps {
   /** Optional namespace for the upload */
   namespace?: string;
   /** Callback when upload completes */
-  onUploadComplete?: (results: any) => void;
+  onUploadComplete?: (results: unknown) => void;
   /** Additional className */
   className?: string;
 }

package/src/core/Pipeline.ts

@@ -282,13 +282,31 @@ export class Pipeline {
         });
 
         // Graph ingestion
+        // Graph ingestion - Batched to prevent timeouts and LLM overload
         if (this.graphDB && this.entityExtractor) {
-          console.log(`[Pipeline] Extracting entities for doc ${doc.docId}...`);
-          for (const chunk of chunks) {
-            const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
-            if (nodes.length > 0) await this.graphDB.addNodes(nodes);
-            if (edges.length > 0) await this.graphDB.addEdges(edges);
-          }
+          console.log(`[Pipeline] Extracting entities for doc ${doc.docId} (${chunks.length} chunks)...`);
+          
+          const extractionOptions: BatchOptions = {
+            batchSize: 2, // Low concurrency for LLM extraction
+            maxRetries: 1,
+            initialDelayMs: 500,
+          };
+
+          await BatchProcessor.processBatch(
+            chunks,
+            async (batch) => {
+              for (const chunk of batch) {
+                try {
+                  const { nodes, edges } = await this.entityExtractor!.extract(chunk.content);
+                  if (nodes.length > 0) await this.graphDB!.addNodes(nodes);
+                  if (edges.length > 0) await this.graphDB!.addEdges(edges);
+                } catch (err) {
+                  console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+                }
+              }
+            },
+            extractionOptions
+          );
         }
       } catch (error) {
         console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);

package/src/providers/vectordb/MongoDBProvider.ts

@@ -94,10 +94,10 @@ export class MongoDBProvider extends BaseVectorProvider {
 
     const results = await this.collection!.aggregate(pipeline).toArray();
 
-    return results.map((res: any) => ({
+    return (results as unknown as Array<{ _id: string; score: number; [key: string]: unknown }>).map((res) => ({
       id: res._id,
-      content: res[this.contentKey],
-      metadata: res[this.metadataKey],
+      content: res[this.contentKey] as string,
+      metadata: (res[this.metadataKey] as Record<string, unknown>) || {},
       score: res.score,
     }));
   }

package/src/rag/EntityExtractor.ts

@@ -35,7 +35,7 @@ Output JSON:
       // Clean up LLM response (sometimes it adds markdown blocks)
       const cleanJson = response.replace(/```json\n?|\n?```/g, '').trim();
       return JSON.parse(cleanJson);
-    } catch (error) {
+    } catch {
       console.warn('[EntityExtractor] Failed to parse LLM response as JSON:', response);
       return { nodes: [], edges: [] };
     }

package/src/utils/DocumentParser.ts

@@ -35,7 +35,7 @@ export class DocumentParser {
     if (extension === 'pdf' || mimeType === 'application/pdf') {
        try {
          // Dynamic import to avoid errors if not installed
-         const pdf = await import('pdf-parse/lib/pdf-parse.js');
+         const pdf = await import('pdf-parse');
          const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await (file as File).arrayBuffer());
          const data = await pdf.default(buffer);
          return data.text;