@retrivora-ai/rag-engine 1.8.1 → 1.8.2

package/dist/{chunk-BFYLQYQU.mjs → chunk-LZVVLSDN.mjs}

@@ -1,3 +1,6 @@
+import {
+  resolveMetadataValue
+} from "./chunk-ICKRMZQK.mjs";
 import {
   buildPayload,
   mergeDefined,
@@ -1348,7 +1351,7 @@ var ProviderRegistry = class {
       }
       case "pgvector":
       case "postgresql": {
-        const { MultiTablePostgresProvider } = await import("./MultiTablePostgresProvider-YY7LPNJK.mjs");
+        const { MultiTablePostgresProvider } = await import("./MultiTablePostgresProvider-ZLGSKTJR.mjs");
         return MultiTablePostgresProvider;
       }
       case "mongodb": {
@@ -2380,28 +2383,6 @@ var UITransformer = class {
       data: pieData
     };
   }
-  /**
-   * Transform data to bar chart format
-   */
-  static transformToBarChart(data) {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-    const barData = Object.entries(categoryData).map(([category, value]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(category, data);
-      return {
-        category,
-        value,
-        inStockCount,
-        outOfStockCount
-      };
-    });
-    return {
-      type: "bar_chart",
-      title: "Comparison by Category",
-      description: `Comparing ${categories.length} categories`,
-      data: barData
-    };
-  }
   /**
    * Transform data to line chart format
    */
@@ -2680,15 +2661,7 @@ var UITransformer = class {
       const trainedKey = trainedSchema[uiKey];
       if (trainedKey && meta[trainedKey] !== void 0) return meta[trainedKey];
     }
-    const synonyms = this.SYNONYMS[uiKey] || [];
-    const searchKeys = [uiKey, ...synonyms].map((k) => k.toLowerCase());
-    const foundDirectKey = Object.keys(meta).find((k) => searchKeys.includes(k.toLowerCase()));
-    if (foundDirectKey) return meta[foundDirectKey];
-    const foundPartialKey = Object.keys(meta).find((k) => {
-      const keyLow = k.toLowerCase();
-      return searchKeys.some((sk) => keyLow.includes(sk) || sk.includes(keyLow));
-    });
-    return foundPartialKey ? meta[foundPartialKey] : void 0;
+    return resolveMetadataValue(meta, uiKey);
   }
   static extractProductInfo(item, config, trainedSchema) {
     const meta = item.metadata || {};
@@ -2758,7 +2731,7 @@ var UITransformer = class {
     data.forEach((item) => {
       const meta = item.metadata || {};
       const itemCategory = meta.category || meta.type || "Other";
-      if (result.hasOwnProperty(itemCategory)) {
+      if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
       } else {
         result["Other"] = (result["Other"] || 0) + 1;
@@ -2979,18 +2952,6 @@ IMPORTANT:
     return JSON.stringify(partial, null, 2) + "\n// ... truncated";
   }
 };
-/**
- * Central dictionary of common synonyms for UI properties.
- * This allows the system to be schema-agnostic by guessing field names.
- */
-UITransformer.SYNONYMS = {
-  name: ["product", "item", "title", "label", "heading", "subject"],
-  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
-  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
-  image: ["imageUrl", "thumbnail", "img", "url", "photo", "picture", "media", "image_url", "main_image", "product_image", "thumb"],
-  stock: ["inventory", "quantity", "count", "availability", "stock_level", "inStock", "is_available"],
-  description: ["summary", "content", "body", "text", "info", "details"]
-};
 
 // src/utils/SchemaMapper.ts
 var SchemaMapper = class {
@@ -3111,6 +3072,57 @@ var LRUEmbeddingCache = class {
     return this.cache.size;
   }
 };
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+var MODEL_COST_PER_1K = {
+  "gpt-4o": { input: 25e-4, output: 0.01 },
+  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
+  "gpt-4-turbo": { input: 0.01, output: 0.03 },
+  "gpt-3.5-turbo": { input: 5e-4, output: 15e-4 },
+  "claude-3-5-sonnet": { input: 3e-3, output: 0.015 },
+  "claude-3-haiku": { input: 25e-5, output: 125e-5 },
+  "gemini-1.5-flash": { input: 75e-6, output: 3e-4 },
+  "gemini-1.5-pro": { input: 125e-5, output: 5e-3 }
+};
+function estimateCostUsd(promptTokens, completionTokens, model) {
+  if (!model) return void 0;
+  const key = Object.keys(MODEL_COST_PER_1K).find((k) => model.toLowerCase().includes(k));
+  if (!key) return void 0;
+  const prices = MODEL_COST_PER_1K[key];
+  return promptTokens / 1e3 * prices.input + completionTokens / 1e3 * prices.output;
+}
+async function scoreHallucination(llm, answer, context) {
+  const maxContextChars = 3e3;
+  const truncatedContext = context.length > maxContextChars ? context.slice(0, maxContextChars) + "\n...[truncated]" : context;
+  const prompt = `You are an AI quality checker. Given the CONTEXT retrieved from a knowledge base and the ANSWER generated from it, rate how well-grounded the answer is.
+
+CONTEXT:
+${truncatedContext}
+
+ANSWER:
+${answer}
+
+Return ONLY a valid JSON object with no markdown fences:
+{"score": <float 0-1>, "reason": "<one sentence>"}
+
+Where score 0 = fully grounded in context, score 1 = likely hallucinated or unsupported.`;
+  try {
+    const raw = await llm.chat(
+      [{ role: "user", content: prompt }],
+      "",
+      { temperature: 0, maxTokens: 120 }
+    );
+    const jsonMatch = raw.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return void 0;
+    const parsed = JSON.parse(jsonMatch[0]);
+    if (typeof parsed.score === "number" && typeof parsed.reason === "string") {
+      return { score: Math.min(1, Math.max(0, parsed.score)), reason: parsed.reason };
+    }
+  } catch (e) {
+  }
+  return void 0;
+}
 var Pipeline = class {
   constructor(config) {
     this.config = config;
@@ -3137,8 +3149,7 @@ var Pipeline = class {
   async initialize() {
     var _a;
     if (this.initialised) return;
-    const chartInstruction = `
-You are a helpful product assistant. Use the provided context to answer questions accurately.
+    const chartInstruction = `You are a helpful product assistant. Use the provided context to answer questions accurately.
 
 ### UI STYLE RULES (CRITICAL):
 - NEVER generate markdown tables. If you do, the UI will break.
@@ -3152,7 +3163,6 @@ You are a helpful product assistant. Use the provided context to answer question
 - Do NOT try to format product lists as tables.
 `;
     this.config.llm.systemPrompt = chartInstruction;
-    console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
     const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
       this.config.llm,
@@ -3174,7 +3184,6 @@ You are a helpful product assistant. Use the provided context to answer question
   }
   /**
    * Ingest documents with automatic chunking, embedding, and batch upsert.
-   * Handles retries for transient failures.
    */
   async ingest(documents, namespace) {
     await this.initialize();
@@ -3191,10 +3200,7 @@ You are a helpful product assistant. Use the provided context to answer question
           metadata: chunk.metadata
         }));
         const totalProcessed = await this.processUpserts(upsertDocs, ns);
-        results.push({
-          docId: doc.docId,
-          chunksIngested: totalProcessed
-        });
+        results.push({ docId: doc.docId, chunksIngested: totalProcessed });
         if (this.graphDB && this.entityExtractor) {
           await this.processGraphIngestion(doc.docId, chunks);
         }
@@ -3205,23 +3211,18 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return results;
   }
-  /**
-   * Step 1: Chunk the document content.
-   */
+  /** Step 1: Chunk the document content. */
   async prepareChunks(doc) {
     var _a, _b;
     if (this.llamaIngestor) {
-      return await this.llamaIngestor.chunk(doc.content, {
+      return this.llamaIngestor.chunk(doc.content, {
         docId: doc.docId,
         metadata: doc.metadata,
         chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
         chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
       });
     }
-    return this.chunker.chunk(doc.content, {
-      docId: doc.docId,
-      metadata: doc.metadata
-    });
+    return this.chunker.chunk(doc.content, { docId: doc.docId, metadata: doc.metadata });
   }
   /**
    * Step 2: Generate embeddings for chunks with retry logic.
@@ -3245,9 +3246,7 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return vectors;
   }
-  /**
-   * Step 3: Upsert chunks to vector database with retry logic.
-   */
+  /** Step 3: Upsert chunks to vector database with retry logic. */
   async processUpserts(upsertDocs, namespace) {
     const upsertBatchOptions = {
       batchSize: 100,
@@ -3264,14 +3263,10 @@ You are a helpful product assistant. Use the provided context to answer question
     }
     return upsertResult.totalProcessed;
   }
-  /**
-   * Step 4: Optional graph-based entity extraction and ingestion.
-   */
+  /** Step 4: Optional graph-based entity extraction and ingestion. */
   async processGraphIngestion(docId, chunks) {
-    console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
     const extractionOptions = {
       batchSize: 2,
-      // Low concurrency for LLM extraction
       maxRetries: 1,
       initialDelayMs: 500
     };
@@ -3284,7 +3279,7 @@ You are a helpful product assistant. Use the provided context to answer question
             if (nodes.length > 0) await this.graphDB.addNodes(nodes);
             if (edges.length > 0) await this.graphDB.addEdges(edges);
           } catch (err) {
-            console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
+            console.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}:`, err);
           }
         }
       },
@@ -3295,7 +3290,6 @@ You are a helpful product assistant. Use the provided context to answer question
     var _a;
     await this.initialize();
     if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
-      console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
       const agentReply = await this.agent.run(question, history);
       return { reply: agentReply, sources: [] };
     }
@@ -3304,6 +3298,7 @@ You are a helpful product assistant. Use the provided context to answer question
     let sources = [];
     let graphData;
     let uiTransformation;
+    let trace;
     try {
       for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
         const chunk = temp.value;
@@ -3313,6 +3308,7 @@ You are a helpful product assistant. Use the provided context to answer question
           if ("sources" in chunk) sources = chunk.sources;
           if ("graphData" in chunk) graphData = chunk.graphData;
           if ("ui_transformation" in chunk) uiTransformation = chunk.ui_transformation;
+          if ("trace" in chunk) trace = chunk.trace;
         }
       }
     } catch (temp) {
@@ -3325,38 +3321,47 @@ You are a helpful product assistant. Use the provided context to answer question
           throw error[0];
       }
     }
-    return { reply, sources, graphData, ui_transformation: uiTransformation };
+    return { reply, sources, graphData, ui_transformation: uiTransformation, trace };
   }
   /**
    * High-performance streaming RAG flow.
-   * Yields text chunks first, then the retrieval metadata at the end.
+   * Yields text chunks first, then the retrieval metadata + observability trace at the end.
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f, _g;
+      var _a, _b, _c, _d, _e, _f, _g, _h, _i;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
       const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
+      const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
+      const requestStart = performance.now();
       try {
         let searchQuery = question;
+        let rewrittenQuery;
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
+          rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
         }
         const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
-        console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
+        const embedStart = performance.now();
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
           topK: topK * 2,
           filter
         }));
+        const retrieveEnd = performance.now();
+        const embedMs = retrieveEnd - embedStart;
+        const retrieveMs = retrieveEnd - embedStart;
+        const rerankStart = performance.now();
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
         if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);
         }
+        const rerankMs = performance.now() - rerankStart;
         let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
 ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (graphData && graphData.nodes.length > 0) {
@@ -3375,6 +3380,10 @@ ${context}`;
         const hardenedHistory = [...history];
         const userQuestion = { role: "user", content: question + restrictionSuffix };
         const messages = [...hardenedHistory, userQuestion];
+        const systemPrompt = (_h = this.config.llm.systemPrompt) != null ? _h : "";
+        const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
+        let fullReply = "";
+        const generateStart = performance.now();
         if (this.llmProvider.chatStream) {
           const stream = this.llmProvider.chatStream(messages, context);
           if (!stream) {
@@ -3383,6 +3392,7 @@ ${context}`;
           try {
             for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
               const chunk = temp.value;
+              fullReply += chunk;
               yield chunk;
             }
           } catch (temp) {
@@ -3397,15 +3407,58 @@ ${context}`;
           }
         } else {
           const reply = yield new __await(this.llmProvider.chat(messages, context));
+          fullReply = reply;
           yield reply;
         }
+        const generateMs = performance.now() - generateStart;
+        const totalMs = performance.now() - requestStart;
+        const latency = {
+          embedMs: Math.round(embedMs),
+          retrieveMs: Math.round(retrieveMs),
+          rerankMs: ((_i = this.config.rag) == null ? void 0 : _i.useReranking) ? Math.round(rerankMs) : void 0,
+          generateMs: Math.round(generateMs),
+          totalMs: Math.round(totalMs)
+        };
+        const promptText = systemPrompt + "\n" + context + "\n" + userPrompt;
+        const promptTokens = estimateTokens(promptText);
+        const completionTokens = estimateTokens(fullReply);
+        const tokens = {
+          promptTokens,
+          completionTokens,
+          totalTokens: promptTokens + completionTokens,
+          estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
+        };
+        const hallucinationResult = yield new __await(scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0));
         const trainedSchema = yield new __await(trainingPromise);
         const uiTransformation = yield new __await(this.generateUiTransformation(question, sources, context, trainedSchema));
+        const trace = {
+          requestId,
+          query: question,
+          rewrittenQuery,
+          systemPrompt,
+          userPrompt: question + restrictionSuffix,
+          chunks: sources.map((s) => {
+            var _a2;
+            return {
+              id: s.id,
+              score: s.score,
+              content: s.content,
+              metadata: (_a2 = s.metadata) != null ? _a2 : {},
+              namespace: ns
+            };
+          }),
+          latency,
+          tokens,
+          hallucinationScore: hallucinationResult == null ? void 0 : hallucinationResult.score,
+          hallucinationReason: hallucinationResult == null ? void 0 : hallucinationResult.reason,
+          timestamp: (/* @__PURE__ */ new Date()).toISOString()
+        };
         yield {
           reply: "",
           sources,
           graphData,
-          ui_transformation: uiTransformation
+          ui_transformation: uiTransformation,
+          trace
         };
       } catch (error2) {
         throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
@@ -3444,12 +3497,9 @@ ${context}`;
     const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
     return { sources, graphData };
   }
-  /**
-   * Rewrite the user query for better retrieval performance.
-   */
+  /** Rewrite the user query for better retrieval performance. */
   async rewriteQuery(question, history) {
-    const prompt = `
-Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
+    const prompt = `Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database. 
 Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
 
 History:
@@ -3467,23 +3517,16 @@ Optimized Search Query:`;
     );
     return rewrite.trim() || question;
   }
-  /**
-   * Generate 3-5 short, relevant questions based on the vector database content.
-   */
+  /** Generate 3-5 short, relevant questions based on the vector database content. */
   async getSuggestions(query, namespace) {
-    if (!query || query.trim().length < 3) {
-      return [];
-    }
+    if (!query || query.trim().length < 3) return [];
     await this.initialize();
     const ns = namespace != null ? namespace : this.config.projectId;
     try {
       const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
-      if (sources.length === 0) {
-        return [];
-      }
+      if (sources.length === 0) return [];
       const context = sources.map((s) => s.content).join("\n\n---\n\n");
-      const prompt = `
-Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
+      const prompt = `Based on the following snippets from a document, what are 3 short, relevant questions a user might ask? 
 Focus on questions that can be answered by the context.
 Keep each question under 10 words and make them very specific to the content.
 Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
@@ -3760,6 +3803,11 @@ function sseMetaFrame(meta) {
 function sseUIFrame(uiTransformation) {
   return `data: ${JSON.stringify({ type: "ui_transformation", data: uiTransformation })}
 
+`;
+}
+function sseObservabilityFrame(trace) {
+  return `data: ${JSON.stringify({ type: "observability", data: trace })}
+
 `;
 }
 function sseErrorFrame(message) {
@@ -3826,6 +3874,9 @@ function createStreamHandler(configOrPlugin) {
                 enqueue(sseMetaFrame(chunk));
                 const responseChunk = chunk;
                 const sources = (responseChunk == null ? void 0 : responseChunk.sources) || [];
+                if (responseChunk == null ? void 0 : responseChunk.trace) {
+                  enqueue(sseObservabilityFrame(responseChunk.trace));
+                }
                 if (sources.length > 0) {
                   try {
                     const llmProvider = (_a = plugin.getLLMProvider) == null ? void 0 : _a.call(plugin);
@@ -3907,24 +3958,64 @@ function createUploadHandler(configOrPlugin) {
       const formData = await req.formData();
       const files = formData.getAll("files");
       const namespace = formData.get("namespace") || void 0;
+      const dimensionRaw = formData.get("dimension");
+      const dimension = dimensionRaw ? parseInt(dimensionRaw, 10) : void 0;
       if (!files || files.length === 0) {
         return NextResponse.json({ error: "No files provided" }, { status: 400 });
       }
-      const documents = await Promise.all(
-        files.map(async (file) => {
+      const documents = [];
+      for (const file of files) {
+        if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
+          const text = await file.text();
+          const Papa = await import("papaparse");
+          const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
+          if (parsed.data && parsed.data.length > 0) {
+            let i = 0;
+            let lastRowData = null;
+            for (const row of parsed.data) {
+              i++;
+              const rowData = row;
+              const groupingKey = Object.keys(rowData)[0];
+              if (lastRowData && groupingKey && rowData[groupingKey] && rowData[groupingKey] === lastRowData[groupingKey]) {
+                for (const key of Object.keys(rowData)) {
+                  if (!rowData[key] && lastRowData[key]) {
+                    rowData[key] = lastRowData[key];
+                  }
+                }
+              }
+              lastRowData = __spreadValues({}, rowData);
+              const contentParts = [];
+              for (const [key, val] of Object.entries(rowData)) {
+                if (key && val) {
+                  contentParts.push(`${key}: ${val}`);
+                }
+              }
+              documents.push({
+                docId: `${file.name}-row-${i}`,
+                content: contentParts.join(", "),
+                metadata: __spreadValues(__spreadValues({
+                  fileName: file.name,
+                  fileSize: file.size,
+                  fileType: file.type,
+                  uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
+                }, dimension ? { dimension } : {}), rowData)
+              });
+            }
+          }
+        } else {
           const content = await DocumentParser.parse(file, file.name, file.type);
-          return {
+          documents.push({
             docId: file.name,
             content,
-            metadata: {
+            metadata: __spreadValues({
               fileName: file.name,
               fileSize: file.size,
               fileType: file.type,
               uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
-            }
-          };
-        })
-      );
+            }, dimension ? { dimension } : {})
+          });
+        }
+      }
       const results = await plugin.ingest(documents, namespace);
       return NextResponse.json({ message: "Upload successful", results });
     } catch (err) {
@@ -3975,6 +4066,7 @@ export {
   sseTextFrame,
   sseMetaFrame,
   sseUIFrame,
+  sseObservabilityFrame,
   sseErrorFrame,
   createChatHandler,
   createStreamHandler,