@retrivora-ai/rag-engine 2.0.6 → 2.0.7

package/dist/server.js

@@ -4034,9 +4034,9 @@ ${context != null ? context : "None"}` },
     }
     const isSelfHost = this.baseUrl.includes("retrivora.com") || this.baseUrl.includes("localhost");
     if (isSelfHost || Boolean(process.env.VERCEL)) {
-      try {
-        const _g2 = globalThis;
-        if (typeof _g2.__retrivoraDispatchChat === "function") {
+      const _g2 = globalThis;
+      if (typeof _g2.__retrivoraDispatchChat === "function") {
+        try {
           const res = await _g2.__retrivoraDispatchChat({
             model: this.model,
             messages: formattedMessages,
@@ -4046,8 +4046,9 @@ ${context != null ? context : "None"}` },
           if ((_e = (_d = (_c = (_b = res == null ? void 0 : res.response) == null ? void 0 : _b.choices) == null ? void 0 : _c[0]) == null ? void 0 : _d.message) == null ? void 0 : _e.content) {
             return res.response.choices[0].message.content;
           }
+        } catch (dispatchErr) {
+          throw dispatchErr;
         }
-      } catch (e) {
       }
     }
     const { data } = await this.http.post(path2, payload);
@@ -4099,9 +4100,9 @@ ${context != null ? context : "None"}` },
       const isSelfHost = this.baseUrl.includes("retrivora.com") || this.baseUrl.includes("localhost");
       let streamBody = null;
       if (isSelfHost || Boolean(process.env.VERCEL)) {
-        try {
-          const _g3 = globalThis;
-          if (typeof _g3.__retrivoraDispatchChat === "function") {
+        const _g3 = globalThis;
+        if (typeof _g3.__retrivoraDispatchChat === "function") {
+          try {
             const res = yield new __await(_g3.__retrivoraDispatchChat({
               model: this.model,
               messages: formattedMessages,
@@ -4115,8 +4116,9 @@ ${context != null ? context : "None"}` },
               yield res.response.choices[0].message.content;
               return;
             }
+          } catch (dispatchErr) {
+            throw dispatchErr;
           }
-        } catch (e) {
         }
       }
       if (!streamBody) {
@@ -4187,9 +4189,9 @@ ${context != null ? context : "None"}` },
     }
     const isSelfHost = this.baseUrl.includes("retrivora.com") || this.baseUrl.includes("localhost");
     if (isSelfHost || Boolean(process.env.VERCEL)) {
-      try {
-        const _g2 = globalThis;
-        if (typeof _g2.__retrivoraDispatchEmbedding === "function") {
+      const _g2 = globalThis;
+      if (typeof _g2.__retrivoraDispatchEmbedding === "function") {
+        try {
           const res = await _g2.__retrivoraDispatchEmbedding({
             model: this.model,
             input: text
@@ -4197,8 +4199,9 @@ ${context != null ? context : "None"}` },
           if ((_c = (_b = res == null ? void 0 : res.data) == null ? void 0 : _b[0]) == null ? void 0 : _c.embedding) {
             return res.data[0].embedding;
           }
+        } catch (dispatchErr) {
+          throw dispatchErr;
         }
-      } catch (e) {
       }
     }
     const { data } = await this.http.post(path2, payload);
@@ -4791,7 +4794,8 @@ var Reranker = class {
         return matches.sort((a, b) => b.score - a.score).slice(0, limit);
       }
       const scoredMatches = matches.map((match) => {
-        const contentLower = match.content.toLowerCase();
+        var _a2;
+        const contentLower = ((_a2 = match == null ? void 0 : match.content) != null ? _a2 : "").toLowerCase();
         let keywordScore = 0;
         keywords.forEach((keyword) => {
           if (contentLower.includes(keyword)) {
@@ -4812,7 +4816,10 @@ var Reranker = class {
         [{ role: "user", content: `Query: "${query}"
 
 Documents:
-${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, " ")}`).join("\n")}` }],
+${topN.map((m, i) => {
+          var _a2;
+          return `[${i}] ${((_a2 = m == null ? void 0 : m.content) != null ? _a2 : "").replace(/\n/g, " ")}`;
+        }).join("\n")}` }],
         "",
         {
           systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',
@@ -5983,10 +5990,10 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
           "",
           { temperature: 0, maxTokens: 10 }
         );
-        const cleanResponse = response.trim().toLowerCase();
-        if (cleanResponse.includes("vector")) return "vector";
-        if (cleanResponse.includes("graph")) return "graph";
+        const cleanResponse = (response != null ? response : "").trim().toLowerCase();
         if (cleanResponse.includes("both")) return "both";
+        if (cleanResponse.includes("graph")) return "graph";
+        if (cleanResponse.includes("vector")) return "vector";
       } catch (error) {
         console.warn("[QueryProcessor] LLM strategy classification failed, falling back to keywords:", error);
       }
@@ -6160,7 +6167,10 @@ var IntentClassifier = class {
       hasCategoricalFields = catKeys.size > 0;
       numericFieldCount = numericKeys.size;
       categoricalFieldCount = catKeys.size;
-      if (productKeyMatches >= 2 || docs.some((d) => d.content.toLowerCase().includes("price") || d.content.toLowerCase().includes("brand"))) {
+      if (productKeyMatches >= 2 || docs.some((d) => {
+        var _a2, _b;
+        return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
+      })) {
         isProductLike = true;
       }
     }
@@ -6440,7 +6450,10 @@ var TextRendererStrategy = class {
       return { type: "text", title, data: { content: data } };
     }
     const docs = Array.isArray(data) ? data : [];
-    const text = docs.map((d) => d.content).join("\n\n");
+    const text = docs.map((d) => {
+      var _a2;
+      return (_a2 = d == null ? void 0 : d.content) != null ? _a2 : "";
+    }).join("\n\n");
     return { type: "text", title, data: { content: text || "No detailed content available." } };
   }
 };
@@ -6697,6 +6710,19 @@ var UITransformer = class _UITransformer {
     if (!retrievedData || retrievedData.length === 0) {
       return this.createTextResponse("No data available", "No relevant data found for your query.");
     }
+    console.log("[UITransformer.transform] Processing retrievedData:", {
+      userQuery,
+      retrievedCount: retrievedData.length,
+      sample: retrievedData.slice(0, 2).map((item) => {
+        var _a3;
+        return {
+          id: item == null ? void 0 : item.id,
+          contentType: typeof (item == null ? void 0 : item.content),
+          contentSnippet: String((_a3 = item == null ? void 0 : item.content) != null ? _a3 : "").substring(0, 150),
+          metadataKeys: Object.keys((item == null ? void 0 : item.metadata) || {})
+        };
+      })
+    });
     const resolvedIntent = intent != null ? intent : this.detectIntentHeuristic(userQuery);
     const filteredData = resolvedIntent.filterInStockOnly ? retrievedData.filter((item) => this.determineStockStatus(item)) : retrievedData;
     const profile = this.profileData(filteredData);
@@ -7186,7 +7212,10 @@ ${schemaProfileText}` : ""}`;
       type: "radar_chart",
       title: "Product Comparison",
       description: `Comparing ${data.length} items across ${radarData.length} attributes`,
-      data: radarData.length > 0 ? radarData : data.map((d) => ({ attribute: d.content.substring(0, 40) }))
+      data: radarData.length > 0 ? radarData : data.map((d) => {
+        var _a2;
+        return { attribute: ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").substring(0, 40) };
+      })
     };
   }
   static transformToTable(data, query = "") {
@@ -7203,7 +7232,10 @@ ${schemaProfileText}` : ""}`;
   static transformToText(data) {
     return this.createTextResponse(
       "Retrieved Context",
-      data.map((item) => item.content).join("\n\n"),
+      data.map((item) => {
+        var _a2;
+        return (_a2 = item == null ? void 0 : item.content) != null ? _a2 : "";
+      }).join("\n\n"),
       `Found ${data.length} relevant results`
     );
   }
@@ -7364,6 +7396,7 @@ ${schemaProfileText}` : ""}`;
     return productTerms && productAction && !nonProductEntityTerms;
   }
   static isProductData(item) {
+    if (!item) return false;
     const content = (item.content || "").toLowerCase();
     const productKeywords = [
       "product",
@@ -7622,12 +7655,12 @@ ${schemaProfileText}` : ""}`;
   }
   static extractTimeSeriesData(data) {
     return data.map((item) => {
-      var _a2, _b, _c, _d, _e;
+      var _a2, _b, _c, _d, _e, _f;
       const meta = item.metadata || {};
       return {
         timestamp: (_b = (_a2 = meta.timestamp) != null ? _a2 : meta.date) != null ? _b : (/* @__PURE__ */ new Date()).toISOString(),
         value: (_d = (_c = meta.value) != null ? _c : item.score) != null ? _d : 0,
-        label: (_e = meta.label) != null ? _e : item.content.substring(0, 50)
+        label: (_f = meta.label) != null ? _f : ((_e = item == null ? void 0 : item.content) != null ? _e : "").substring(0, 50)
       };
     });
   }
@@ -7741,7 +7774,8 @@ ${schemaProfileText}` : ""}`;
     }, query.includes(normalizedField) ? 3 : 0);
   }
   static resolveTableCellValue(item, column) {
-    if (column === "Content") return item.content.substring(0, 100);
+    var _a2, _b, _c;
+    if (column === "Content") return ((_a2 = item == null ? void 0 : item.content) != null ? _a2 : "").substring(0, 100);
     const meta = item.metadata || {};
     const normalizedColumn = this.normalizeComparableField(column);
     const exactMetadata = Object.entries(meta).find(
@@ -7752,9 +7786,9 @@ ${schemaProfileText}` : ""}`;
     }
     const aliasValue = this.resolveAliasedTableCell(meta, column);
     if (aliasValue !== void 0) return this.toDisplayValue(aliasValue);
-    const contentValue = this.extractContentFieldValue(item.content, column);
+    const contentValue = this.extractContentFieldValue((_b = item == null ? void 0 : item.content) != null ? _b : "", column);
     if (contentValue !== null) return contentValue;
-    const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
+    const aliasContentValue = this.extractAliasedContentFieldValue((_c = item == null ? void 0 : item.content) != null ? _c : "", column);
     return aliasContentValue != null ? aliasContentValue : "";
   }
   static resolveAliasedTableCell(meta, column) {
@@ -7804,6 +7838,7 @@ ${schemaProfileText}` : ""}`;
     return { inStockCount: inStock, outOfStockCount: outOfStock };
   }
   static determineStockStatus(item) {
+    if (!item) return true;
     const meta = item.metadata || {};
     const stockValue = resolveMetadataValue(meta, "stock");
     if (stockValue !== void 0) {
@@ -7849,35 +7884,64 @@ ${schemaProfileText}` : ""}`;
     return resolveMetadataValue(meta, uiKey);
   }
   static extractProductInfo(item, config, trainedSchema) {
-    var _a2;
+    var _a2, _b;
+    if (!item) return null;
     const meta = item.metadata || {};
+    const content = item.content || "";
     const name = this.getDynamicVal(meta, "name", config, trainedSchema);
     const price = this.getDynamicVal(meta, "price", config, trainedSchema);
     const brand = this.getDynamicVal(meta, "brand", config, trainedSchema);
     const description = this.cleanProductDescription(
-      (_a2 = this.extractProductDescriptionFromContent(item.content)) != null ? _a2 : this.getProductDescriptionValue(meta, config, trainedSchema)
+      (_a2 = this.extractProductDescriptionFromContent(content)) != null ? _a2 : this.getProductDescriptionValue(meta, config, trainedSchema)
     );
     if (name || this.isProductData(item)) {
       let finalName = name ? String(name) : void 0;
+      if (!finalName && content) {
+        const nameMatch = content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+        finalName = nameMatch ? nameMatch[1].trim() : (_b = content.split("\n")[0]) == null ? void 0 : _b.substring(0, 60);
+      }
       if (!finalName) {
-        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-        finalName = nameMatch ? nameMatch[1].trim() : item.content.split("\n")[0].substring(0, 60);
+        console.warn(
+          `[UITransformer] Unable to read product name from metadata or content for item ${item.id}. Defaulting to 'Unnamed Product'.`,
+          { metadataKeys: Object.keys(meta), contentLength: content.length }
+        );
       }
       let finalPrice = typeof price === "number" || typeof price === "string" ? price : void 0;
-      if (!finalPrice) {
-        const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
+      if (!finalPrice && content) {
+        const priceMatch = content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || content.match(/\$\s*([\d,.]+)/);
         if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, "");
       }
+      if (finalPrice === void 0) {
+        console.warn(
+          `[UITransformer] Unable to read product price from metadata or content for item ${item.id}.`,
+          { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) }
+        );
+      }
       const imageValue = this.getDynamicVal(meta, "image", config, trainedSchema);
+      if (!imageValue) {
+        console.debug(
+          `[UITransformer] Unable to read product image URL from metadata for item ${item.id}.`
+        );
+      }
+      if (!description) {
+        console.debug(
+          `[UITransformer] Unable to read product description from metadata or content for item ${item.id}.`
+        );
+      }
       return {
         id: item.id,
-        name: finalName,
+        name: finalName || "Unnamed Product",
         price: finalPrice,
         image: typeof imageValue === "string" ? imageValue : void 0,
         brand: brand ? String(brand) : void 0,
         description,
         inStock: this.determineStockStatus(item)
       };
+    } else {
+      console.warn(
+        `[UITransformer] Item ${item.id} skipped: unable to read name or identify as product data from metadata/content.`,
+        { metadataKeys: Object.keys(meta), contentSnippet: content.substring(0, 100) }
+      );
     }
     return null;
   }
@@ -7962,13 +8026,13 @@ RULES:
 5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
   }
   static buildContextSummary(sources, maxChars = 6e3) {
-    const items = sources.map((s, i) => {
+    const items = (sources || []).filter(Boolean).map((s, i) => {
       var _a2, _b, _c, _d;
       return {
         index: i + 1,
-        content: (_b = (_a2 = s.content) == null ? void 0 : _a2.substring(0, 400)) != null ? _b : "",
-        metadata: (_c = s.metadata) != null ? _c : {},
-        score: (_d = s.score) != null ? _d : 0
+        content: (_b = (_a2 = s == null ? void 0 : s.content) == null ? void 0 : _a2.substring(0, 400)) != null ? _b : "",
+        metadata: (_c = s == null ? void 0 : s.metadata) != null ? _c : {},
+        score: (_d = s == null ? void 0 : s.score) != null ? _d : 0
       };
     });
     const full = JSON.stringify(items, null, 2);
@@ -8537,17 +8601,33 @@ ${m.content}`).join("\n\n---\n\n");
         const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
         const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
         const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
-        const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
+        const rawSources = ((strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : []).filter((s) => Boolean(s && typeof s === "object"));
+        console.log("[Pipeline] Raw vectorDB.query response:", {
+          rawCount: rawSources.length,
+          sample: rawSources.slice(0, 2).map((s) => {
+            var _a3;
+            return {
+              id: s == null ? void 0 : s.id,
+              score: s == null ? void 0 : s.score,
+              contentType: typeof (s == null ? void 0 : s.content),
+              contentSnippet: String((_a3 = s == null ? void 0 : s.content) != null ? _a3 : "").substring(0, 150),
+              metadataKeys: Object.keys((s == null ? void 0 : s.metadata) || {})
+            };
+          })
+        });
         const retrieveEnd = performance.now();
         const embedMs = retrieveEnd - embedStart;
         const retrieveMs = retrieveEnd - embedStart;
         const rerankStart = performance.now();
-        const structuredSources = this.applyStructuredFilters(rawSources, filter);
-        let fullSources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
+        const structuredSources = this.applyStructuredFilters(rawSources, filter).filter((s) => Boolean(s && typeof s === "object"));
+        let fullSources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => {
+          var _a3;
+          return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
+        });
         const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
         const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_m = this.config.rag) == null ? void 0 : _m.useReranking));
         if (!hasNumericPredicates && useReranking) {
-          fullSources = yield new __await(this.reranker.rerank(fullSources, question, rerankLimit));
+          fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
         } else if (!wantsExhaustiveList) {
           fullSources = fullSources.slice(0, topK);
         }
@@ -8562,13 +8642,33 @@ ${(_a3 = m == null ? void 0 : m.content) != null ? _a3 : ""}`;
           displayCount = fullSources.length;
         } else {
           const baseThreshold = Math.max(scoreThreshold, 0.4);
-          const highlyRelevant = fullSources.filter((m) => m.score >= baseThreshold);
+          const highlyRelevant = fullSources.filter((m) => {
+            var _a3;
+            return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= baseThreshold;
+          });
           displayCount = Math.max(highlyRelevant.length, topK);
           if (displayCount > 15) {
             displayCount = 15;
           }
         }
-        const sources = [...fullSources].sort((a, b) => b.score - a.score).slice(0, displayCount);
+        const sources = [...fullSources].filter((s) => Boolean(s && typeof s === "object")).sort((a, b) => {
+          var _a3, _b2;
+          return ((_a3 = b == null ? void 0 : b.score) != null ? _a3 : 0) - ((_b2 = a == null ? void 0 : a.score) != null ? _b2 : 0);
+        }).slice(0, displayCount);
+        console.log("[Pipeline] Final sources for prompt & UI:", {
+          count: sources.length,
+          sources: sources.map((s, idx) => {
+            var _a3;
+            return {
+              rank: idx + 1,
+              id: s == null ? void 0 : s.id,
+              score: s == null ? void 0 : s.score,
+              contentType: typeof (s == null ? void 0 : s.content),
+              contentSnippet: String((_a3 = s == null ? void 0 : s.content) != null ? _a3 : "").substring(0, 150),
+              metadata: s == null ? void 0 : s.metadata
+            };
+          })
+        });
         if (graphData && graphData.nodes.length > 0) {
           const graphContext = graphData.nodes.map(
             (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
@@ -8594,18 +8694,18 @@ ${context}`;
         }
         yield {
           reply: "",
-          sources: sources.map((s) => {
-            var _a3;
+          sources: (sources || []).filter((s) => Boolean(s && typeof s === "object")).map((s) => {
+            var _a3, _b2, _c2;
             return {
               id: s.id,
-              score: s.score,
-              content: s.content,
-              metadata: (_a3 = s.metadata) != null ? _a3 : {},
+              score: (_a3 = s.score) != null ? _a3 : 0,
+              content: (_b2 = s == null ? void 0 : s.content) != null ? _b2 : "",
+              metadata: (_c2 = s == null ? void 0 : s.metadata) != null ? _c2 : {},
               namespace: ns
             };
           })
         };
-        const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
+        const allMetadataKeys = Array.from(new Set(sources.filter(Boolean).flatMap((s) => Object.keys((s == null ? void 0 : s.metadata) || {}))));
         const cachedSchema = allMetadataKeys.length > 0 ? SchemaMapper.getCached(ns, allMetadataKeys) : void 0;
         if (allMetadataKeys.length > 0 && !cachedSchema) {
           SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
@@ -8897,6 +8997,8 @@ ${context}`;
     });
   }
   resolveNumericPredicateValue(source, predicate) {
+    var _a2;
+    if (!source) return null;
     const meta = source.metadata || {};
     const field = predicate.field;
     const entries = Object.entries(meta).filter(([, value]) => value !== null && value !== void 0 && typeof value !== "object");
@@ -8908,7 +9010,7 @@ ${context}`;
         const value = this.toFiniteNumber(Array.isArray(fuzzy) ? fuzzy[1] : fuzzy.value);
         if (value !== null) return value;
       }
-      const contentValue = this.extractNumericValueFromContent(source.content, field);
+      const contentValue = this.extractNumericValueFromContent((_a2 = source.content) != null ? _a2 : "", field);
       if (contentValue !== null) return contentValue;
     }
     for (const [key, value] of entries) {
@@ -8919,6 +9021,7 @@ ${context}`;
     return null;
   }
   extractNumericValueFromContent(content, field) {
+    if (!content || typeof content !== "string") return null;
     const escapedWords = field.split(/\s+|_+|-+/).filter(Boolean).map((word) => word.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
     if (escapedWords.length === 0) return null;
     const pattern = new RegExp(`(?:${escapedWords.join("[\\\\s_:-]*")})\\s*[:\\-\u2013]?\\s*([\\d,]+(?:\\.\\d+)?)`, "i");
@@ -9004,6 +9107,20 @@ ${context}`;
         resolvedSources.push(source);
       }
     }
+    console.log("[Pipeline] retrieve() response:", {
+      query,
+      namespace: ns,
+      count: resolvedSources.length,
+      sample: resolvedSources.slice(0, 2).map((s) => {
+        var _a3;
+        return {
+          id: s == null ? void 0 : s.id,
+          score: s == null ? void 0 : s.score,
+          contentSnippet: String((_a3 = s == null ? void 0 : s.content) != null ? _a3 : "").substring(0, 150),
+          metadata: s == null ? void 0 : s.metadata
+        };
+      })
+    });
     return { sources: resolvedSources, graphData };
   }
   /** Rewrite the user query for better retrieval performance. */