@retrivora-ai/rag-engine 1.9.1 → 1.9.3

package/dist/server.js

@@ -540,6 +540,82 @@ var init_PostgreSQLProvider = __esm({
   }
 });
 
+// src/utils/synonyms.ts
+function resolveMetadataValue(meta, uiKey) {
+  var _a;
+  const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
+  const keys = Object.keys(meta);
+  const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
+  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
+  if (match !== void 0) return meta[match];
+  const isBlacklisted = (kl) => {
+    return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
+  };
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return kl.includes(uiKey.toLowerCase());
+  });
+  if (match !== void 0) return meta[match];
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
+  });
+  return match !== void 0 ? meta[match] : void 0;
+}
+var FIELD_SYNONYMS;
+var init_synonyms = __esm({
+  "src/utils/synonyms.ts"() {
+    "use strict";
+    FIELD_SYNONYMS = {
+      name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
+      price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd", "variant price"],
+      brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
+      image: [
+        "imageUrl",
+        "thumbnail",
+        "img",
+        "url",
+        "photo",
+        "picture",
+        "media",
+        "image_url",
+        "main_image",
+        "product_image",
+        "thumb",
+        "image src",
+        "variant image"
+      ],
+      stock: [
+        "inventory",
+        "quantity",
+        "count",
+        "availability",
+        "stock_level",
+        "inStock",
+        "is_available",
+        "in stock",
+        "status",
+        "variant inventory qty"
+      ],
+      category: [
+        "product_category",
+        "product category",
+        "category_name",
+        "category name",
+        "department",
+        "collection",
+        "type"
+      ],
+      description: ["summary", "content", "body", "text", "info", "details", "body (html)", "seo description"],
+      link: ["url", "href", "product_url", "page_url", "link"]
+    };
+  }
+});
+
 // src/providers/vectordb/MultiTablePostgresProvider.ts
 var MultiTablePostgresProvider_exports = {};
 __export(MultiTablePostgresProvider_exports, {
@@ -551,6 +627,7 @@ var init_MultiTablePostgresProvider = __esm({
     "use strict";
     import_pg2 = require("pg");
     init_BaseVectorProvider();
+    init_synonyms();
     MultiTablePostgresProvider = class extends BaseVectorProvider {
       constructor(config) {
         var _a, _b, _c;
@@ -631,8 +708,13 @@ var init_MultiTablePostgresProvider = __esm({
             }
             tableName = tableName.replace(/[^a-z0-9_]/gi, "_").toLowerCase() || this.uploadTable;
             const firstMeta = fileDocs[0].metadata || {};
-            const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding"];
-            const csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+            let csvHeaders = [];
+            if (Array.isArray(firstMeta.csvHeaders)) {
+              csvHeaders = firstMeta.csvHeaders;
+            } else {
+              const systemKeys = ["fileName", "fileSize", "fileType", "uploadedAt", "dimension", "chunkIndex", "id", "namespace", "content", "metadata", "embedding", "docId", "docid", "chunkId", "chunkid", "csvHeaders", "csvheaders"];
+              csvHeaders = Object.keys(firstMeta).filter((k) => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+            }
             const columnDefs = csvHeaders.map((h) => `"${h}" TEXT`).join(",\n          ");
             const createTableSql = `
           CREATE TABLE IF NOT EXISTS "${tableName}" (
@@ -725,10 +807,26 @@ var init_MultiTablePostgresProvider = __esm({
         };
         const dynamicKeywordQuery = getDynamicKeywordQuery();
         const tableLimit = Math.max(topK, 50);
+        const metadataFilters = _filter == null ? void 0 : _filter.metadata;
         const queryPromises = this.tables.map(async (table) => {
           try {
             let sqlQuery = "";
             let params = [];
+            let whereClause = "";
+            const filterParams = [];
+            if (metadataFilters && Object.keys(metadataFilters).length > 0) {
+              const conditions = Object.entries(metadataFilters).map(([key, val]) => {
+                var _a3;
+                filterParams.push(String(val));
+                const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
+                const paramIdx = baseOffset + filterParams.length;
+                const synonyms = (_a3 = FIELD_SYNONYMS[key]) != null ? _a3 : [];
+                const keysToCheck = [key, ...synonyms];
+                const coalesceExprs = keysToCheck.map((k) => `LOWER(metadata->>'${k}')`);
+                return `COALESCE(${coalesceExprs.join(", ")}) = LOWER($${paramIdx})`;
+              });
+              whereClause = `WHERE ${conditions.join(" AND ")}`;
+            }
             if (queryText && dynamicKeywordQuery) {
               const hasEntityHints = entityHints.length > 0;
               const exactNameScoreExpr = hasEntityHints ? `+ (
@@ -745,19 +843,21 @@ var init_MultiTablePostgresProvider = __esm({
              COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
              ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
+            ${whereClause}
             ORDER BY hybrid_score DESC
             LIMIT ${tableLimit}
           `;
-              params = [vectorLiteral, dynamicKeywordQuery];
+              params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
             } else {
               sqlQuery = `
             SELECT *, 
             (1 - (embedding <=> $1::vector)) AS hybrid_score
             FROM "${table}" t
+            ${whereClause}
             ORDER BY hybrid_score DESC
             LIMIT ${tableLimit}
           `;
-              params = [vectorLiteral];
+              params = [vectorLiteral, ...filterParams];
             }
             const result = await this.pool.query(sqlQuery, params);
             if (result.rowCount && result.rowCount > 0) {
@@ -990,12 +1090,15 @@ var init_MongoDBProvider = __esm({
           }
         );
         const results = await this.collection.aggregate(pipeline).toArray();
-        return results.map((res) => ({
-          id: res._id,
-          content: res[this.contentKey],
-          metadata: res[this.metadataKey] || {},
-          score: res.score
-        }));
+        return results.map((res) => {
+          const normalizedScore = res.score * 2 - 1;
+          return {
+            id: res._id,
+            content: res[this.contentKey],
+            metadata: res[this.metadataKey] || {},
+            score: normalizedScore
+          };
+        });
       }
       async delete(id, namespace) {
         await this.collection.deleteOne(__spreadValues({ _id: id }, namespace ? { namespace } : {}));
@@ -4430,7 +4533,8 @@ var QueryProcessor = class {
     const fieldValuePatterns = [
       new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
       new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
-      new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
+      new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi"),
+      new RegExp(`\\b(?:under|in|for|from|of)\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, "gi")
     ];
     for (const pattern of fieldValuePatterns) {
       for (const match of question.matchAll(pattern)) {
@@ -4664,73 +4768,8 @@ var LLMRouter = class {
   }
 };
 
-// src/utils/synonyms.ts
-var FIELD_SYNONYMS = {
-  name: ["product", "item", "title", "label", "heading", "subject", "product name", "item name"],
-  price: ["cost", "amount", "msrp", "price", "rate", "value", "price_usd"],
-  brand: ["manufacturer", "vendor", "make", "company", "brand_name", "supplier"],
-  image: [
-    "imageUrl",
-    "thumbnail",
-    "img",
-    "url",
-    "photo",
-    "picture",
-    "media",
-    "image_url",
-    "main_image",
-    "product_image",
-    "thumb"
-  ],
-  stock: [
-    "inventory",
-    "quantity",
-    "count",
-    "availability",
-    "stock_level",
-    "inStock",
-    "is_available",
-    "in stock",
-    "status"
-  ],
-  category: [
-    "product_category",
-    "product category",
-    "category_name",
-    "category name",
-    "department",
-    "collection"
-  ],
-  description: ["summary", "content", "body", "text", "info", "details"],
-  link: ["url", "href", "product_url", "page_url", "link"]
-};
-function resolveMetadataValue(meta, uiKey) {
-  var _a;
-  const synonyms = (_a = FIELD_SYNONYMS[uiKey]) != null ? _a : [];
-  const keys = Object.keys(meta);
-  const systemKeys = ["namespace", "filename", "filesize", "filetype", "docid", "uploadedat", "dimension", "chunkindex"];
-  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
-  if (match !== void 0) return meta[match];
-  match = keys.find((k) => synonyms.map((s) => s.toLowerCase()).includes(k.toLowerCase()));
-  if (match !== void 0) return meta[match];
-  const isBlacklisted = (kl) => {
-    return systemKeys.includes(kl) || kl.startsWith("option1") || kl.startsWith("option2") || kl.startsWith("option3");
-  };
-  match = keys.find((k) => {
-    const kl = k.toLowerCase();
-    if (isBlacklisted(kl)) return false;
-    return kl.includes(uiKey.toLowerCase());
-  });
-  if (match !== void 0) return meta[match];
-  match = keys.find((k) => {
-    const kl = k.toLowerCase();
-    if (isBlacklisted(kl)) return false;
-    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
-  });
-  return match !== void 0 ? meta[match] : void 0;
-}
-
 // src/utils/UITransformer.ts
+init_synonyms();
 var UITransformer = class {
   // ─── Public Entry Points ─────────────────────────────────────────────────
   /**
@@ -5027,7 +5066,7 @@ RULES:
   }
   // ─── Transform Helpers ────────────────────────────────────────────────────
   static transformToProductCarousel(data, config, trainedSchema) {
-    const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null);
+    const products = data.map((item) => this.extractProductInfo(item, config, trainedSchema)).filter((p) => p !== null).slice(0, 15);
     return {
       type: "product_carousel",
       title: "Recommended Products",
@@ -5986,14 +6025,16 @@ Given these metadata keys from a database: [${keys.join(", ")}]
 Identify which keys best correspond to these standard UI properties:
 ${propertyList}
 
-Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys. 
+Return ONLY a valid JSON object where the keys are the UI properties and the values are the EXACT matching database keys from the list above. 
 If no good match is found for a property, omit it.
 
 Example:
 {
-  "name": "Product_Title",
-  "price": "MSRP_USD",
-  "brand": "VendorName"
+  "name": "Title",
+  "price": "Variant Price",
+  "brand": "Vendor",
+  "image": "Image Src",
+  "stock": "Variant Inventory Qty"
 }
 `;
     try {
@@ -6284,7 +6325,10 @@ var Pipeline = class {
       upsertBatchOptions
     );
     if (upsertResult.errors.length > 0) {
-      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
+      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed. Error details:`);
+      upsertResult.errors.forEach((err, idx) => {
+        console.warn(`  Batch ${idx + 1} Error: ${err.error.message || String(err.error)}`);
+      });
     }
     return upsertResult.totalProcessed;
   }
@@ -6399,8 +6443,9 @@ var Pipeline = class {
           this.embeddingCache.set(cacheKey, queryVector);
         }
         const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
+        const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
         const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
-        const wantsExhaustiveList = hasNumericPredicates || /\b(list|all|show|provide|give|get)\b/i.test(question);
+        const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
         const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
         const rawSources = (strategyResult === "vector" || strategyResult === "both") && queryVector && queryVector.length > 0 ? yield new __await(this.vectorDB.query(queryVector, retrievalLimit, ns, filter)) : [];
         const retrieveEnd = performance.now();
@@ -6408,15 +6453,28 @@ var Pipeline = class {
         const retrieveMs = retrieveEnd - embedStart;
         const rerankStart = performance.now();
         const structuredSources = this.applyStructuredFilters(rawSources, filter);
-        let sources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
+        let fullSources = hasNumericPredicates ? structuredSources : structuredSources.filter((m) => m.score >= scoreThreshold);
+        const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
         if (!hasNumericPredicates && ((_k = this.config.rag) == null ? void 0 : _k.useReranking)) {
-          sources = yield new __await(this.reranker.rerank(sources, question, topK));
+          fullSources = yield new __await(this.reranker.rerank(fullSources, question, rerankLimit));
         } else if (!wantsExhaustiveList) {
-          sources = sources.slice(0, topK);
+          fullSources = fullSources.slice(0, topK);
         }
         const rerankMs = performance.now() - rerankStart;
-        let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
+        let context = fullSources.length ? fullSources.map((m, i) => `[Source ${i + 1}]
 ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
+        let displayCount = 15;
+        if (hasMetadataFilter) {
+          displayCount = fullSources.length;
+        } else {
+          const baseThreshold = Math.max(scoreThreshold, 0.4);
+          const highlyRelevant = fullSources.filter((m) => m.score >= baseThreshold);
+          displayCount = Math.max(highlyRelevant.length, topK);
+          if (displayCount > 15) {
+            displayCount = 15;
+          }
+        }
+        const sources = [...fullSources].sort((a, b) => b.score - a.score).slice(0, displayCount);
         if (graphData && graphData.nodes.length > 0) {
           const graphContext = graphData.nodes.map(
             (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
@@ -6429,7 +6487,18 @@ ${context}`;
         }
         const allMetadataKeys = Array.from(new Set(sources.flatMap((s) => Object.keys(s.metadata || {}))));
         const trainedSchemaPromise = allMetadataKeys.length > 0 ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => void 0) : Promise.resolve(void 0);
-        const restrictionSuffix = "\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. The UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization. If listing products, use simple bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
+        const uiTransformationPromise = trainedSchemaPromise.then(
+          (trainedSchema) => this.generateUiTransformation(
+            question,
+            sources,
+            trainedSchema,
+            hasNumericPredicates
+          )
+        ).catch((uiError) => {
+          console.warn("[Pipeline] UI transformation failed concurrently:", uiError);
+          return UITransformer.transform(question, sources, this.config);
+        });
+        const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
         const hardenedHistory = [...history];
         const userQuestion = { role: "user", content: question + restrictionSuffix };
         const messages = [...hardenedHistory, userQuestion];
@@ -6499,19 +6568,7 @@ ${context}`;
           tokens,
           timestamp: (/* @__PURE__ */ new Date()).toISOString()
         };
-        let uiTransformation;
-        try {
-          const trainedSchema = yield new __await(trainedSchemaPromise);
-          uiTransformation = yield new __await(this.generateUiTransformation(
-            question,
-            sources,
-            trainedSchema,
-            hasNumericPredicates
-          ));
-        } catch (uiError) {
-          console.warn("[Pipeline] UI transformation failed before metadata yield:", uiError);
-          uiTransformation = UITransformer.transform(question, sources, this.config);
-        }
+        const uiTransformation = yield new __await(uiTransformationPromise);
         yield {
           reply: "",
           sources,
@@ -7245,15 +7302,24 @@ function createStreamHandler(configOrPlugin) {
       });
     }
     const encoder = new TextEncoder();
+    let isActive = true;
     const stream = new ReadableStream({
       async start(controller) {
         var _a;
-        const enqueue = (text) => controller.enqueue(encoder.encode(text));
+        const enqueue = (text) => {
+          if (!isActive) return;
+          try {
+            controller.enqueue(encoder.encode(text));
+          } catch (err) {
+            console.warn("[createStreamHandler] Failed to enqueue (stream already closed):", err);
+          }
+        };
         try {
           const pipelineStream = plugin.chatStream(message, history, namespace);
           try {
             for (var iter = __forAwait(pipelineStream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
               const chunk = temp.value;
+              if (!isActive) break;
               if (typeof chunk === "string") {
                 enqueue(sseTextFrame(chunk));
               } else {
@@ -7291,12 +7357,27 @@ function createStreamHandler(configOrPlugin) {
             }
           }
         } catch (streamError) {
-          const errorMessage = streamError instanceof Error ? streamError.message : String(streamError);
-          console.error("[createStreamHandler] Stream error:", streamError);
-          enqueue(sseErrorFrame(errorMessage));
+          if (isActive) {
+            const errorMessage = streamError instanceof Error ? streamError.message : String(streamError);
+            console.error("[createStreamHandler] Stream error:", streamError);
+            try {
+              enqueue(sseErrorFrame(errorMessage));
+            } catch (e) {
+            }
+          }
         } finally {
-          controller.close();
+          if (isActive) {
+            isActive = false;
+            try {
+              controller.close();
+            } catch (e) {
+            }
+          }
         }
+      },
+      cancel(reason) {
+        isActive = false;
+        console.log("[createStreamHandler] Stream connection closed by client:", reason);
       }
     });
     return new Response(stream, { headers: SSE_HEADERS });
@@ -7357,6 +7438,7 @@ function createUploadHandler(configOrPlugin) {
           if (parsed.data && parsed.data.length > 0) {
             let i = 0;
             let lastRowData = null;
+            const csvCols = Object.keys(parsed.data[0]);
             for (const row of parsed.data) {
               i++;
               const rowData = row;
@@ -7378,12 +7460,14 @@ function createUploadHandler(configOrPlugin) {
               documents.push({
                 docId: `${file.name}-row-${i}`,
                 content: contentParts.join(", "),
-                metadata: __spreadValues(__spreadValues({
+                metadata: __spreadValues(__spreadProps(__spreadValues({
                   fileName: file.name,
                   fileSize: file.size,
                   fileType: file.type,
                   uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
-                }, dimension ? { dimension } : {}), rowData)
+                }, dimension ? { dimension } : {}), {
+                  csvHeaders: csvCols
+                }), rowData)
               });
             }
           }