@retrivora-ai/rag-engine 1.7.1 → 1.7.3

package/dist/{chunk-OHXRQTQH.mjs → chunk-M32ZXLTT.mjs}

@@ -2259,18 +2259,17 @@ var Pipeline = class {
   async initialize() {
     var _a, _b;
     if (this.initialised) return;
-    const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
+    const CHART_MARKER = "<!-- UI_PROTOCOL_V6 -->";
     const chartInstruction = `
 
 ${CHART_MARKER}
-### UNIVERSAL UI PROTOCOL (MANDATORY)
-CRITICAL: When providing visual data (carousels, charts, tables), you MUST wrap the JSON payload in a \`\`\`ui\`\`\` code block. 
-NEVER output naked JSON. If you provide a visualization, do not also provide a markdown table of the same data unless explicitly asked.
+### UNIVERSAL UI PROTOCOL
+When the user asks for a visual representation, comparison, distribution, breakdown, table, list, catalog, products, carousel, stock view, or category summary, you MUST include exactly one \`\`\`ui\`\`\` block.
 
 1. VIEW SELECTION
-   - Use "carousel" for browsing products, items, or catalogs. (e.g., "show me beauty products").
-   - Use "chart" ONLY for numerical aggregations (counts, sums, percentages).
-   - Use "table" for technical specs or row-heavy comparisons.
+   - Use "carousel" for browsing specific products, items, or a catalog. This is the DEFAULT for "show me", "what are", or "find me" requests for a specific category (e.g. "beauty").
+   - Use "chart" ONLY for numerical aggregations: distributions, percentages, counts, comparisons over time, or trends (e.g. "how many beauty products?").
+   - Use "table" for detailed technical specifications, row-heavy comparisons, or when the user explicitly asks for a list/table format.
 
 2. REQUIRED JSON SHAPE
    {
@@ -2278,31 +2277,52 @@ NEVER output naked JSON. If you provide a visualization, do not also provide a m
      "title": "Short heading",
      "description": "One sentence describing the view",
      "chartType": "pie" | "bar" | "line", // Required if view is "chart"
-     "data": [] // Array of objects
+     "xAxisKey": "label",
+     "dataKeys": ["value"],
+     "columns": ["dynamicFieldKey1", "dynamicFieldKey2"],
+     "insights": ["Short takeaway 1", "Short takeaway 2"],
+     "data": []
    }
 
 3. DATA RULES
-   - Build the UI payload from retrieved context and metadata.
-   - For CAROUSEL: "data" items MUST have { id, name, brand, price, image, link }.
-   - IMPORTANT: Extract "image" and "price" from Source metadata. Image URLs are often in fields like "images", "thumbnail", or "img".
+   - Build the UI payload from retrieved context only. Do not invent products or values.
+   - For CAROUSEL: "data" must be a list of product objects (id, name, brand, price, image, link).
+   - For CHART: "data" must be aggregated counts or sums. Never put raw product lists in a chart view.
+   - If the user asks for "products in beauty", do NOT return a pie chart of categories; return a carousel of the beauty products themselves.
 
 4. FORMAT RULES
-   - Wrap the JSON in \`\`\`ui ... \`\`\` blocks.
-   - Place natural language text OUTSIDE the block.
+   - Return valid JSON inside the \`\`\`ui\`\`\` block.
+   - Put natural language explanation before or after the block, not inside.
 
-5. EXAMPLE
-User: "find me beauty products"
-Assistant: "I found several beauty products in our catalog:"
+5. EXAMPLES
+User: "show me beauty products"
+Assistant:
 \`\`\`ui
 {
   "view": "carousel",
   "title": "Beauty Products",
-  "description": "Latest skincare and makeup items.",
+  "description": "Browse our latest skincare and makeup collection.",
   "data": [
-    { "id": "1", "name": "Mascara", "brand": "Essence", "price": "$9.99", "image": "https://placehold.co/600x400?text=Product", "link": "https://example.com/product" }
+    { "id": "1", "name": "Mascara", "brand": "Essence", "price": "$9.99", "image": "...", "link": "..." },
+    { "id": "2", "name": "Lipstick", "brand": "Chic", "price": "$12.99", "image": "...", "link": "..." }
   ]
 }
 \`\`\`
+
+User: "breakdown of products by category"
+Assistant:
+\`\`\`ui
+{
+  "view": "chart",
+  "title": "Category Breakdown",
+  "chartType": "pie",
+  "xAxisKey": "label",
+  "dataKeys": ["value"],
+  "data": [
+    { "label": "Beauty", "value": 15 },
+    { "label": "Fragrances", "value": 8 }
+  ]
+}
 \`\`\``;
     if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
       this.config.llm.systemPrompt = (this.config.llm.systemPrompt || "") + chartInstruction;
@@ -2511,8 +2531,7 @@ Assistant: "I found several beauty products in our catalog:"
           sources = sources.slice(0, topK);
         }
         let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
-Content: ${m.content}
-Metadata: ${JSON.stringify(m.metadata)}`).join("\n\n---\n\n") : "No relevant context found.";
+${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (graphData && graphData.nodes.length > 0) {
           const graphContext = graphData.nodes.map(
             (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
@@ -2864,6 +2883,351 @@ var DocumentParser = class {
   }
 };
 
+// src/utils/UITransformer.ts
+var UITransformer = class {
+  /**
+   * Main transformation method
+   * Analyzes user query and retrieved data to determine the best visualization format
+   *
+   * @param userQuery - The original user question/query
+   * @param retrievedData - Vector database retrieval results
+   * @returns Structured JSON response for UI rendering
+   */
+  static transform(userQuery, retrievedData) {
+    if (!retrievedData || retrievedData.length === 0) {
+      return this.createTextResponse("No data available", "No relevant data found for your query.");
+    }
+    const analysis = this.analyzeData(userQuery, retrievedData);
+    switch (analysis.suggestedType) {
+      case "product_carousel":
+        return this.transformToProductCarousel(retrievedData);
+      case "pie_chart":
+        return this.transformToPieChart(retrievedData);
+      case "bar_chart":
+        return this.transformToBarChart(retrievedData);
+      case "line_chart":
+        return this.transformToLineChart(retrievedData);
+      case "table":
+        return this.transformToTable(retrievedData);
+      default:
+        return this.transformToText(retrievedData);
+    }
+  }
+  /**
+   * Analyzes the data and query to suggest the best visualization type
+   */
+  static analyzeData(query, data) {
+    const queryLower = query.toLowerCase();
+    const hasProducts = data.some(
+      (item) => this.isProductData(item)
+    );
+    const hasTimeSeries = data.some(
+      (item) => this.isTimeSeriesData(item)
+    );
+    const hasCategories = this.detectCategories(data).length > 0;
+    const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
+    const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
+    const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
+    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
+      return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isTrendQuery && hasTimeSeries) {
+      return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isDistributionQuery && hasCategories) {
+      return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isComparisonQuery && hasCategories && data.length > 2) {
+      return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (data.length > 5 && this.hasMultipleFields(data)) {
+      return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+    }
+    return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
+  }
+  /**
+   * Transform data to product carousel format
+   */
+  static transformToProductCarousel(data) {
+    const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
+    return {
+      type: "product_carousel",
+      title: "Recommended Products",
+      description: `Found ${products.length} relevant products`,
+      data: products
+    };
+  }
+  /**
+   * Transform data to pie chart format
+   */
+  static transformToPieChart(data) {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+    const pieData = Object.entries(categoryData).map(([label, count]) => ({
+      label,
+      value: count,
+      inStock: this.checkStockStatus(label, data)
+    }));
+    return {
+      type: "pie_chart",
+      title: "Distribution by Category",
+      description: `Showing breakdown across ${categories.length} categories`,
+      data: pieData
+    };
+  }
+  /**
+   * Transform data to bar chart format
+   */
+  static transformToBarChart(data) {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+    const barData = Object.entries(categoryData).map(([category, value]) => ({
+      category,
+      value,
+      inStock: this.checkStockStatus(category, data)
+    }));
+    return {
+      type: "bar_chart",
+      title: "Comparison by Category",
+      description: `Comparing ${categories.length} categories`,
+      data: barData
+    };
+  }
+  /**
+   * Transform data to line chart format
+   */
+  static transformToLineChart(data) {
+    const timePoints = this.extractTimeSeriesData(data);
+    const lineData = timePoints.map((point) => ({
+      timestamp: point.timestamp,
+      value: point.value,
+      label: point.label
+    }));
+    return {
+      type: "line_chart",
+      title: "Trend Over Time",
+      description: `Showing ${lineData.length} data points`,
+      data: lineData
+    };
+  }
+  /**
+   * Transform data to table format
+   */
+  static transformToTable(data) {
+    const columns = this.extractTableColumns(data);
+    const rows = data.map((item) => this.extractTableRow(item, columns));
+    const tableData = {
+      columns,
+      rows
+    };
+    return {
+      type: "table",
+      title: "Detailed Results",
+      description: `Showing ${data.length} results`,
+      data: tableData
+    };
+  }
+  /**
+   * Transform data to text format (fallback)
+   */
+  static transformToText(data) {
+    const textContent = data.map((item) => item.content).join("\n\n");
+    return this.createTextResponse(
+      "Search Results",
+      textContent,
+      `Found ${data.length} relevant results`
+    );
+  }
+  /**
+   * Helper: Create text response
+   */
+  static createTextResponse(title, content, description) {
+    return {
+      type: "text",
+      title,
+      description,
+      data: { content }
+    };
+  }
+  /**
+   * Helper: Check if data item is product-related
+   */
+  static isProductData(item) {
+    const content = (item.content || "").toLowerCase();
+    const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
+    return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+      (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
+    );
+  }
+  /**
+   * Helper: Check if data contains time series
+   */
+  static isTimeSeriesData(item) {
+    const content = (item.content || "").toLowerCase();
+    const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
+    return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+      (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
+    );
+  }
+  /**
+   * Helper: Extract product information from vector match
+   */
+  static extractProductInfo(item) {
+    const meta = item.metadata || {};
+    if (meta.name || meta.product) {
+      return {
+        id: item.id,
+        name: meta.name || meta.product,
+        price: meta.price,
+        image: meta.image || meta.imageUrl,
+        brand: meta.brand,
+        description: item.content,
+        inStock: this.determineStockStatus(item)
+      };
+    }
+    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
+    if (nameMatch) {
+      return {
+        id: item.id,
+        name: nameMatch[1],
+        price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
+        description: item.content,
+        inStock: this.determineStockStatus(item)
+      };
+    }
+    return null;
+  }
+  /**
+   * Helper: Detect categories in data
+   */
+  static detectCategories(data) {
+    const categories = /* @__PURE__ */ new Set();
+    data.forEach((item) => {
+      const meta = item.metadata || {};
+      if (meta.category) {
+        categories.add(String(meta.category));
+      }
+      if (meta.type) {
+        categories.add(String(meta.type));
+      }
+      if (meta.tag) {
+        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
+        tags.forEach((t) => categories.add(String(t)));
+      }
+      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
+      if (categoryMatch) {
+        categories.add(categoryMatch[1].trim());
+      }
+    });
+    return Array.from(categories);
+  }
+  /**
+   * Helper: Aggregate data by category
+   */
+  static aggregateByCategory(data, categories) {
+    const result = {};
+    categories.forEach((cat) => {
+      result[cat] = 0;
+    });
+    data.forEach((item) => {
+      const meta = item.metadata || {};
+      const itemCategory = meta.category || meta.type || "Other";
+      if (result.hasOwnProperty(itemCategory)) {
+        result[itemCategory]++;
+      } else {
+        result["Other"] = (result["Other"] || 0) + 1;
+      }
+    });
+    return result;
+  }
+  /**
+   * Helper: Extract time series data
+   */
+  static extractTimeSeriesData(data) {
+    return data.map((item) => {
+      const meta = item.metadata || {};
+      return {
+        timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
+        value: meta.value || item.score || 0,
+        label: meta.label || item.content.substring(0, 50)
+      };
+    });
+  }
+  /**
+   * Helper: Extract table columns
+   */
+  static extractTableColumns(data) {
+    const columnSet = /* @__PURE__ */ new Set();
+    columnSet.add("Content");
+    data.forEach((item) => {
+      Object.keys(item.metadata || {}).forEach((key) => {
+        columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
+      });
+    });
+    return Array.from(columnSet);
+  }
+  /**
+   * Helper: Extract table row
+   */
+  static extractTableRow(item, columns) {
+    const meta = item.metadata || {};
+    return columns.map((col) => {
+      if (col === "Content") {
+        return item.content.substring(0, 100);
+      }
+      const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
+      const value = meta[metaKey];
+      return value !== void 0 ? String(value) : "";
+    });
+  }
+  /**
+   * Helper: Check stock status
+   */
+  static checkStockStatus(category, data) {
+    const item = data.find((d) => {
+      const meta = d.metadata || {};
+      return meta.category === category || meta.type === category;
+    });
+    return this.determineStockStatus(item || {});
+  }
+  /**
+   * Helper: Determine if item is in stock
+   */
+  static determineStockStatus(item) {
+    const meta = item.metadata || {};
+    if (meta.inStock !== void 0) {
+      return Boolean(meta.inStock);
+    }
+    if (meta.stock !== void 0) {
+      return Boolean(meta.stock);
+    }
+    if (meta.available !== void 0) {
+      return Boolean(meta.available);
+    }
+    const content = (item.content || "").toLowerCase();
+    if (content.includes("out of stock") || content.includes("unavailable")) {
+      return false;
+    }
+    if (content.includes("in stock") || content.includes("available")) {
+      return true;
+    }
+    return true;
+  }
+  /**
+   * Helper: Check if data has multiple fields
+   */
+  static hasMultipleFields(data) {
+    const fieldCount = /* @__PURE__ */ new Set();
+    data.forEach((item) => {
+      Object.keys(item.metadata || {}).forEach((key) => {
+        fieldCount.add(key);
+      });
+    });
+    return fieldCount.size > 2;
+  }
+};
+
 // src/handlers/index.ts
 function sseFrame(payload) {
   return `data: ${JSON.stringify(payload)}
@@ -2878,6 +3242,11 @@ function sseTextFrame(text) {
 function sseMetaFrame(meta) {
   return `data: ${JSON.stringify(__spreadValues({ type: "metadata" }, meta != null ? meta : {}))}
 
+`;
+}
+function sseUIFrame(uiTransformation) {
+  return `data: ${JSON.stringify(__spreadValues({ type: "ui_transformation" }, uiTransformation != null ? uiTransformation : {}))}
+
 `;
 }
 function sseErrorFrame(message) {
@@ -2941,6 +3310,15 @@ function createStreamHandler(configOrPlugin) {
                 enqueue(sseTextFrame(chunk));
               } else {
                 enqueue(sseMetaFrame(chunk));
+                const sources = (chunk == null ? void 0 : chunk.sources) || [];
+                if (sources && sources.length > 0) {
+                  try {
+                    const uiTransformation = UITransformer.transform(message, sources);
+                    enqueue(sseUIFrame(uiTransformation));
+                  } catch (transformError) {
+                    console.warn("[createStreamHandler] UI transformation warning:", transformError);
+                  }
+                }
               }
             }
           } catch (temp) {
@@ -3073,6 +3451,7 @@ export {
   sseFrame,
   sseTextFrame,
   sseMetaFrame,
+  sseUIFrame,
   sseErrorFrame,
   createChatHandler,
   createStreamHandler,

package/dist/handlers/index.d.mts

@@ -1,3 +1,3 @@
 import '../index-Cti1u0y1.mjs';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from '../index-kUXnRvuI.mjs';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame, l as sseUIFrame } from '../index-BejNscWZ.mjs';
 import 'next/server';

package/dist/handlers/index.d.ts

@@ -1,3 +1,3 @@
 import '../index-Cti1u0y1.js';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from '../index-B67KQ9NN.js';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, k as createSuggestionsHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame, l as sseUIFrame } from '../index-CbkMJvu5.js';
 import 'next/server';