@retrivora-ai/rag-engine 1.7.1 → 1.7.3

package/dist/handlers/index.js

@@ -1617,7 +1617,8 @@ __export(handlers_exports, {
   sseErrorFrame: () => sseErrorFrame,
   sseFrame: () => sseFrame,
   sseMetaFrame: () => sseMetaFrame,
-  sseTextFrame: () => sseTextFrame
+  sseTextFrame: () => sseTextFrame,
+  sseUIFrame: () => sseUIFrame
 });
 module.exports = __toCommonJS(handlers_exports);
 var import_server = require("next/server");
@@ -3822,18 +3823,17 @@ var Pipeline = class {
   async initialize() {
     var _a, _b;
     if (this.initialised) return;
-    const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
+    const CHART_MARKER = "<!-- UI_PROTOCOL_V6 -->";
     const chartInstruction = `
 
 ${CHART_MARKER}
-### UNIVERSAL UI PROTOCOL (MANDATORY)
-CRITICAL: When providing visual data (carousels, charts, tables), you MUST wrap the JSON payload in a \`\`\`ui\`\`\` code block. 
-NEVER output naked JSON. If you provide a visualization, do not also provide a markdown table of the same data unless explicitly asked.
+### UNIVERSAL UI PROTOCOL
+When the user asks for a visual representation, comparison, distribution, breakdown, table, list, catalog, products, carousel, stock view, or category summary, you MUST include exactly one \`\`\`ui\`\`\` block.
 
 1. VIEW SELECTION
-   - Use "carousel" for browsing products, items, or catalogs. (e.g., "show me beauty products").
-   - Use "chart" ONLY for numerical aggregations (counts, sums, percentages).
-   - Use "table" for technical specs or row-heavy comparisons.
+   - Use "carousel" for browsing specific products, items, or a catalog. This is the DEFAULT for "show me", "what are", or "find me" requests for a specific category (e.g. "beauty").
+   - Use "chart" ONLY for numerical aggregations: distributions, percentages, counts, comparisons over time, or trends (e.g. "how many beauty products?").
+   - Use "table" for detailed technical specifications, row-heavy comparisons, or when the user explicitly asks for a list/table format.
 
 2. REQUIRED JSON SHAPE
    {
@@ -3841,31 +3841,52 @@ NEVER output naked JSON. If you provide a visualization, do not also provide a m
      "title": "Short heading",
      "description": "One sentence describing the view",
      "chartType": "pie" | "bar" | "line", // Required if view is "chart"
-     "data": [] // Array of objects
+     "xAxisKey": "label",
+     "dataKeys": ["value"],
+     "columns": ["dynamicFieldKey1", "dynamicFieldKey2"],
+     "insights": ["Short takeaway 1", "Short takeaway 2"],
+     "data": []
    }
 
 3. DATA RULES
-   - Build the UI payload from retrieved context and metadata.
-   - For CAROUSEL: "data" items MUST have { id, name, brand, price, image, link }.
-   - IMPORTANT: Extract "image" and "price" from Source metadata. Image URLs are often in fields like "images", "thumbnail", or "img".
+   - Build the UI payload from retrieved context only. Do not invent products or values.
+   - For CAROUSEL: "data" must be a list of product objects (id, name, brand, price, image, link).
+   - For CHART: "data" must be aggregated counts or sums. Never put raw product lists in a chart view.
+   - If the user asks for "products in beauty", do NOT return a pie chart of categories; return a carousel of the beauty products themselves.
 
 4. FORMAT RULES
-   - Wrap the JSON in \`\`\`ui ... \`\`\` blocks.
-   - Place natural language text OUTSIDE the block.
+   - Return valid JSON inside the \`\`\`ui\`\`\` block.
+   - Put natural language explanation before or after the block, not inside.
 
-5. EXAMPLE
-User: "find me beauty products"
-Assistant: "I found several beauty products in our catalog:"
+5. EXAMPLES
+User: "show me beauty products"
+Assistant:
 \`\`\`ui
 {
   "view": "carousel",
   "title": "Beauty Products",
-  "description": "Latest skincare and makeup items.",
+  "description": "Browse our latest skincare and makeup collection.",
   "data": [
-    { "id": "1", "name": "Mascara", "brand": "Essence", "price": "$9.99", "image": "https://placehold.co/600x400?text=Product", "link": "https://example.com/product" }
+    { "id": "1", "name": "Mascara", "brand": "Essence", "price": "$9.99", "image": "...", "link": "..." },
+    { "id": "2", "name": "Lipstick", "brand": "Chic", "price": "$12.99", "image": "...", "link": "..." }
   ]
 }
 \`\`\`
+
+User: "breakdown of products by category"
+Assistant:
+\`\`\`ui
+{
+  "view": "chart",
+  "title": "Category Breakdown",
+  "chartType": "pie",
+  "xAxisKey": "label",
+  "dataKeys": ["value"],
+  "data": [
+    { "label": "Beauty", "value": 15 },
+    { "label": "Fragrances", "value": 8 }
+  ]
+}
 \`\`\``;
     if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
       this.config.llm.systemPrompt = (this.config.llm.systemPrompt || "") + chartInstruction;
@@ -4074,8 +4095,7 @@ Assistant: "I found several beauty products in our catalog:"
           sources = sources.slice(0, topK);
         }
         let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
-Content: ${m.content}
-Metadata: ${JSON.stringify(m.metadata)}`).join("\n\n---\n\n") : "No relevant context found.";
+${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
         if (graphData && graphData.nodes.length > 0) {
           const graphContext = graphData.nodes.map(
             (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
@@ -4427,6 +4447,351 @@ var DocumentParser = class {
   }
 };
 
+// src/utils/UITransformer.ts
+var UITransformer = class {
+  /**
+   * Main transformation method
+   * Analyzes user query and retrieved data to determine the best visualization format
+   *
+   * @param userQuery - The original user question/query
+   * @param retrievedData - Vector database retrieval results
+   * @returns Structured JSON response for UI rendering
+   */
+  static transform(userQuery, retrievedData) {
+    if (!retrievedData || retrievedData.length === 0) {
+      return this.createTextResponse("No data available", "No relevant data found for your query.");
+    }
+    const analysis = this.analyzeData(userQuery, retrievedData);
+    switch (analysis.suggestedType) {
+      case "product_carousel":
+        return this.transformToProductCarousel(retrievedData);
+      case "pie_chart":
+        return this.transformToPieChart(retrievedData);
+      case "bar_chart":
+        return this.transformToBarChart(retrievedData);
+      case "line_chart":
+        return this.transformToLineChart(retrievedData);
+      case "table":
+        return this.transformToTable(retrievedData);
+      default:
+        return this.transformToText(retrievedData);
+    }
+  }
+  /**
+   * Analyzes the data and query to suggest the best visualization type
+   */
+  static analyzeData(query, data) {
+    const queryLower = query.toLowerCase();
+    const hasProducts = data.some(
+      (item) => this.isProductData(item)
+    );
+    const hasTimeSeries = data.some(
+      (item) => this.isTimeSeriesData(item)
+    );
+    const hasCategories = this.detectCategories(data).length > 0;
+    const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
+    const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
+    const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
+    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
+      return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isTrendQuery && hasTimeSeries) {
+      return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isDistributionQuery && hasCategories) {
+      return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (isComparisonQuery && hasCategories && data.length > 2) {
+      return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
+    }
+    if (data.length > 5 && this.hasMultipleFields(data)) {
+      return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+    }
+    return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
+  }
+  /**
+   * Transform data to product carousel format
+   */
+  static transformToProductCarousel(data) {
+    const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
+    return {
+      type: "product_carousel",
+      title: "Recommended Products",
+      description: `Found ${products.length} relevant products`,
+      data: products
+    };
+  }
+  /**
+   * Transform data to pie chart format
+   */
+  static transformToPieChart(data) {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+    const pieData = Object.entries(categoryData).map(([label, count]) => ({
+      label,
+      value: count,
+      inStock: this.checkStockStatus(label, data)
+    }));
+    return {
+      type: "pie_chart",
+      title: "Distribution by Category",
+      description: `Showing breakdown across ${categories.length} categories`,
+      data: pieData
+    };
+  }
+  /**
+   * Transform data to bar chart format
+   */
+  static transformToBarChart(data) {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+    const barData = Object.entries(categoryData).map(([category, value]) => ({
+      category,
+      value,
+      inStock: this.checkStockStatus(category, data)
+    }));
+    return {
+      type: "bar_chart",
+      title: "Comparison by Category",
+      description: `Comparing ${categories.length} categories`,
+      data: barData
+    };
+  }
+  /**
+   * Transform data to line chart format
+   */
+  static transformToLineChart(data) {
+    const timePoints = this.extractTimeSeriesData(data);
+    const lineData = timePoints.map((point) => ({
+      timestamp: point.timestamp,
+      value: point.value,
+      label: point.label
+    }));
+    return {
+      type: "line_chart",
+      title: "Trend Over Time",
+      description: `Showing ${lineData.length} data points`,
+      data: lineData
+    };
+  }
+  /**
+   * Transform data to table format
+   */
+  static transformToTable(data) {
+    const columns = this.extractTableColumns(data);
+    const rows = data.map((item) => this.extractTableRow(item, columns));
+    const tableData = {
+      columns,
+      rows
+    };
+    return {
+      type: "table",
+      title: "Detailed Results",
+      description: `Showing ${data.length} results`,
+      data: tableData
+    };
+  }
+  /**
+   * Transform data to text format (fallback)
+   */
+  static transformToText(data) {
+    const textContent = data.map((item) => item.content).join("\n\n");
+    return this.createTextResponse(
+      "Search Results",
+      textContent,
+      `Found ${data.length} relevant results`
+    );
+  }
+  /**
+   * Helper: Create text response
+   */
+  static createTextResponse(title, content, description) {
+    return {
+      type: "text",
+      title,
+      description,
+      data: { content }
+    };
+  }
+  /**
+   * Helper: Check if data item is product-related
+   */
+  static isProductData(item) {
+    const content = (item.content || "").toLowerCase();
+    const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
+    return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+      (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
+    );
+  }
+  /**
+   * Helper: Check if data contains time series
+   */
+  static isTimeSeriesData(item) {
+    const content = (item.content || "").toLowerCase();
+    const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
+    return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
+      (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
+    );
+  }
+  /**
+   * Helper: Extract product information from vector match
+   */
+  static extractProductInfo(item) {
+    const meta = item.metadata || {};
+    if (meta.name || meta.product) {
+      return {
+        id: item.id,
+        name: meta.name || meta.product,
+        price: meta.price,
+        image: meta.image || meta.imageUrl,
+        brand: meta.brand,
+        description: item.content,
+        inStock: this.determineStockStatus(item)
+      };
+    }
+    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
+    if (nameMatch) {
+      return {
+        id: item.id,
+        name: nameMatch[1],
+        price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
+        description: item.content,
+        inStock: this.determineStockStatus(item)
+      };
+    }
+    return null;
+  }
+  /**
+   * Helper: Detect categories in data
+   */
+  static detectCategories(data) {
+    const categories = /* @__PURE__ */ new Set();
+    data.forEach((item) => {
+      const meta = item.metadata || {};
+      if (meta.category) {
+        categories.add(String(meta.category));
+      }
+      if (meta.type) {
+        categories.add(String(meta.type));
+      }
+      if (meta.tag) {
+        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
+        tags.forEach((t) => categories.add(String(t)));
+      }
+      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
+      if (categoryMatch) {
+        categories.add(categoryMatch[1].trim());
+      }
+    });
+    return Array.from(categories);
+  }
+  /**
+   * Helper: Aggregate data by category
+   */
+  static aggregateByCategory(data, categories) {
+    const result = {};
+    categories.forEach((cat) => {
+      result[cat] = 0;
+    });
+    data.forEach((item) => {
+      const meta = item.metadata || {};
+      const itemCategory = meta.category || meta.type || "Other";
+      if (result.hasOwnProperty(itemCategory)) {
+        result[itemCategory]++;
+      } else {
+        result["Other"] = (result["Other"] || 0) + 1;
+      }
+    });
+    return result;
+  }
+  /**
+   * Helper: Extract time series data
+   */
+  static extractTimeSeriesData(data) {
+    return data.map((item) => {
+      const meta = item.metadata || {};
+      return {
+        timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
+        value: meta.value || item.score || 0,
+        label: meta.label || item.content.substring(0, 50)
+      };
+    });
+  }
+  /**
+   * Helper: Extract table columns
+   */
+  static extractTableColumns(data) {
+    const columnSet = /* @__PURE__ */ new Set();
+    columnSet.add("Content");
+    data.forEach((item) => {
+      Object.keys(item.metadata || {}).forEach((key) => {
+        columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
+      });
+    });
+    return Array.from(columnSet);
+  }
+  /**
+   * Helper: Extract table row
+   */
+  static extractTableRow(item, columns) {
+    const meta = item.metadata || {};
+    return columns.map((col) => {
+      if (col === "Content") {
+        return item.content.substring(0, 100);
+      }
+      const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
+      const value = meta[metaKey];
+      return value !== void 0 ? String(value) : "";
+    });
+  }
+  /**
+   * Helper: Check stock status
+   */
+  static checkStockStatus(category, data) {
+    const item = data.find((d) => {
+      const meta = d.metadata || {};
+      return meta.category === category || meta.type === category;
+    });
+    return this.determineStockStatus(item || {});
+  }
+  /**
+   * Helper: Determine if item is in stock
+   */
+  static determineStockStatus(item) {
+    const meta = item.metadata || {};
+    if (meta.inStock !== void 0) {
+      return Boolean(meta.inStock);
+    }
+    if (meta.stock !== void 0) {
+      return Boolean(meta.stock);
+    }
+    if (meta.available !== void 0) {
+      return Boolean(meta.available);
+    }
+    const content = (item.content || "").toLowerCase();
+    if (content.includes("out of stock") || content.includes("unavailable")) {
+      return false;
+    }
+    if (content.includes("in stock") || content.includes("available")) {
+      return true;
+    }
+    return true;
+  }
+  /**
+   * Helper: Check if data has multiple fields
+   */
+  static hasMultipleFields(data) {
+    const fieldCount = /* @__PURE__ */ new Set();
+    data.forEach((item) => {
+      Object.keys(item.metadata || {}).forEach((key) => {
+        fieldCount.add(key);
+      });
+    });
+    return fieldCount.size > 2;
+  }
+};
+
 // src/handlers/index.ts
 function sseFrame(payload) {
   return `data: ${JSON.stringify(payload)}
@@ -4441,6 +4806,11 @@ function sseTextFrame(text) {
 function sseMetaFrame(meta) {
   return `data: ${JSON.stringify(__spreadValues({ type: "metadata" }, meta != null ? meta : {}))}
 
+`;
+}
+function sseUIFrame(uiTransformation) {
+  return `data: ${JSON.stringify(__spreadValues({ type: "ui_transformation" }, uiTransformation != null ? uiTransformation : {}))}
+
 `;
 }
 function sseErrorFrame(message) {
@@ -4504,6 +4874,15 @@ function createStreamHandler(configOrPlugin) {
                 enqueue(sseTextFrame(chunk));
               } else {
                 enqueue(sseMetaFrame(chunk));
+                const sources = (chunk == null ? void 0 : chunk.sources) || [];
+                if (sources && sources.length > 0) {
+                  try {
+                    const uiTransformation = UITransformer.transform(message, sources);
+                    enqueue(sseUIFrame(uiTransformation));
+                  } catch (transformError) {
+                    console.warn("[createStreamHandler] UI transformation warning:", transformError);
+                  }
+                }
               }
             }
           } catch (temp) {
@@ -4623,5 +5002,6 @@ function createSuggestionsHandler(configOrPlugin) {
   sseErrorFrame,
   sseFrame,
   sseMetaFrame,
-  sseTextFrame
+  sseTextFrame,
+  sseUIFrame
 });

package/dist/handlers/index.mjs

@@ -8,8 +8,9 @@ import {
   sseErrorFrame,
   sseFrame,
   sseMetaFrame,
-  sseTextFrame
-} from "../chunk-OHXRQTQH.mjs";
+  sseTextFrame,
+  sseUIFrame
+} from "../chunk-M32ZXLTT.mjs";
 import "../chunk-YLTMFW4M.mjs";
 import "../chunk-X4TOT24V.mjs";
 export {
@@ -22,5 +23,6 @@ export {
   sseErrorFrame,
   sseFrame,
   sseMetaFrame,
-  sseTextFrame
+  sseTextFrame,
+  sseUIFrame
 };

package/dist/{index-kUXnRvuI.d.mts → index-BejNscWZ.d.mts}

@@ -125,8 +125,10 @@ declare class VectorPlugin {
 declare function sseFrame(payload: unknown): string;
 /** Encode a plain text chunk as an SSE frame. */
 declare function sseTextFrame(text: string): string;
-/** Encode the retrieval metadata as an SSE frame. */
+/** Encode the retrieval metadata with UI transformation as an SSE frame. */
 declare function sseMetaFrame(meta: unknown): string;
+/** Encode the UI transformation result as an SSE frame. */
+declare function sseUIFrame(uiTransformation: unknown): string;
 /** Encode a stream error as an SSE frame. */
 declare function sseErrorFrame(message: string): string;
 /**
@@ -215,4 +217,4 @@ declare function createSuggestionsHandler(configOrPlugin?: Partial<RagConfig> |
     suggestions: string[];
 }>>;
 
-export { ConfigValidator as C, type HealthCheckResult as H, type IProviderValidator as I, type ValidationError as V, type IProviderHealthChecker as a, VectorPlugin as b, createChatHandler as c, createHealthHandler as d, createIngestHandler as e, createStreamHandler as f, createUploadHandler as g, sseFrame as h, sseMetaFrame as i, sseTextFrame as j, createSuggestionsHandler as k, sseErrorFrame as s };
+export { ConfigValidator as C, type HealthCheckResult as H, type IProviderValidator as I, type ValidationError as V, type IProviderHealthChecker as a, VectorPlugin as b, createChatHandler as c, createHealthHandler as d, createIngestHandler as e, createStreamHandler as f, createUploadHandler as g, sseFrame as h, sseMetaFrame as i, sseTextFrame as j, createSuggestionsHandler as k, sseUIFrame as l, sseErrorFrame as s };

package/dist/{index-B67KQ9NN.d.ts → index-CbkMJvu5.d.ts}

@@ -125,8 +125,10 @@ declare class VectorPlugin {
 declare function sseFrame(payload: unknown): string;
 /** Encode a plain text chunk as an SSE frame. */
 declare function sseTextFrame(text: string): string;
-/** Encode the retrieval metadata as an SSE frame. */
+/** Encode the retrieval metadata with UI transformation as an SSE frame. */
 declare function sseMetaFrame(meta: unknown): string;
+/** Encode the UI transformation result as an SSE frame. */
+declare function sseUIFrame(uiTransformation: unknown): string;
 /** Encode a stream error as an SSE frame. */
 declare function sseErrorFrame(message: string): string;
 /**
@@ -215,4 +217,4 @@ declare function createSuggestionsHandler(configOrPlugin?: Partial<RagConfig> |
     suggestions: string[];
 }>>;
 
-export { ConfigValidator as C, type HealthCheckResult as H, type IProviderValidator as I, type ValidationError as V, type IProviderHealthChecker as a, VectorPlugin as b, createChatHandler as c, createHealthHandler as d, createIngestHandler as e, createStreamHandler as f, createUploadHandler as g, sseFrame as h, sseMetaFrame as i, sseTextFrame as j, createSuggestionsHandler as k, sseErrorFrame as s };
+export { ConfigValidator as C, type HealthCheckResult as H, type IProviderValidator as I, type ValidationError as V, type IProviderHealthChecker as a, VectorPlugin as b, createChatHandler as c, createHealthHandler as d, createIngestHandler as e, createStreamHandler as f, createUploadHandler as g, sseFrame as h, sseMetaFrame as i, sseTextFrame as j, createSuggestionsHandler as k, sseUIFrame as l, sseErrorFrame as s };