@retrivora-ai/rag-engine 1.7.3 → 1.7.5

package/dist/server.js

@@ -2482,7 +2482,8 @@ ${context}`;
         stopSequences: options == null ? void 0 : options.stop
       }
     });
-    return (_f = response.text) != null ? _f : "";
+    const text = typeof response.text === "function" ? response.text() : (_f = response.text) != null ? _f : "";
+    return text;
   }
   chatStream(messages, context, options) {
     return __asyncGenerator(this, null, function* () {
@@ -2516,7 +2517,7 @@ ${context}`;
       try {
         for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
           const chunk = temp.value;
-          const text = chunk.text;
+          const text = typeof chunk.text === "function" ? chunk.text() : chunk.text;
           if (text) yield text;
         }
       } catch (temp) {
@@ -3913,73 +3914,196 @@ var Pipeline = class {
   async initialize() {
     var _a, _b;
     if (this.initialised) return;
-    const CHART_MARKER = "<!-- UI_PROTOCOL_V6 -->";
+    const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
     const chartInstruction = `
 
 ${CHART_MARKER}
-### UNIVERSAL UI PROTOCOL
-When the user asks for a visual representation, comparison, distribution, breakdown, table, list, catalog, products, carousel, stock view, or category summary, you MUST include exactly one \`\`\`ui\`\`\` block.
+### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
 
-1. VIEW SELECTION
-   - Use "carousel" for browsing specific products, items, or a catalog. This is the DEFAULT for "show me", "what are", or "find me" requests for a specific category (e.g. "beauty").
-   - Use "chart" ONLY for numerical aggregations: distributions, percentages, counts, comparisons over time, or trends (e.g. "how many beauty products?").
-   - Use "table" for detailed technical specifications, row-heavy comparisons, or when the user explicitly asks for a list/table format.
+You are responsible for returning a SINGLE structured UI block when visualization is required.
 
-2. REQUIRED JSON SHAPE
-   {
-     "view": "chart" | "carousel" | "table",
-     "title": "Short heading",
-     "description": "One sentence describing the view",
-     "chartType": "pie" | "bar" | "line", // Required if view is "chart"
-     "xAxisKey": "label",
-     "dataKeys": ["value"],
-     "columns": ["dynamicFieldKey1", "dynamicFieldKey2"],
-     "insights": ["Short takeaway 1", "Short takeaway 2"],
-     "data": []
-   }
+---
 
-3. DATA RULES
-   - Build the UI payload from retrieved context only. Do not invent products or values.
-   - For CAROUSEL: "data" must be a list of product objects (id, name, brand, price, image, link).
-   - For CHART: "data" must be aggregated counts or sums. Never put raw product lists in a chart view.
-   - If the user asks for "products in beauty", do NOT return a pie chart of categories; return a carousel of the beauty products themselves.
+## 1. INTENT CLASSIFICATION (MANDATORY)
 
-4. FORMAT RULES
-   - Return valid JSON inside the \`\`\`ui\`\`\` block.
-   - Put natural language explanation before or after the block, not inside.
+Classify the query into ONE of the following intents:
+
+- "explore" \u2192 user wants to browse items (e.g., "show me products")
+- "analyze" \u2192 user wants aggregation/distribution (e.g., "distribution of products")
+- "compare" \u2192 user wants side-by-side comparison
+- "detail" \u2192 user wants structured/tabular data
+
+---
+
+## 2. VIEW MAPPING (STRICT)
+
+| Intent   | View       |
+|----------|-----------|
+| explore  | carousel  |
+| analyze  | chart     |
+| compare  | table     |
+| detail   | table     |
+
+\u26A0\uFE0F Override Rules:
+- If query contains "distribution", "percentage", "ratio" \u2192 ALWAYS chart
+- If query contains "show products", "list items" \u2192 ALWAYS carousel
+- NEVER mix views
+
+---
+
+## 3. RESPONSE FORMAT (STRICT JSON ONLY INSIDE BLOCK)
+
+\`\`\`ui
+{
+  "view": "carousel" | "chart" | "table",
+  "title": "string",
+  "description": "string",
+  "metadata": {
+    "intent": "explore" | "analyze" | "compare" | "detail",
+    "confidence": 0.0-1.0
+  },
+
+  // CHART ONLY
+  "chart": {
+    "type": "pie" | "bar" | "line",
+    "xKey": "label",
+    "yKeys": ["value"],
+    "data": []
+  },
+
+  // TABLE ONLY
+  "table": {
+    "columns": [],
+    "rows": []
+  },
+
+  // CAROUSEL ONLY
+  "carousel": {
+    "items": []
+  },
+
+  "insights": []
+}
+\`\`\`
+
+---
+
+## 4. DATA CONTRACT (CRITICAL)
+
+### \u{1F539} CAROUSEL ITEM FORMAT
+{
+  "id": "string",
+  "name": "string",
+  "brand": "string",
+  "price": number,
+  "image": "url",
+  "link": "url",
+  "inStock": boolean
+}
+
+### \u{1F539} CHART DATA FORMAT
+{
+  "label": "string",
+  "value": number,
+  "inStock": boolean (optional)
+}
+
+### \u{1F539} TABLE FORMAT
+- columns MUST match row keys
+- rows MUST be flat objects
+
+---
+
+## 5. DATA RULES
+
+- NEVER hallucinate fields
+- ONLY use retrieved vector DB data
+- NO placeholders ("...", "N/A")
+- If image/link missing \u2192 REMOVE FIELD
+- Aggregate BEFORE rendering chart
+
+---
+
+## 6. SMART RULES (IMPORTANT)
+
+- If user asks BOTH:
+  "show products AND distribution"
+  \u2192 PRIORITIZE "explore" \u2192 carousel
+
+- If dataset size > 20:
+  \u2192 aggregate \u2192 chart or table
+
+---
+
+## 7. OUTPUT RULES
+
+- ALWAYS return EXACTLY ONE \`\`\`ui\`\`\` block
+- JSON must be VALID
+- No explanation inside block
+
+---
+
+## 8. EXAMPLES
 
-5. EXAMPLES
 User: "show me beauty products"
-Assistant:
+
 \`\`\`ui
 {
   "view": "carousel",
   "title": "Beauty Products",
-  "description": "Browse our latest skincare and makeup collection.",
-  "data": [
-    { "id": "1", "name": "Mascara", "brand": "Essence", "price": "$9.99", "image": "...", "link": "..." },
-    { "id": "2", "name": "Lipstick", "brand": "Chic", "price": "$12.99", "image": "...", "link": "..." }
-  ]
+  "description": "Browse available beauty items",
+  "metadata": {
+    "intent": "explore",
+    "confidence": 0.95
+  },
+  "carousel": {
+    "items": [
+      {
+        "id": "1",
+        "name": "Mascara",
+        "brand": "Essence",
+        "price": 9.99,
+        "image": "https://example.com/mascara.jpg",
+        "link": "https://example.com/p1",
+        "inStock": true
+      }
+    ]
+  },
+  "insights": ["Most products are affordable"]
 }
 \`\`\`
 
-User: "breakdown of products by category"
-Assistant:
+---
+
+User: "distribution of products across categories"
+
 \`\`\`ui
 {
   "view": "chart",
-  "title": "Category Breakdown",
-  "chartType": "pie",
-  "xAxisKey": "label",
-  "dataKeys": ["value"],
-  "data": [
-    { "label": "Beauty", "value": 15 },
-    { "label": "Fragrances", "value": 8 }
-  ]
+  "title": "Product Distribution",
+  "description": "Category-wise product distribution",
+  "metadata": {
+    "intent": "analyze",
+    "confidence": 0.98
+  },
+  "chart": {
+    "type": "pie",
+    "xKey": "label",
+    "yKeys": ["value"],
+    "data": [
+      { "label": "Beauty", "value": 15 },
+      { "label": "Electronics", "value": 10 }
+    ]
+  },
+  "insights": ["Beauty dominates inventory"]
 }
-\`\`\``;
+\`\`\`
+`;
     if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
-      this.config.llm.systemPrompt = (this.config.llm.systemPrompt || "") + chartInstruction;
+      let cleanPrompt = this.config.llm.systemPrompt || "";
+      cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
+      cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
+      this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
     }
     console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
     this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
@@ -4803,11 +4927,15 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
     ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
     console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
     console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
+    const cleanKeywordQuery = (text) => {
+      return text.toLowerCase().replace(/\b(show|me|the|product|cards|of|category|find|list|browse|what|are|display)\b/g, "").trim().replace(/\s+/g, " & ");
+    };
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = "";
         let params = [];
         if (queryText) {
+          const cleanedText = cleanKeywordQuery(queryText) || queryText;
           const hasEntityHints = entityHints.length > 0;
           const exactNameScoreExpr = hasEntityHints ? `+ (
                  SELECT COALESCE(MAX(
@@ -4820,13 +4948,13 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
+             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
-          params = [vectorLiteral, queryText];
+          params = [vectorLiteral, cleanedText];
         } else {
           sqlQuery = `
             SELECT *, 

package/dist/server.mjs

@@ -39,7 +39,7 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "./chunk-M32ZXLTT.mjs";
+} from "./chunk-QOO37S2Z.mjs";
 import "./chunk-YLTMFW4M.mjs";
 import {
   PineconeProvider
@@ -325,11 +325,15 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
     ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
     console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
     console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
+    const cleanKeywordQuery = (text) => {
+      return text.toLowerCase().replace(/\b(show|me|the|product|cards|of|category|find|list|browse|what|are|display)\b/g, "").trim().replace(/\s+/g, " & ");
+    };
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = "";
         let params = [];
         if (queryText) {
+          const cleanedText = cleanKeywordQuery(queryText) || queryText;
           const hasEntityHints = entityHints.length > 0;
           const exactNameScoreExpr = hasEntityHints ? `+ (
                  SELECT COALESCE(MAX(
@@ -342,13 +346,13 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
+             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
-          params = [vectorLiteral, queryText];
+          params = [vectorLiteral, cleanedText];
         } else {
           sqlQuery = `
             SELECT *, 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "1.7.3",
+  "version": "1.7.5",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",

package/src/components/MessageBubble.tsx

@@ -32,8 +32,8 @@ function sanitizeJson(raw: string): string {
     },
   );
 
-  // 2. Strip JS-style single-line comments
-  s = s.replace(/\/\/[^\n]*/g, '');
+  // 2. Strip JS-style comments while preserving strings (e.g. URLs)
+  s = s.replace(/("(?:\\.|[^\\"])*")|\/\/[^\n]*|\/\*[\s\S]*?\*\//g, (m, g1) => g1 || '');
 
   // 3. Quote bare (unquoted) object keys
   s = s.replace(/([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)\s*:/g, '$1"$2":');
@@ -204,11 +204,14 @@ function isLikelyUiOnlyMessage(text: string): boolean {
 // ─── Tiny helpers ─────────────────────────────────────────────────────────────
 
 function resolveImage(data: Record<string, unknown>): string | undefined {
+  const isPlaceholder = (val: unknown) => 
+    typeof val === 'string' && (val === '...' || val === '' || val.toLowerCase() === 'null' || val.toLowerCase() === 'undefined');
+
   for (const key of ['image', 'img', 'thumbnail']) {
     const v = data[key];
-    if (typeof v === 'string' && v) return v;
+    if (typeof v === 'string' && v && !isPlaceholder(v)) return v;
   }
-  if (Array.isArray(data.images) && typeof data.images[0] === 'string') {
+  if (Array.isArray(data.images) && typeof data.images[0] === 'string' && !isPlaceholder(data.images[0])) {
     return data.images[0];
   }
   return undefined;
@@ -378,14 +381,46 @@ interface UIConfig {
   rows?: Record<string, unknown>[];
   products?: Record<string, unknown>[];
   results?: Record<string, unknown>[];
+
+  // V7 Protocol Fields
+  carousel?: {
+    items: Record<string, unknown>[];
+  };
+  chart?: {
+    type: 'pie' | 'bar' | 'line';
+    xKey: string;
+    yKeys: string[];
+    data: Record<string, unknown>[];
+  };
+  table?: {
+    columns: string[];
+    rows?: Record<string, unknown>[];
+    data?: Record<string, unknown>[];
+  };
 }
 
-interface RawUIConfig extends Omit<UIConfig, 'data' | 'items' | 'rows' | 'products' | 'results'> {
+interface RawUIConfig extends Omit<UIConfig, 'data' | 'items' | 'rows' | 'products' | 'results' | 'carousel' | 'chart' | 'table'> {
   data: Array<Record<string, unknown> | unknown[]>;
   items?: Array<Record<string, unknown> | unknown[]>;
   rows?: Array<Record<string, unknown> | unknown[]>;
   products?: Array<Record<string, unknown> | unknown[]>;
   results?: Array<Record<string, unknown> | unknown[]>;
+
+  // V7 Protocol Fields (Raw)
+  carousel?: {
+    items: Array<Record<string, unknown> | unknown[]>;
+  };
+  chart?: {
+    type: 'pie' | 'bar' | 'line';
+    xKey: string;
+    yKeys: string[];
+    data: Array<Record<string, unknown> | unknown[]>;
+  };
+  table?: {
+    columns: string[];
+    rows?: Array<Record<string, unknown> | unknown[]>;
+    data?: Array<Record<string, unknown> | unknown[]>;
+  };
 }
 
 function normalizeTabularRows(
@@ -417,6 +452,13 @@ function normalizeTabularRows(
 }
 
 function resolveStructuredRows(config: Partial<RawUIConfig> & Record<string, unknown>) {
+  // V7 Nested Structures
+  if (config.carousel?.items && Array.isArray(config.carousel.items)) return config.carousel.items;
+  if (config.chart?.data && Array.isArray(config.chart.data)) return config.chart.data;
+  if (config.table?.rows && Array.isArray(config.table.rows)) return config.table.rows;
+  if (config.table?.data && Array.isArray(config.table.data)) return config.table.data;
+
+  // V6 Flat Structures
   if (Array.isArray(config.data) && config.data.length > 0) return config.data;
   if (Array.isArray(config.items) && config.items.length > 0) return config.items;
   if (Array.isArray(config.products) && config.products.length > 0) return config.products;
@@ -442,6 +484,17 @@ function UIDispatcher({ rawContent, primaryColor, accentColor, isStreaming, onAd
         .trim();
       const parsed = JSON.parse(sanitizeJson(extractLikelyJsonBlock(clean)));
       const config = { ...parsed } as Partial<RawUIConfig> & Record<string, unknown>;
+      
+      // V7 Field Mapping
+      if (config.chart) {
+        if (!config.chartType) config.chartType = config.chart.type;
+        if (!config.xAxisKey) config.xAxisKey = config.chart.xKey;
+        if (!config.dataKeys) config.dataKeys = config.chart.yKeys;
+      }
+      if (config.table) {
+        if (!config.columns) config.columns = config.table.columns;
+      }
+
       const resolvedRows = resolveStructuredRows(config);
 
       // ─── Intent Healing & Auto-Detection ───────────────────────────────────