@retrivora-ai/rag-engine 1.7.9 → 1.8.1

package/src/components/MessageBubble.tsx

@@ -2,15 +2,13 @@
 
 import React from 'react';
 import { Bot, User, ChevronDown, ChevronRight } from 'lucide-react';
-import ReactMarkdown from 'react-markdown';
-import remarkGfm from 'remark-gfm';
-import { MessageBubbleProps, Product } from '../types';
+import ReactMarkdown, { Components } from 'react-markdown';
+import { MessageBubbleProps, Product, UITransformationResponse } from '../types';
 import { ChatViewportSize } from '../types/props';
 import { SourceCard } from './SourceCard';
 import { ProductCarousel } from './ProductCarousel';
 import { DynamicChart } from './DynamicChart';
 import { VisualizationRenderer } from './VisualizationRenderer';
-import { UITransformationResponse } from '../utils/UITransformer';
 
 // ─── JSON sanitization ────────────────────────────────────────────────────────
 // Order matters: each step is a targeted fix, no step undoes a previous one.
@@ -126,50 +124,52 @@ function stripStructuredUiLabels(raw: string): string {
 }
 
 function extractBareJsonObject(raw: string): string | null {
-  const start = raw.indexOf('{');
+  // Find the first occurrence of '{'
+  let start = raw.indexOf('{');
   if (start === -1) return null;
 
-  let depth = 0;
-  let inString = false;
-  let escape = false;
+  // We want to find the LARGEST valid JSON object in the string.
+  // Sometimes there might be a small JSON object followed by a larger one.
+  // We'll iterate through all { and find the one that spans the most.
+  let bestJson: string | null = null;
+  let maxLen = 0;
 
-  for (let i = start; i < raw.length; i += 1) {
-    const ch = raw[i];
-
-    if (escape) {
-      escape = false;
-      continue;
-    }
-
-    if (ch === '\\' && inString) {
-      escape = true;
-      continue;
-    }
-
-    if (ch === '"') {
-      inString = !inString;
-      continue;
-    }
+  while (start !== -1) {
+    let depth = 0;
+    let inString = false;
+    let escape = false;
 
-    if (inString) continue;
+    for (let i = start; i < raw.length; i += 1) {
+      const ch = raw[i];
+      if (escape) { escape = false; continue; }
+      if (ch === '\\' && inString) { escape = true; continue; }
+      if (ch === '"') { inString = !inString; continue; }
+      if (inString) continue;
 
-    if (ch === '{') depth += 1;
-    if (ch === '}') {
-      depth -= 1;
-      if (depth === 0) {
-        return raw.slice(start, i + 1);
+      if (ch === '{') depth += 1;
+      else if (ch === '}') {
+        depth -= 1;
+        if (depth === 0) {
+          const candidate = raw.slice(start, i + 1);
+          if (candidate.length > maxLen) {
+            maxLen = candidate.length;
+            bestJson = candidate;
+          }
+          break; // Found the end of this object
+        }
       }
     }
+    start = raw.indexOf('{', start + 1);
   }
 
-  return null;
+  return bestJson;
 }
 
 function extractStructuredPayload(raw: string): { payload: string | null; text: string } {
   let working = raw;
   const fenceRegex = /```(?:[a-z]+)?\s*([\s\S]*?)```/gi;
 
-  for (const match of raw.matchAll(fenceRegex)) {
+  for (const match of Array.from(raw.matchAll(fenceRegex))) {
     const body = match[1]?.trim() ?? '';
     if (!looksLikeStructuredPayload(body)) continue;
 
@@ -199,28 +199,44 @@ function isLikelyUiOnlyMessage(text: string): boolean {
   const trimmed = text.trim();
   return (
     trimmed.length === 0 ||
-    /^(chart data|table data|visualization|visual representation|product catalog|product recommendations|catalog summary)\s*:?\s*$/i.test(trimmed)
+    trimmed.length < 30 ||
+    /^(chart data|table data|visualization|visual representation|product catalog|product recommendations|catalog summary|availability|inventory status|details)\s*:?\s*$/i.test(trimmed)
   );
 }
 
 // ─── Tiny helpers ─────────────────────────────────────────────────────────────
 
 function resolveImage(data: Record<string, unknown>): string | undefined {
-  const isPlaceholder = (val: unknown) => 
+  const isPlaceholder = (val: unknown) =>
     typeof val === 'string' && (val === '...' || val === '' || val.toLowerCase() === 'null' || val.toLowerCase() === 'undefined');
 
-  for (const key of ['image', 'img', 'thumbnail']) {
+  for (const key of ['image', 'img', 'thumbnail', 'images', 'product_image', 'Image', 'Thumbnail']) {
     const v = data[key];
     if (typeof v === 'string' && v && !isPlaceholder(v)) return v;
-  }
-  if (Array.isArray(data.images) && typeof data.images[0] === 'string' && !isPlaceholder(data.images[0])) {
-    return data.images[0];
+    if (Array.isArray(v) && typeof v[0] === 'string' && v[0] && !isPlaceholder(v[0])) return v[0];
   }
   return undefined;
 }
 
+/** Robustly extract a value from metadata by checking synonyms and case-insensitivity. */
+function getMetadataValue(meta: Record<string, unknown>, keys: string[]): unknown {
+  const searchKeys = keys.map(k => k.toLowerCase());
+  const metaKeys = Object.keys(meta);
+
+  // Try direct case-insensitive match
+  const foundKey = metaKeys.find(k => searchKeys.includes(k.toLowerCase()));
+  if (foundKey) return meta[foundKey];
+
+  // Try partial match
+  const partialKey = metaKeys.find(k => {
+    const kLow = k.toLowerCase();
+    return searchKeys.some(sk => kLow.includes(sk) || sk.includes(kLow));
+  });
+  return partialKey ? meta[partialKey] : undefined;
+}
+
 /** Safely render table cell children that might be plain objects. */
-function normaliseChild(children: React.ReactNode): React.ReactNode {
+function normaliseChild(children: unknown): React.ReactNode {
   if (
     typeof children === 'object' &&
     children !== null &&
@@ -229,7 +245,7 @@ function normaliseChild(children: React.ReactNode): React.ReactNode {
   ) {
     return JSON.stringify(children);
   }
-  return children;
+  return children as React.ReactNode;
 }
 
 function normaliseFieldName(value: string): string {
@@ -431,6 +447,15 @@ function normalizeTabularRows(
 ): Record<string, unknown>[] {
   if (!Array.isArray(rows)) return [];
 
+  // Common synonyms for mapping natural language headers to technical keys
+  const SYNONYMS: Record<string, string[]> = {
+    name: ['product', 'item', 'title', 'label', 'heading', 'product name', 'item name'],
+    price: ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd'],
+    brand: ['manufacturer', 'vendor', 'make', 'company', 'brand_name', 'supplier'],
+    image: ['imageUrl', 'thumbnail', 'img', 'url', 'photo', 'picture', 'media'],
+    stock: ['inventory', 'quantity', 'count', 'availability', 'stock_level', 'in stock', 'status'],
+  };
+
   return rows
     .map((row) => {
       if (Array.isArray(row)) {
@@ -445,7 +470,46 @@ function normalizeTabularRows(
       }
 
       if (row && typeof row === 'object') {
-        return row as Record<string, unknown>;
+        const result: Record<string, unknown> = { ...row };
+
+        // Self-Healing: If columns are provided, ensure the row has keys matching those columns
+        if (Array.isArray(columns)) {
+          columns.forEach(col => {
+            if (result[col] !== undefined) return; // Already matches
+
+            // Try to find a match among row keys
+            const rowKeys = Object.keys(row);
+            const colLower = col.toLowerCase().replace(/_/g, ' ');
+
+            // 1. Direct case-insensitive / space-to-underscore match
+            const foundKey = rowKeys.find(rk => {
+              const rkLower = rk.toLowerCase().replace(/_/g, ' ');
+              return rkLower === colLower || rkLower.includes(colLower) || colLower.includes(rkLower);
+            });
+
+            if (foundKey) {
+              result[col] = row[foundKey];
+              return;
+            }
+
+            // 2. Synonym match
+            for (const [target, aliases] of Object.entries(SYNONYMS)) {
+              const isMatch = target === colLower || aliases.some(a => colLower.includes(a) || a.includes(colLower));
+              if (isMatch) {
+                const mappedKey = rowKeys.find(rk => {
+                  const rkL = rk.toLowerCase();
+                  return rkL === target || aliases.some(a => rkL.includes(a) || a.includes(rkL));
+                });
+                if (mappedKey) {
+                  result[col] = row[mappedKey];
+                  break;
+                }
+              }
+            }
+          });
+        }
+
+        return result;
       }
 
       return { value: row };
@@ -486,7 +550,7 @@ function UIDispatcher({ rawContent, primaryColor, accentColor, isStreaming, onAd
         .trim();
       const parsed = JSON.parse(sanitizeJson(extractLikelyJsonBlock(clean)));
       const config = { ...parsed } as Partial<RawUIConfig> & Record<string, unknown>;
-      
+
       // V7 Field Mapping
       if (config.chart) {
         if (!config.chartType) config.chartType = config.chart.type;
@@ -521,18 +585,25 @@ function UIDispatcher({ rawContent, primaryColor, accentColor, isStreaming, onAd
         config.chartType = (config.chartType as 'pie' | 'bar' | 'line') || 'bar';
       }
 
-      // 2. Auto-Detection: Fill in missing view
-      if (!config.view) {
-        if (config.type === 'products' || Array.isArray(config.items) || hasProductLikeData) {
+      // 2. Auto-Detection: Fill in missing or dynamic view
+      if (!config.view || ['pie', 'bar', 'line', 'pie_chart', 'bar_chart', 'line_chart'].includes(String(config.view))) {
+        const viewStr = String(config.view || '').toLowerCase();
+
+        if (viewStr.includes('carousel') || config.type === 'products' || Array.isArray(config.items) || hasProductLikeData) {
           config.view = 'carousel';
         } else if (
+          viewStr.includes('chart') ||
+          viewStr.includes('pie') || viewStr.includes('bar') || viewStr.includes('line') ||
           hasAggregationLikeData ||
           (typeof config.type === 'string' && ['pie', 'bar', 'line'].includes(config.type)) ||
           (typeof config.chartType === 'string' && ['pie', 'bar', 'line'].includes(config.chartType))
         ) {
           config.view = 'chart';
           config.chartType = (config.chartType as 'pie' | 'bar' | 'line') ||
+            (viewStr.replace('_chart', '') as 'pie' | 'bar' | 'line') ||
             (config.type as 'pie' | 'bar' | 'line') || 'bar';
+        } else if (viewStr.includes('table')) {
+          config.view = 'table';
         } else {
           config.view = 'table';
         }
@@ -589,12 +660,12 @@ function UIDispatcher({ rawContent, primaryColor, accentColor, isStreaming, onAd
           />
           {hasInsights && (
             <div className="mt-4 flex flex-wrap gap-2 px-2">
-              {config.insights?.map((insight) => (
+              {config.insights?.map((insight, i) => (
                 <span
-                  key={insight}
+                  key={`insight-${i}`}
                   className={`rounded-full border border-emerald-200 bg-emerald-50 ${isCompact ? 'px-2.5 py-1 text-[10px]' : 'px-3 py-1 text-[11px]'} font-medium text-emerald-700 dark:border-emerald-500/20 dark:bg-emerald-500/10 dark:text-emerald-300`}
                 >
-                  {insight}
+                  {String(insight)}
                 </span>
               ))}
             </div>
@@ -654,16 +725,23 @@ export function MessageBubble({
   const isCompact = viewportSize === 'compact';
   const isMedium = viewportSize === 'medium';
   const [showSources, setShowSources] = React.useState(false);
-  const hasStructuredProductBlock = React.useMemo(
-    () => /```(?:ui|json|chart)\s*[\s\S]*?"?(?:view|type|data|products)"?\s*:\s*"?(?:carousel|products|table)"?/i.test(message.content) ||
-      (/\{[\s\S]*?"?(?:view|type|data|products)"?\s*:\s*"?(?:carousel|products)"?[\s\S]*\}/i.test(message.content) && looksLikeStructuredPayload(message.content)),
-    [message.content],
-  );
   const structuredContent = React.useMemo(
     () => isUser ? { payload: null, text: message.content } : extractStructuredPayload(message.content),
     [isUser, message.content],
   );
-  const shouldRenderStructuredOnly = !isUser && Boolean(structuredContent.payload) && isLikelyUiOnlyMessage(structuredContent.text);
+  const hasRichUI = React.useMemo(() => {
+    if (message.uiTransformation) {
+      const type = (message.uiTransformation as UITransformationResponse).type;
+      // Table is not considered "Rich enough" to block a product carousel
+      if (type && !['text', 'table'].includes(type)) return true;
+    }
+    if (structuredContent.payload) {
+      return /"?(?:view|type|chartType)"?\s*:\s*"?(?:chart|carousel|pie|bar|line|product_carousel)"?/i.test(structuredContent.payload);
+    }
+    return false;
+  }, [message.uiTransformation, structuredContent.payload]);
+
+  const shouldRenderStructuredOnly = !isUser && hasRichUI && isLikelyUiOnlyMessage(structuredContent.text);
 
   // ── Products from sources ──────────────────────────────────────────────────
   const productsFromSources = React.useMemo<Product[]>(() => {
@@ -671,17 +749,26 @@ export function MessageBubble({
     return sources
       .filter(s => {
         const m = s.metadata ?? {};
-        return m.price || m.image || m.img || m.thumbnail || m.images || m.brand || m.type === 'product';
+        const keys = Object.keys(m).map(k => k.toLowerCase());
+        const hasProductKey = keys.some(k =>
+          ['price', 'image', 'img', 'thumbnail', 'images', 'brand', 'product', 'sku', 'category', 'model', 'cost'].includes(k)
+        );
+        const hasPricePattern = /\$\s*\d+/.test(s.content);
+        return hasProductKey || hasPricePattern || m.type === 'product';
       })
       .map(s => {
         const m = (s.metadata ?? {}) as Record<string, unknown>;
+        const name = getMetadataValue(m, ['name', 'product', 'title', 'label', 'item']) as string;
+        const brand = getMetadataValue(m, ['brand', 'manufacturer', 'vendor', 'make']) as string;
+        const price = getMetadataValue(m, ['price', 'cost', 'amount', 'msrp', 'rate']) as string | number;
+
         return {
           id: s.id,
-          name: (m.name ?? m.title ?? s.content.split('\n')[0] ?? 'Unknown Product') as string,
-          brand: m.brand as string,
-          price: m.price as string | number,
+          name: name ?? s.content.split('\n')[0] ?? 'Unknown Product',
+          brand: brand,
+          price: price,
           image: resolveImage(m),
-          link: m.link as string,
+          link: getMetadataValue(m, ['link', 'url', 'product_url']) as string,
           description: s.content,
         };
       });
@@ -697,7 +784,7 @@ export function MessageBubble({
     const products: Product[] = [];
     let content = structuredContent.text;
 
-    const payloadCandidates = [structuredContent.payload, ...content.matchAll(jsonRegex).map((match) => match[1])].filter(Boolean) as string[];
+    const payloadCandidates = [structuredContent.payload, ...Array.from(content.matchAll(jsonRegex)).map((match) => match[1])].filter(Boolean) as string[];
 
     for (const candidate of payloadCandidates) {
       try {
@@ -725,7 +812,7 @@ export function MessageBubble({
     }
 
     if (content.includes('"type": "products"') || content.includes('"type":"products"')) {
-      for (const match of content.matchAll(jsonRegex)) {
+      for (const match of Array.from(content.matchAll(jsonRegex))) {
         try {
           const data = JSON.parse(sanitizeJson(match[1]));
           if (data.type === 'products' && Array.isArray(data.items)) {
@@ -743,8 +830,52 @@ export function MessageBubble({
       }
     }
 
+    // 4. Fallback: Scan for plain text bulleted products (e.g. • Name - $Price)
+    // Support various bullets (optional) and dash types (-, –, —)
+    const bulletRegex = /(?:[•*-]\s*)?([^•\n\-\$*–—\(]+?)(?:\s*\(?Price\s*[:\-–—]?\s*)?(?:\s*[:\-–—]\s*|\s+)\$?([\d,.]+)(?:\s*USD)?/gi;
+    const matches = Array.from(content.matchAll(bulletRegex));
+
+    if (matches.length >= 2) {
+      const normalize = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, '');
+
+      for (const match of matches) {
+        let name = match[1]?.trim() || '';
+        // Clean up common leftovers
+        name = name.replace(/\s*\(?Price\s*$/i, '').replace(/[:\-–—]\s*$/g, '').trim();
+
+        let price = match[2]?.trim() || '';
+        price = price.replace(/[,\s]+$/, '').trim(); // Remove trailing commas or spaces
+
+        if (name && price) {
+          const newProduct: Product = {
+            id: `text-prod-${name}-${price}`,
+            name,
+            price: `$${price}`,
+          };
+
+          // Rehydration: Fuzzy match in sources to get image/brand/link
+          const normName = normalize(name);
+          const sourceMatch = productsFromSources.find(s => {
+            const sn = normalize(s.name);
+            return sn.includes(normName) || normName.includes(sn);
+          });
+
+          if (sourceMatch) {
+            newProduct.image = sourceMatch.image;
+            newProduct.brand = sourceMatch.brand;
+            newProduct.link = sourceMatch.link;
+            newProduct.description = sourceMatch.description;
+          }
+
+          products.push(newProduct);
+          content = content.replace(match[0], `\n**${name}** — $${price}`);
+        }
+      }
+      content = content.replace(/\n{3,}/g, '\n\n').trim();
+    }
+
     return { productsFromContent: products, cleanContent: content.trim() };
-  }, [message.content, isUser, structuredContent]);
+  }, [message.content, isUser, structuredContent, productsFromSources]);
 
   // ── Merge & deduplicate products ───────────────────────────────────────────
   const allProducts = React.useMemo<Product[]>(() => {
@@ -764,7 +895,7 @@ export function MessageBubble({
     });
   }, [productsFromSources, productsFromContent, message.content]);
 
-  const processedMarkdown = React.useMemo(() => {
+  const processedMarkdown = React.useMemo<string>(() => {
     let raw = structuredContent.payload ? structuredContent.text : (cleanContent || message.content);
     const hasStructuredUiBlock = Boolean(structuredContent.payload) || /```(?:ui|json|chart)\s*[\s\S]*?"(?:view|chartType|type|data|items|rows|products|results)"\s*:/i.test(raw);
 
@@ -806,7 +937,7 @@ export function MessageBubble({
   }, [cleanContent, message.content, isStreaming, structuredContent]);
 
   // ── Markdown component overrides ───────────────────────────────────────────
-  const markdownComponents = React.useMemo(
+  const markdownComponents: Components = React.useMemo(
     () => ({
       // Wrap in not-prose so Tailwind Typography resets don't clobber our styles.
       // prose applies display:block and padding:0 to table/th/td — not-prose opts out.
@@ -985,18 +1116,12 @@ export function MessageBubble({
               )}
 
               {!shouldRenderStructuredOnly && (
-                <ReactMarkdown remarkPlugins={[remarkGfm]} components={markdownComponents}>
+                <ReactMarkdown components={markdownComponents}>
                   {processedMarkdown}
                 </ReactMarkdown>
               )}
 
               {/* Automated UI Fallback (if LLM failed to output structured block but UITransformer found data) */}
-              {!isUser && !structuredContent.payload && !!message.uiTransformation && (
-                <div className="mt-4 border-t border-slate-100 dark:border-white/5 pt-4">
-                  <VisualizationRenderer data={message.uiTransformation as UITransformationResponse} />
-                </div>
-              )}
-
               {isStreaming && message.content && (
                 <span className="inline-block w-1.5 h-3.5 ml-1 bg-emerald-500 animate-pulse align-middle" />
               )}
@@ -1004,8 +1129,32 @@ export function MessageBubble({
           )}
         </div>
 
-        {/* Product Carousel */}
-        {!isUser && !hasStructuredProductBlock && allProducts.length > 0 && (
+        {/* Auto-generated visualization fallback (only if no inline UI payload and not a redundant text/table block) */}
+        {(() => {
+          if (isUser || structuredContent.payload || !message.uiTransformation) return null;
+          const ui = message.uiTransformation as UITransformationResponse;
+          const textContent = (ui.data as { content?: string })?.content ?? '';
+          const shouldShow =
+            (ui.type === 'table' && allProducts.length === 0) ||
+            !['text', 'table'].includes(ui.type) ||
+            (ui.type === 'text' && !hasRichUI && allProducts.length === 0 && textContent &&
+              !processedMarkdown.toLowerCase().includes(textContent.trim().toLowerCase()));
+          if (!shouldShow) return null;
+          return (
+            <div className="w-full mt-3">
+              <VisualizationRenderer
+                data={ui}
+                primaryColor={primaryColor}
+                onAddToCart={onAddToCart}
+                className="rounded-3xl border border-slate-200/60 dark:border-white/10 bg-white/90 dark:bg-slate-900/80 p-4"
+              />
+            </div>
+          );
+        })()}
+
+
+        {/* Product Carousel (only if no other rich UI is present) */}
+        {!isUser && !hasRichUI && allProducts.length > 0 && (
           <div className="w-full mt-1">
             <ProductCarousel
               products={allProducts}

package/src/components/ProductCard.tsx

@@ -24,7 +24,7 @@ export function ProductCard({ product, primaryColor = '#6366f1', onAddToCart }:
             <ShoppingCart className="w-12 h-12" />
           </div>
         )}
-        
+
         {/* Brand Badge */}
         {product.brand && (
           <div className="absolute top-2 left-2 px-2 py-1 bg-white/90 dark:bg-black/60 backdrop-blur-md rounded-md text-[10px] font-bold uppercase tracking-wider text-slate-700 dark:text-white/90 shadow-sm">
@@ -45,7 +45,7 @@ export function ProductCard({ product, primaryColor = '#6366f1', onAddToCart }:
             </span>
           )}
         </div>
-        
+
         {product.description && (
           <p className="text-[11px] text-slate-500 dark:text-white/50 line-clamp-2 leading-relaxed">
             {product.description}