@retrivora-ai/rag-engine 1.3.1 → 1.3.3

package/src/components/MessageBubble.tsx

@@ -9,6 +9,132 @@ import { SourceCard } from './SourceCard';
 import { ProductCarousel } from './ProductCarousel';
 import { DynamicChart, ChartConfig } from './DynamicChart';
 
+// ─── JSON sanitization ────────────────────────────────────────────────────────
+// Order matters: each step is a targeted fix, no step undoes a previous one.
+
+function sanitizeJson(raw: string): string {
+  let s = raw.trim();
+
+  // 1. Evaluate simple arithmetic expressions in values  e.g.  "v": 495 / 1000
+  s = s.replace(
+    /:\s*(-?\d+(?:\.\d+)?)\s*([+\-*/])\s*(-?\d+(?:\.\d+)?)/g,
+    (_, a, op, b) => {
+      const n1 = parseFloat(a);
+      const n2 = parseFloat(b);
+      const result =
+        op === '+' ? n1 + n2
+          : op === '-' ? n1 - n2
+            : op === '*' ? n1 * n2
+              : n2 !== 0 ? n1 / n2
+                : n1;
+      return `: ${result}`;
+    },
+  );
+
+  // 2. Strip JS-style single-line comments
+  s = s.replace(/\/\/[^\n]*/g, '');
+
+  // 3. Quote bare (unquoted) object keys
+  s = s.replace(/([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)\s*:/g, '$1"$2":');
+
+  // 4. Replace single-quoted strings with double-quoted ones
+  s = s.replace(/'([^'\\]*(?:\\.[^'\\]*)*)'/g, '"$1"');
+
+  // 5. Remove trailing commas before ] or }
+  s = s.replace(/,\s*([}\]])/g, '$1');
+
+  // 6. Remove completely empty key-value pairs left by prior cleanup
+  s = s.replace(/([{,])\s*""\s*(?=[,}])/g, '$1');
+
+  // 7. Balance braces/brackets: walk char-by-char, track depth, append closers
+  //    Do this BEFORE the lastValidIndex trim so we don't cut valid JSON.
+  {
+    let inString = false;
+    let escape = false;
+    let braces = 0;
+    let brackets = 0;
+
+    for (const ch of s) {
+      if (escape) { escape = false; continue; }
+      if (ch === '\\' && inString) { escape = true; continue; }
+      if (ch === '"') { inString = !inString; continue; }
+      if (inString) continue;
+      if (ch === '{') braces++;
+      else if (ch === '}') braces--;
+      else if (ch === '[') brackets++;
+      else if (ch === ']') brackets--;
+    }
+
+    // Close any unclosed string first
+    if (inString) s += '"';
+    // Then close structures in the correct order
+    while (brackets > 0) { s += ']'; brackets--; }
+    while (braces > 0) { s += '}'; braces--; }
+  }
+
+  // 8. Trim to the last valid closing char only if there is trailing garbage
+  //    (e.g. a partial second object after the first one ends)
+  const lastClose = Math.max(s.lastIndexOf('}'), s.lastIndexOf(']'));
+  if (lastClose !== -1 && lastClose < s.length - 1) {
+    s = s.substring(0, lastClose + 1);
+  }
+
+  return s;
+}
+
+// ─── Tiny helpers ─────────────────────────────────────────────────────────────
+
+function resolveImage(data: Record<string, unknown>): string | undefined {
+  for (const key of ['image', 'img', 'thumbnail']) {
+    const v = data[key];
+    if (typeof v === 'string' && v) return v;
+  }
+  if (Array.isArray(data.images) && typeof data.images[0] === 'string') {
+    return data.images[0];
+  }
+  return undefined;
+}
+
+// ─── Chart code block renderer ────────────────────────────────────────────────
+
+interface ChartBlockProps {
+  rawContent: string;
+  primaryColor: string;
+  accentColor: string;
+}
+
+function ChartBlock({ rawContent, primaryColor, accentColor }: ChartBlockProps) {
+  const result = React.useMemo<{ config: ChartConfig } | { error: string }>(() => {
+    if (!rawContent || rawContent === 'undefined') return { error: 'Empty chart config.' };
+    try {
+      const sanitized = sanitizeJson(rawContent);
+      const config = JSON.parse(sanitized) as ChartConfig;
+      return { config };
+    } catch (err) {
+      const sanitized = sanitizeJson(rawContent);
+      console.error('[ChartBlock] Parsing failed.\nError:', err, '\nSanitized:\n', sanitized);
+      return { error: String(err) };
+    }
+  }, [rawContent]);
+
+  if ('error' in result) {
+    return (
+      <div className="p-4 bg-red-50 dark:bg-red-900/20 text-red-600 dark:text-red-400 rounded-lg text-sm border border-red-100 dark:border-red-900/30">
+        <p className="font-medium mb-1">Failed to render chart</p>
+        <p className="opacity-70 text-xs">The generated configuration is invalid. Check the console for details.</p>
+      </div>
+    );
+  }
+
+  return (
+    <div className="my-4 p-4 bg-white dark:bg-slate-900 rounded-xl border border-slate-200 dark:border-white/10 shadow-sm">
+      <DynamicChart config={result.config} primaryColor={primaryColor} accentColor={accentColor} />
+    </div>
+  );
+}
+
+// ─── Component ────────────────────────────────────────────────────────────────
+
 export function MessageBubble({
   message,
   sources,
@@ -20,71 +146,51 @@ export function MessageBubble({
   const isUser = message.role === 'user';
   const [showSources, setShowSources] = React.useState(false);
 
-  // Helper to extract image from metadata or object
-  const resolveImage = (data: Record<string, unknown>): string | undefined => {
-    if (!data) return undefined;
-
-    // Check single string fields
-    const singleFields = ['image', 'img', 'thumbnail'];
-    for (const field of singleFields) {
-      const val = data[field];
-      if (typeof val === 'string' && val) return val;
-    }
-
-    // Check images array
-    if (Array.isArray(data.images) && data.images.length > 0) {
-      if (typeof data.images[0] === 'string') return data.images[0];
-    }
-
-    return undefined;
-  };
-
-  // Extract products from sources
-  const productsFromSources = React.useMemo(() => {
+  // ── Products from sources ──────────────────────────────────────────────────
+  const productsFromSources = React.useMemo<Product[]>(() => {
     if (isUser || !sources) return [];
     return sources
       .filter(s => {
-        const m = s.metadata || {};
+        const m = s.metadata ?? {};
         return m.price || m.image || m.img || m.thumbnail || m.images || m.brand || m.type === 'product';
       })
       .map(s => {
-        const m = (s.metadata || {}) as Record<string, unknown>;
+        const m = (s.metadata ?? {}) as Record<string, unknown>;
         return {
           id: s.id,
-          name: (m.name || m.title || s.content.split('\n')[0] || 'Unknown Product') as string,
+          name: (m.name ?? m.title ?? s.content.split('\n')[0] ?? 'Unknown Product') as string,
           brand: m.brand as string,
           price: m.price as string | number,
           image: resolveImage(m),
           link: m.link as string,
-          description: s.content
+          description: s.content,
         };
       });
   }, [sources, isUser]);
 
-  // Extract products from content (structured JSON)
+  // ── Products + cleaned content from structured JSON blocks in the message ──
   const { productsFromContent, cleanContent } = React.useMemo(() => {
-    if (isUser) return { productsFromContent: [], cleanContent: message.content };
+    if (isUser) return { productsFromContent: [] as Product[], cleanContent: message.content };
 
     const jsonRegex = /```json\s*([\s\S]*?)\s*```/g;
     const products: Product[] = [];
     let content = message.content;
 
-    // Only process if it looks like it might contain product data
     if (content.includes('"type": "products"') || content.includes('"type":"products"')) {
-      const matches = Array.from(content.matchAll(jsonRegex));
-      for (const m of matches) {
+      for (const match of content.matchAll(jsonRegex)) {
         try {
-          const data = JSON.parse(m[1]);
+          const data = JSON.parse(match[1]);
           if (data.type === 'products' && Array.isArray(data.items)) {
-            const formattedItems = data.items.map((item: Record<string, unknown>) => ({
-              ...item,
-              image: (item.image as string) || resolveImage(item)
-            })) as Product[];
-            products.push(...formattedItems);
-            content = content.replace(m[0], '');
+            products.push(
+              ...data.items.map((item: Record<string, unknown>) => ({
+                ...item,
+                image: (item.image as string) ?? resolveImage(item),
+              })) as Product[],
+            );
+            content = content.replace(match[0], '');
           }
         } catch {
-          // Skip invalid JSON
+          // Malformed product block — skip silently
         }
       }
     }
@@ -92,60 +198,110 @@ export function MessageBubble({
     return { productsFromContent: products, cleanContent: content.trim() };
   }, [message.content, isUser]);
 
-  const allProducts = React.useMemo(() => {
-    // If the LLM explicitly provided products in a structured format, only show those.
+  // ── Merge & deduplicate products ───────────────────────────────────────────
+  const allProducts = React.useMemo<Product[]>(() => {
     if (productsFromContent.length > 0) return productsFromContent;
 
-    // Fallback: Show unique products from sources, but ONLY if they are mentioned in the message text.
-    // This aligns the carousel with what the assistant is actually "displaying" in its response.
-    const uniqueProducts: Product[] = [];
-    const seenIds = new Set();
+    const seen = new Set<string>();
     const contentLower = message.content.toLowerCase();
 
-    for (const p of productsFromSources) {
-      const identifier = p.id || p.name;
-      if (seenIds.has(identifier)) continue;
-
-      // Check if the product name or brand is mentioned in the assistant's text
-      const nameMatch = contentLower.includes(p.name.toLowerCase());
-      const brandMatch = p.brand ? contentLower.includes(p.brand.toLowerCase()) : false;
-
-      if (nameMatch || brandMatch) {
-        seenIds.add(identifier);
-        uniqueProducts.push(p);
+    return productsFromSources.filter(p => {
+      const id = String(p.id ?? p.name);
+      if (seen.has(id)) return false;
+      const mentioned =
+        contentLower.includes(p.name.toLowerCase()) ||
+        (p.brand ? contentLower.includes(p.brand.toLowerCase()) : false);
+      if (mentioned) {
+        seen.add(id);
+        return true;
       }
-    }
-
-    return uniqueProducts;
+      return false;
+    });
   }, [productsFromSources, productsFromContent, message.content]);
 
-  const hasProducts = allProducts.length > 0;
+  // ── Markdown component overrides ───────────────────────────────────────────
+  const markdownComponents = React.useMemo(
+    () => ({
+      table: ({ ...props }: React.HTMLAttributes<HTMLTableElement>) => (
+        <div className="overflow-x-auto my-4 rounded-xl border border-slate-200 dark:border-white/10 shadow-sm">
+          <table className="w-full text-left border-collapse min-w-[400px]" {...props} />
+        </div>
+      ),
+      thead: ({ ...props }: React.HTMLAttributes<HTMLTableSectionElement>) => (
+        <thead
+          className="bg-slate-50 dark:bg-white/5 border-b border-slate-200 dark:border-white/10"
+          {...props}
+        />
+      ),
+      th: ({ children, ...props }: React.ThHTMLAttributes<HTMLTableCellElement>) => (
+        <th
+          className="p-3 font-semibold text-slate-700 dark:text-white/90 first:rounded-tl-xl last:rounded-tr-xl"
+          {...props}
+        >
+          {normaliseChild(children)}
+        </th>
+      ),
+      td: ({ children, ...props }: React.TdHTMLAttributes<HTMLTableCellElement>) => (
+        <td
+          className="p-3 border-b border-slate-100 dark:border-white/5 last:border-0 text-slate-600 dark:text-white/70"
+          {...props}
+        >
+          {normaliseChild(children)}
+        </td>
+      ),
+      code({
+        inline,
+        className,
+        children,
+        ...props
+      }: React.HTMLAttributes<HTMLElement> & { inline?: boolean }) {
+        const lang = /language-(\w+)/.exec(className ?? '')?.[1];
 
+        if (!inline && lang === 'chart') {
+          return (
+            <ChartBlock
+              rawContent={String(children ?? '').trim()}
+              primaryColor={primaryColor}
+              accentColor={accentColor}
+            />
+          );
+        }
+
+        return (
+          <code className={className} {...props}>
+            {typeof children === 'string' || typeof children === 'number'
+              ? children
+              : JSON.stringify(children)}
+          </code>
+        );
+      },
+    }),
+    [primaryColor, accentColor],
+  );
+
+  // ── Render ─────────────────────────────────────────────────────────────────
   return (
     <div className={`flex gap-3 ${isUser ? 'flex-row-reverse' : 'flex-row'} items-start`}>
       {/* Avatar */}
       <div
-        className={`flex-shrink-0 w-8 h-8 rounded-full flex items-center justify-center shadow-lg ${isUser ? 'text-white' : 'bg-slate-100 dark:bg-white/10 text-slate-700 dark:text-white/80'
+        className={`flex-shrink-0 w-8 h-8 rounded-full flex items-center justify-center shadow-lg ${isUser
+          ? 'text-white'
+          : 'bg-slate-100 dark:bg-white/10 text-slate-700 dark:text-white/80'
           }`}
         style={isUser ? { background: `linear-gradient(135deg, ${primaryColor}, ${accentColor})` } : {}}
       >
-        {isUser
-          ? <User className="w-4 h-4 text-white" />
-          : <Bot className="w-4 h-4" />
-        }
+        {isUser ? <User className="w-4 h-4 text-white" /> : <Bot className="w-4 h-4" />}
       </div>
 
       <div className={`flex flex-col gap-1 max-w-[90%] ${isUser ? 'items-end' : 'items-start'}`}>
         {/* Bubble */}
         <div
           className={`relative px-4 py-3 rounded-2xl text-sm leading-relaxed shadow-sm dark:shadow-lg ${isUser
-              ? 'text-white rounded-tr-sm'
-              : 'bg-white dark:bg-white/5 border border-slate-200 dark:border-white/10 text-slate-800 dark:text-white/90 rounded-tl-sm'
+            ? 'text-white rounded-tr-sm'
+            : 'bg-white dark:bg-white/5 border border-slate-200 dark:border-white/10 text-slate-800 dark:text-white/90 rounded-tl-sm'
             }`}
           style={
-            isUser
-              ? { background: `linear-gradient(135deg, ${primaryColor}, ${accentColor})` }
-              : {}
+            isUser ? { background: `linear-gradient(135deg, ${primaryColor}, ${accentColor})` } : {}
           }
         >
           {isStreaming && !message.content ? (
@@ -156,89 +312,7 @@ export function MessageBubble({
             </span>
           ) : (
             <div className={`prose prose-sm max-w-none ${isUser ? 'prose-invert' : 'dark:prose-invert'}`}>
-              <ReactMarkdown 
-                remarkPlugins={[remarkGfm]}
-                components={{
-                  table: ({...props}) => (
-                    <div className="overflow-x-auto my-4 rounded-xl border border-slate-200 dark:border-white/10 shadow-sm">
-                      <table className="w-full text-left border-collapse min-w-[400px]" {...props} />
-                    </div>
-                  ),
-                  thead: ({...props}) => (
-                    <thead className="bg-slate-50 dark:bg-white/5 border-b border-slate-200 dark:border-white/10" {...props} />
-                  ),
-                  th: ({children, ...props}) => (
-                    <th className="p-3 font-semibold text-slate-700 dark:text-white/90 first:rounded-tl-xl last:rounded-tr-xl" {...props}>
-                      {typeof children === 'object' && children !== null && !Array.isArray(children) && !(children as any).$$typeof ? JSON.stringify(children) : children}
-                    </th>
-                  ),
-                  td: ({children, ...props}) => (
-                    <td className="p-3 border-b border-slate-100 dark:border-white/5 last:border-0 text-slate-600 dark:text-white/70" {...props}>
-                      {typeof children === 'object' && children !== null && !Array.isArray(children) && !(children as any).$$typeof ? JSON.stringify(children) : children}
-                    </td>
-                  ),
-                  code({ inline, className, children, ...props }: React.HTMLAttributes<HTMLElement> & { inline?: boolean }) {
-                    const match = /language-(\w+)/.exec(className || '');
-                    const isChart = match && match[1] === 'chart';
-                    
-                    if (!inline && isChart) {
-                      const rawContent = String(children || '').trim();
-                      
-                      if (!rawContent || rawContent === 'undefined') {
-                        return null;
-                      }
-                      
-                      // Optimized sanitization helper to handle LLM quirks
-                      const sanitizeJson = (json: string) => {
-                        return json
-                          // 1. Resolve arithmetic: "value": 495 / 1000 => "value": 0.495
-                          .replace(/:\s*(\d+(\.\d+)?)\s*([\+\-\*\/])\s*(\d+(\.\d+)?)/g, (_, n1, __, op, n2) => {
-                            const v1 = parseFloat(n1); const v2 = parseFloat(n2);
-                            const ops: Record<string, number> = { '+': v1 + v2, '-': v1 - v2, '*': v1 * v2, '/': v1 / v2 };
-                            return `: ${ops[op] ?? v1}`;
-                          })
-                          // 2. Quote unquoted keys and fix single quotes
-                          .replace(/([{,])\s*([a-zA-Z_][a-zA-Z0-9_]*)\s*:/g, '$1 "$2":')
-                          .replace(/'([^'\\]*(?:\\.[^'\\]*)*)'/g, '"$1"')
-                          // 3. Syntax cleanup: trailing/missing commas and comments
-                          .replace(/,\s*([\]}])/g, '$1')
-                          .replace(/}\s*{/g, '},{')
-                          .replace(/]\s*\[/g, '],[')
-                          .replace(/\/\/.*$/gm, '')
-                          // 4. Remove dangling empty strings
-                          .replace(/([{,])\s*""\s*([,}])/g, '$1$2');
-                      };
-
-                      try {
-                        const sanitized = sanitizeJson(rawContent);
-                        const config = JSON.parse(sanitized) as ChartConfig;
-                        return (
-                          <div className="my-4 p-4 bg-white dark:bg-slate-900 rounded-xl border border-slate-200 dark:border-white/10 shadow-sm">
-                            <DynamicChart 
-                              config={config} 
-                              primaryColor={primaryColor} 
-                              accentColor={accentColor} 
-                            />
-                          </div>
-                        );
-                      } catch (err) {
-                        console.error('[MessageBubble] Chart parsing failed:', err, '\nSanitized content:', sanitizeJson(rawContent));
-                        return (
-                          <div className="p-4 bg-red-50 dark:bg-red-900/20 text-red-600 dark:text-red-400 rounded-lg text-sm border border-red-100 dark:border-red-900/30">
-                            <p className="font-medium mb-1">Failed to render chart</p>
-                            <p className="opacity-70 text-xs">The generated configuration is invalid. Check console for details.</p>
-                          </div>
-                        );
-                      }
-                    }
-                    return (
-                      <code className={className} {...props}>
-                        {typeof children === 'string' || typeof children === 'number' ? children : JSON.stringify(children)}
-                      </code>
-                    );
-                  }
-                }}
-              >
+              <ReactMarkdown remarkPlugins={[remarkGfm]} components={markdownComponents}>
                 {cleanContent || message.content}
               </ReactMarkdown>
 
@@ -249,8 +323,8 @@ export function MessageBubble({
           )}
         </div>
 
-        {/* Product Carousel (Outside the bubble) */}
-        {!isUser && hasProducts && (
+        {/* Product Carousel */}
+        {!isUser && allProducts.length > 0 && (
           <div className="w-full mt-1">
             <ProductCarousel
               products={allProducts}
@@ -260,17 +334,14 @@ export function MessageBubble({
           </div>
         )}
 
-        {/* Sources toggle */}
+        {/* Sources */}
         {!isUser && sources && sources.length > 0 && (
           <div className="w-full">
             <button
-              onClick={() => setShowSources((s) => !s)}
+              onClick={() => setShowSources(s => !s)}
               className="text-[11px] text-indigo-400 hover:text-indigo-300 transition-colors flex items-center gap-1 mt-1"
             >
-              {showSources
-                ? <ChevronDown className="w-3 h-3" />
-                : <ChevronRight className="w-3 h-3" />
-              }
+              {showSources ? <ChevronDown className="w-3 h-3" /> : <ChevronRight className="w-3 h-3" />}
               {sources.length} source{sources.length !== 1 ? 's' : ''} used
             </button>
 
@@ -287,3 +358,18 @@ export function MessageBubble({
     </div>
   );
 }
+
+// ─── Util ─────────────────────────────────────────────────────────────────────
+
+/** Safely render table cell children that might be plain objects. */
+function normaliseChild(children: React.ReactNode): React.ReactNode {
+  if (
+    typeof children === 'object' &&
+    children !== null &&
+    !Array.isArray(children) &&
+    !React.isValidElement(children)
+  ) {
+    return JSON.stringify(children);
+  }
+  return children;
+}

package/src/core/LangChainAgent.ts

@@ -57,7 +57,8 @@ export class LangChainAgent {
   1. Use ONLY valid JSON (double quotes, no trailing commas).
   2. NO math expressions (e.g., use 100, NOT 50+50).
   3. dataKeys MUST exactly match the property names in the data objects.
-  4. DO NOT output naked JSON; it MUST be wrapped in \`\`\`chart ... \`\`\`.
+  4. data objects MUST be FLAT (no nested objects or arrays).
+  5. DO NOT output naked JSON; it MUST be wrapped in \`\`\`chart ... \`\`\`.
 \`\`\`chart
 {
   "type": "bar" | "line" | "pie",

package/src/core/Pipeline.ts

@@ -107,7 +107,8 @@ export class Pipeline {
   1. Use ONLY valid JSON (double quotes, no trailing commas).
   2. NO math expressions (e.g., use 100, NOT 50+50).
   3. dataKeys MUST exactly match the property names in the data objects.
-  4. DO NOT output naked JSON; it MUST be wrapped in \`\`\`chart ... \`\`\`.
+  4. data objects MUST be FLAT (no nested objects or arrays).
+  5. DO NOT output naked JSON; it MUST be wrapped in \`\`\`chart ... \`\`\`.
 \`\`\`chart
 {
   "type": "bar" | "line" | "pie",