@retrivora-ai/rag-engine 1.8.1 → 1.8.2

package/src/utils/ProductExtractor.ts

@@ -0,0 +1,347 @@
+import { Product, VectorMatch } from '../types';
+import { resolveMetadataValue } from './synonyms';
+
+// ─── JSON sanitization ────────────────────────────────────────────────────────
+
+export function sanitizeJson(raw: string): string {
+  let s = raw.trim();
+
+  // 1. Evaluate simple arithmetic expressions in values  e.g.  "v": 495 / 1000
+  s = s.replace(
+    /:\s*(-?\d+(?:\.\d+)?)\s*([+\-*/])\s*(-?\d+(?:\.\d+)?)/g,
+    (_, a, op, b) => {
+      const n1 = parseFloat(a);
+      const n2 = parseFloat(b);
+      const result =
+        op === '+' ? n1 + n2
+          : op === '-' ? n1 - n2
+            : op === '*' ? n1 * n2
+              : n2 !== 0 ? n1 / n2
+                : n1;
+      return `: ${result}`;
+    },
+  );
+
+  // 2. Strip JS-style comments while preserving strings (e.g. URLs)
+  s = s.replace(/("(?:\\.|[^\\"])*")|\/\/[^\n]*|\/\*[\s\S]*?\*\//g, (m, g1) => g1 || '');
+
+  // 3. Quote bare (unquoted) object keys
+  s = s.replace(/([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)\s*:/g, '$1"$2":');
+
+  // 4. Replace single-quoted strings with double-quoted ones
+  s = s.replace(/'([^'\\]*(?:\\.[^'\\]*)*)'/g, '"$1"');
+
+  // 5. Remove trailing commas before ] or }
+  s = s.replace(/,\s*([}\]])/g, '$1');
+
+  // 6. Remove completely empty key-value pairs left by prior cleanup
+  s = s.replace(/([{,])\s*""\s*(?=[,}])/g, '$1');
+
+  // 7. Stack-based structure balancer.
+  const stack: ('{' | '[')[] = [];
+  let inString = false;
+  let escape = false;
+
+  for (const ch of s) {
+    if (escape) { escape = false; continue; }
+    if (ch === '\\' && inString) { escape = true; continue; }
+    if (ch === '"') { inString = !inString; continue; }
+    if (inString) continue;
+
+    if (ch === '{' || ch === '[') {
+      stack.push(ch);
+    } else if (ch === '}') {
+      if (stack[stack.length - 1] === '{') stack.pop();
+    } else if (ch === ']') {
+      if (stack[stack.length - 1] === '[') stack.pop();
+    }
+  }
+
+  if (inString) s += '"';
+  for (let i = stack.length - 1; i >= 0; i--) {
+    s += stack[i] === '{' ? '}' : ']';
+  }
+
+  const lastClose = Math.max(s.lastIndexOf('}'), s.lastIndexOf(']'));
+  if (lastClose !== -1 && lastClose < s.length - 1 && /\S/.test(s.slice(lastClose + 1))) {
+    s = s.substring(0, lastClose + 1);
+  }
+
+  return s;
+}
+
+export function extractLikelyJsonBlock(raw: string): string {
+  const trimmed = raw.trim();
+  const objectStart = trimmed.indexOf('{');
+  const arrayStart = trimmed.indexOf('[');
+
+  const starts = [objectStart, arrayStart].filter((index) => index >= 0);
+  if (starts.length === 0) return trimmed;
+
+  return trimmed.slice(Math.min(...starts));
+}
+
+export function looksLikeStructuredPayload(raw: string): boolean {
+  return /"?(view|chartType|type|data|items|rows|products|results)"?\s*:/.test(raw);
+}
+
+export function stripMarkdownTables(raw: string): string {
+  return raw.replace(
+    /(?:^|\n)\|[^\n]+\|\n\|(?:\s*:?-+:?\s*\|)+\n(?:\|[^\n]*\|\n?)*/g,
+    '\n\n',
+  );
+}
+
+export function stripStructuredUiLabels(raw: string): string {
+  return raw.replace(
+    /(?:^|\n)\*\*(?:Chart Data|Table Data|Visualization|Visual Representation)\*\:\s*(?=\n|$)/gi,
+    '\n',
+  ).replace(
+    /(?:^|\n)(?:Chart Data|Table Data|Visualization|Visual Representation)\s*\:\s*(?=\n|$)/gi,
+    '\n',
+  );
+}
+
+function extractBareJsonObject(raw: string): string | null {
+  let start = raw.indexOf('{');
+  if (start === -1) return null;
+
+  let bestJson: string | null = null;
+  let maxLen = 0;
+
+  while (start !== -1) {
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+
+    for (let i = start; i < raw.length; i += 1) {
+      const ch = raw[i];
+      if (escape) { escape = false; continue; }
+      if (ch === '\\' && inString) { escape = true; continue; }
+      if (ch === '"') { inString = !inString; continue; }
+      if (inString) continue;
+
+      if (ch === '{') depth += 1;
+      else if (ch === '}') {
+        depth -= 1;
+        if (depth === 0) {
+          const candidate = raw.slice(start, i + 1);
+          if (candidate.length > maxLen) {
+            maxLen = candidate.length;
+            bestJson = candidate;
+          }
+          break;
+        }
+      }
+    }
+    start = raw.indexOf('{', start + 1);
+  }
+
+  return bestJson;
+}
+
+export function extractStructuredPayload(raw: string): { payload: string | null; text: string } {
+  let working = raw;
+  const fenceRegex = /```(?:[a-z]+)?\s*([\s\S]*?)```/gi;
+
+  for (const match of Array.from(raw.matchAll(fenceRegex))) {
+    const body = match[1]?.trim() ?? '';
+    if (!looksLikeStructuredPayload(body)) continue;
+
+    working = working.replace(match[0], '');
+    return {
+      payload: body,
+      text: stripStructuredUiLabels(working).replace(/\n{3,}/g, '\n\n').trim(),
+    };
+  }
+
+  const bareJson = extractBareJsonObject(raw);
+  if (bareJson && looksLikeStructuredPayload(bareJson)) {
+    working = working.replace(bareJson, '');
+    return {
+      payload: bareJson,
+      text: stripStructuredUiLabels(working).replace(/\n{3,}/g, '\n\n').trim(),
+    };
+  }
+
+  return {
+    payload: null,
+    text: stripStructuredUiLabels(raw).replace(/\n{3,}/g, '\n\n').trim(),
+  };
+}
+
+export function resolveImage(data: Record<string, unknown>): string | undefined {
+  const isPlaceholder = (val: unknown) =>
+    typeof val === 'string' && (val === '...' || val === '' || val.toLowerCase() === 'null' || val.toLowerCase() === 'undefined');
+
+  for (const key of ['image', 'img', 'thumbnail', 'images', 'product_image', 'Image', 'Thumbnail']) {
+    const v = data[key];
+    if (typeof v === 'string' && v && !isPlaceholder(v)) return v;
+    if (Array.isArray(v) && typeof v[0] === 'string' && v[0] && !isPlaceholder(v[0])) return v[0];
+  }
+  return undefined;
+}
+
+export function isLikelyUiOnlyMessage(text: string): boolean {
+  const trimmed = text.trim();
+  return (
+    trimmed.length === 0 ||
+    trimmed.length < 30 ||
+    /^(chart data|table data|visualization|visual representation|product catalog|product recommendations|catalog summary|availability|inventory status|details)\s*:?\s*$/i.test(trimmed)
+  );
+}
+
+// ─── Extractors ───────────────────────────────────────────────────────────────
+
+export function extractProductsFromSources(sources: VectorMatch[] | undefined, isUser: boolean): Product[] {
+  if (isUser || !sources) return [];
+  return sources
+    .filter(s => {
+      const m = s.metadata ?? {};
+      const keys = Object.keys(m).map(k => k.toLowerCase());
+      const hasProductKey = keys.some(k =>
+        ['price', 'image', 'img', 'thumbnail', 'images', 'brand', 'product', 'sku', 'category', 'model', 'cost'].includes(k)
+      );
+      const hasPricePattern = /\$\s*\d+/.test(s.content);
+      return hasProductKey || hasPricePattern || m.type === 'product';
+    })
+    .map(s => {
+      const m = (s.metadata ?? {}) as Record<string, unknown>;
+      const name = resolveMetadataValue(m, 'name') as string;
+      const brand = resolveMetadataValue(m, 'brand') as string;
+      const price = resolveMetadataValue(m, 'price') as string | number;
+      const description = resolveMetadataValue(m, 'description') as string;
+      console.log("🚀 ~ extractProductsFromSources ~ description:", s)
+
+      return {
+        id: s.id,
+        name: name ?? s.content.split('\n')[0] ?? 'Unknown Product',
+        brand: brand,
+        price: price,
+        image: resolveImage(m),
+        link: resolveMetadataValue(m, 'link') as string,
+        description: description ? description : '' //description ?? s.content,
+      };
+    });
+}
+
+export function extractProductsFromContent(
+  content: string,
+  payloadText: string,
+  payloadCandidates: string[],
+  productsFromSources: Product[]
+): { productsFromContent: Product[]; cleanContent: string } {
+  const jsonRegex = /```(?:json|ui)?\s*([\s\S]*?)\s*```/g;
+  const products: Product[] = [];
+  let cleanContent = payloadText;
+
+  for (const candidate of payloadCandidates) {
+    try {
+      const data = JSON.parse(sanitizeJson(candidate));
+      const itemSet =
+        Array.isArray(data.data) ? data.data
+          : Array.isArray(data.items) ? data.items
+            : Array.isArray(data.products) ? data.products
+              : null;
+
+      if (
+        (data.type === 'products' || data.view === 'carousel' || data.view === 'table' || Array.isArray(data.products)) &&
+        itemSet
+      ) {
+        products.push(
+          ...itemSet.map((item: Record<string, unknown>) => ({
+            ...item,
+            image: (item.image as string) ?? resolveImage(item),
+          })) as Product[],
+        );
+      }
+    } catch {
+      // Ignore malformed product payloads
+    }
+  }
+
+  if (cleanContent.includes('"type": "products"') || cleanContent.includes('"type":"products"')) {
+    for (const match of Array.from(cleanContent.matchAll(jsonRegex))) {
+      try {
+        const data = JSON.parse(sanitizeJson(match[1]));
+        if (data.type === 'products' && Array.isArray(data.items)) {
+          products.push(
+            ...data.items.map((item: Record<string, unknown>) => ({
+              ...item,
+              image: (item.image as string) ?? resolveImage(item),
+            })) as Product[],
+          );
+          cleanContent = cleanContent.replace(match[0], '');
+        }
+      } catch {
+        // Malformed product block — skip silently
+      }
+    }
+  }
+
+  // Fallback: Scan for plain text bulleted products
+  // Requires a bullet or number at the start of the line to prevent matching plain text paragraphs.
+  const bulletRegex = /^[ \t]*(?:[•*-]|\d+\.)\s+(?:\*\*)?([^\n\-\$–—\(]+?)(?:\*\*)?(?:\s*\(?Price\s*[:\-–—]?\s*)?(?:\s*[:\-–—]\s*|\s+)\$?([\d,.]+)(?:\s*USD)?/gim;
+  const matches = Array.from(cleanContent.matchAll(bulletRegex));
+
+  if (matches.length >= 2) {
+    const normalize = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, '');
+
+    for (const match of matches) {
+      let name = match[1]?.trim() || '';
+      name = name.replace(/\s*\(?Price\s*$/i, '').replace(/[:\-–—]\s*$/g, '').trim();
+      name = name.replace(/^\*\*|\*\*$/g, '').replace(/^\*|\*$/g, '').trim();
+
+      let price = match[2]?.trim() || '';
+      price = price.replace(/[,\s]+$/, '').trim();
+
+      if (name && price) {
+        const newProduct: Product = {
+          id: `text-prod-${name}-${price}`,
+          name,
+          price: `$${price}`,
+        };
+
+        const normName = normalize(name);
+        const sourceMatch = productsFromSources.find(s => {
+          const sn = normalize(s.name);
+          return sn.includes(normName) || normName.includes(sn);
+        });
+
+        if (sourceMatch) {
+          newProduct.image = sourceMatch.image;
+          newProduct.brand = sourceMatch.brand;
+          newProduct.link = sourceMatch.link;
+          newProduct.description = sourceMatch.description;
+        }
+
+        products.push(newProduct);
+        cleanContent = cleanContent.replace(match[0], `\n**${name}** — $${price}`);
+      }
+    }
+    cleanContent = cleanContent.replace(/\n{3,}/g, '\n\n').trim();
+  }
+
+  return { productsFromContent: products, cleanContent: cleanContent.trim() };
+}
+
+export function deduplicateProducts(
+  productsFromSources: Product[],
+  productsFromContent: Product[],
+  messageContent: string
+): Product[] {
+  if (productsFromContent.length > 0) return productsFromContent;
+
+  const seen = new Set<string>();
+  const contentLower = messageContent.toLowerCase();
+
+  return productsFromSources.filter(p => {
+    const id = String(p.id ?? p.name);
+    if (seen.has(id)) return false;
+    const mentioned =
+      contentLower.includes(p.name.toLowerCase()) ||
+      (p.brand ? contentLower.includes(p.brand.toLowerCase()) : false);
+    if (mentioned) { seen.add(id); return true; }
+    return false;
+  });
+}

package/src/utils/UITransformer.ts

@@ -1,5 +1,6 @@
-import { VectorMatch, UITransformationResponse, PieChartData, BarChartData, LineChartDataPoint, TableData, CarouselProduct, VisualizationType } from '../types';
+import { VectorMatch, UITransformationResponse, PieChartData, LineChartDataPoint, TableData, CarouselProduct, VisualizationType } from '../types';
 import { ILLMProvider } from '../llm/ILLMProvider';
+import { resolveMetadataValue } from './synonyms';
 
 /**
  * UITransformer — Converts vector database results into structured UI representations
@@ -8,18 +9,6 @@ import { ILLMProvider } from '../llm/ILLMProvider';
  * format based on the content and structure of the data.
  */
 export class UITransformer {
-  /**
-   * Central dictionary of common synonyms for UI properties.
-   * This allows the system to be schema-agnostic by guessing field names.
-   */
-  private static readonly SYNONYMS: Record<string, string[]> = {
-    name: ['product', 'item', 'title', 'label', 'heading', 'subject'],
-    price: ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd'],
-    brand: ['manufacturer', 'vendor', 'make', 'company', 'brand_name', 'supplier'],
-    image: ['imageUrl', 'thumbnail', 'img', 'url', 'photo', 'picture', 'media', 'image_url', 'main_image', 'product_image', 'thumb'],
-    stock: ['inventory', 'quantity', 'count', 'availability', 'stock_level', 'inStock', 'is_available'],
-    description: ['summary', 'content', 'body', 'text', 'info', 'details'],
-  };
 
   /**
    * Main transformation method
@@ -121,33 +110,6 @@ export class UITransformer {
     };
   }
 
-  /**
-   * Transform data to bar chart format
-   */
-  private static transformToBarChart(
-    data: VectorMatch[]
-  ): UITransformationResponse {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
-
-    const barData: BarChartData[] = Object.entries(categoryData).map(([category, value]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(category, data);
-      return {
-        category,
-        value: value as number,
-        inStockCount,
-        outOfStockCount,
-      };
-    });
-
-    return {
-      type: 'bar_chart',
-      title: 'Comparison by Category',
-      description: `Comparing ${categories.length} categories`,
-      data: barData,
-    };
-  }
-
   /**
    * Transform data to line chart format
    */
@@ -488,18 +450,7 @@ export class UITransformer {
       if (trainedKey && meta[trainedKey as string] !== undefined) return meta[trainedKey as string];
     }
 
-    const synonyms = this.SYNONYMS[uiKey] || [];
-    const searchKeys = [uiKey, ...synonyms].map((k) => k.toLowerCase());
-
-    const foundDirectKey = Object.keys(meta).find((k) => searchKeys.includes(k.toLowerCase()));
-    if (foundDirectKey) return meta[foundDirectKey];
-
-    const foundPartialKey = Object.keys(meta).find((k) => {
-      const keyLow = k.toLowerCase();
-      return searchKeys.some((sk) => keyLow.includes(sk) || sk.includes(keyLow));
-    });
-
-    return foundPartialKey ? meta[foundPartialKey] : undefined;
+    return resolveMetadataValue(meta, uiKey);
   }
 
   private static extractProductInfo(
@@ -605,7 +556,7 @@ export class UITransformer {
       const meta = item.metadata || {};
       const itemCategory = (meta.category || meta.type || 'Other') as string;
 
-      if (result.hasOwnProperty(itemCategory)) {
+      if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
       } else {
         result['Other'] = (result['Other'] || 0) + 1;

package/src/utils/synonyms.ts

@@ -0,0 +1,78 @@
+/**
+ * Shared synonym map used by UITransformer and MessageBubble to resolve
+ * schema-agnostic field names from vector metadata.
+ */
+export const FIELD_SYNONYMS: Record<string, string[]> = {
+  name:        ['product', 'item', 'title', 'label', 'heading', 'subject', 'product name', 'item name'],
+  price:       ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd'],
+  brand:       ['manufacturer', 'vendor', 'make', 'company', 'brand_name', 'supplier'],
+  image:       ['imageUrl', 'thumbnail', 'img', 'url', 'photo', 'picture', 'media', 'image_url',
+                'main_image', 'product_image', 'thumb'],
+  stock:       ['inventory', 'quantity', 'count', 'availability', 'stock_level', 'inStock',
+                'is_available', 'in stock', 'status'],
+  description: ['summary', 'content', 'body', 'text', 'info', 'details'],
+  link:        ['url', 'href', 'product_url', 'page_url', 'link'],
+};
+
+/**
+ * Dynamically registers new synonyms into the global FIELD_SYNONYMS dictionary.
+ * You can call this during your app initialization to support custom schema fields.
+ */
+export function addSynonyms(customSynonyms: Record<string, string[]>) {
+  for (const [key, aliases] of Object.entries(customSynonyms)) {
+    if (!FIELD_SYNONYMS[key]) {
+      FIELD_SYNONYMS[key] = [];
+    }
+    const existing = new Set(FIELD_SYNONYMS[key].map(s => s.toLowerCase()));
+    
+    for (const alias of aliases) {
+      if (!existing.has(alias.toLowerCase())) {
+        FIELD_SYNONYMS[key].push(alias);
+        existing.add(alias.toLowerCase());
+      }
+    }
+  }
+}
+
+/**
+ * Robustly extract a value from metadata by checking exact, case-insensitive,
+ * and synonym-based key matches.
+ */
+export function resolveMetadataValue(
+  meta: Record<string, unknown>,
+  uiKey: string,
+): unknown {
+  const synonyms = FIELD_SYNONYMS[uiKey] ?? [];
+
+  const keys = Object.keys(meta);
+  const systemKeys = ['namespace', 'filename', 'filesize', 'filetype', 'docid', 'uploadedat', 'dimension', 'chunkindex'];
+
+  // 1. Exact case-insensitive match for primary key
+  let match = keys.find((k) => k.toLowerCase() === uiKey.toLowerCase());
+  if (match !== undefined) return meta[match];
+
+  // 2. Exact case-insensitive match for synonyms
+  match = keys.find((k) => synonyms.map(s => s.toLowerCase()).includes(k.toLowerCase()));
+  if (match !== undefined) return meta[match];
+
+  const isBlacklisted = (kl: string) => {
+    return systemKeys.includes(kl) || kl.startsWith('option1') || kl.startsWith('option2') || kl.startsWith('option3');
+  };
+
+  // 3. Partial match (substring) strictly for primary key
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return kl.includes(uiKey.toLowerCase());
+  });
+  if (match !== undefined) return meta[match];
+
+  // 4. Partial match (substring) for synonyms
+  match = keys.find((k) => {
+    const kl = k.toLowerCase();
+    if (isBlacklisted(kl)) return false;
+    return synonyms.some((sk) => kl.includes(sk.toLowerCase()) || sk.toLowerCase().includes(kl));
+  });
+  
+  return match !== undefined ? meta[match] : undefined;
+}