@retrivora-ai/rag-engine 1.8.0 → 1.8.2

package/src/utils/ProductExtractor.ts

@@ -0,0 +1,347 @@
+import { Product, VectorMatch } from '../types';
+import { resolveMetadataValue } from './synonyms';
+
+// ─── JSON sanitization ────────────────────────────────────────────────────────
+
+export function sanitizeJson(raw: string): string {
+  let s = raw.trim();
+
+  // 1. Evaluate simple arithmetic expressions in values  e.g.  "v": 495 / 1000
+  s = s.replace(
+    /:\s*(-?\d+(?:\.\d+)?)\s*([+\-*/])\s*(-?\d+(?:\.\d+)?)/g,
+    (_, a, op, b) => {
+      const n1 = parseFloat(a);
+      const n2 = parseFloat(b);
+      const result =
+        op === '+' ? n1 + n2
+          : op === '-' ? n1 - n2
+            : op === '*' ? n1 * n2
+              : n2 !== 0 ? n1 / n2
+                : n1;
+      return `: ${result}`;
+    },
+  );
+
+  // 2. Strip JS-style comments while preserving strings (e.g. URLs)
+  s = s.replace(/("(?:\\.|[^\\"])*")|\/\/[^\n]*|\/\*[\s\S]*?\*\//g, (m, g1) => g1 || '');
+
+  // 3. Quote bare (unquoted) object keys
+  s = s.replace(/([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)\s*:/g, '$1"$2":');
+
+  // 4. Replace single-quoted strings with double-quoted ones
+  s = s.replace(/'([^'\\]*(?:\\.[^'\\]*)*)'/g, '"$1"');
+
+  // 5. Remove trailing commas before ] or }
+  s = s.replace(/,\s*([}\]])/g, '$1');
+
+  // 6. Remove completely empty key-value pairs left by prior cleanup
+  s = s.replace(/([{,])\s*""\s*(?=[,}])/g, '$1');
+
+  // 7. Stack-based structure balancer.
+  const stack: ('{' | '[')[] = [];
+  let inString = false;
+  let escape = false;
+
+  for (const ch of s) {
+    if (escape) { escape = false; continue; }
+    if (ch === '\\' && inString) { escape = true; continue; }
+    if (ch === '"') { inString = !inString; continue; }
+    if (inString) continue;
+
+    if (ch === '{' || ch === '[') {
+      stack.push(ch);
+    } else if (ch === '}') {
+      if (stack[stack.length - 1] === '{') stack.pop();
+    } else if (ch === ']') {
+      if (stack[stack.length - 1] === '[') stack.pop();
+    }
+  }
+
+  if (inString) s += '"';
+  for (let i = stack.length - 1; i >= 0; i--) {
+    s += stack[i] === '{' ? '}' : ']';
+  }
+
+  const lastClose = Math.max(s.lastIndexOf('}'), s.lastIndexOf(']'));
+  if (lastClose !== -1 && lastClose < s.length - 1 && /\S/.test(s.slice(lastClose + 1))) {
+    s = s.substring(0, lastClose + 1);
+  }
+
+  return s;
+}
+
+export function extractLikelyJsonBlock(raw: string): string {
+  const trimmed = raw.trim();
+  const objectStart = trimmed.indexOf('{');
+  const arrayStart = trimmed.indexOf('[');
+
+  const starts = [objectStart, arrayStart].filter((index) => index >= 0);
+  if (starts.length === 0) return trimmed;
+
+  return trimmed.slice(Math.min(...starts));
+}
+
+export function looksLikeStructuredPayload(raw: string): boolean {
+  return /"?(view|chartType|type|data|items|rows|products|results)"?\s*:/.test(raw);
+}
+
+export function stripMarkdownTables(raw: string): string {
+  return raw.replace(
+    /(?:^|\n)\|[^\n]+\|\n\|(?:\s*:?-+:?\s*\|)+\n(?:\|[^\n]*\|\n?)*/g,
+    '\n\n',
+  );
+}
+
+export function stripStructuredUiLabels(raw: string): string {
+  return raw.replace(
+    /(?:^|\n)\*\*(?:Chart Data|Table Data|Visualization|Visual Representation)\*\:\s*(?=\n|$)/gi,
+    '\n',
+  ).replace(
+    /(?:^|\n)(?:Chart Data|Table Data|Visualization|Visual Representation)\s*\:\s*(?=\n|$)/gi,
+    '\n',
+  );
+}
+
+function extractBareJsonObject(raw: string): string | null {
+  let start = raw.indexOf('{');
+  if (start === -1) return null;
+
+  let bestJson: string | null = null;
+  let maxLen = 0;
+
+  while (start !== -1) {
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+
+    for (let i = start; i < raw.length; i += 1) {
+      const ch = raw[i];
+      if (escape) { escape = false; continue; }
+      if (ch === '\\' && inString) { escape = true; continue; }
+      if (ch === '"') { inString = !inString; continue; }
+      if (inString) continue;
+
+      if (ch === '{') depth += 1;
+      else if (ch === '}') {
+        depth -= 1;
+        if (depth === 0) {
+          const candidate = raw.slice(start, i + 1);
+          if (candidate.length > maxLen) {
+            maxLen = candidate.length;
+            bestJson = candidate;
+          }
+          break;
+        }
+      }
+    }
+    start = raw.indexOf('{', start + 1);
+  }
+
+  return bestJson;
+}
+
+export function extractStructuredPayload(raw: string): { payload: string | null; text: string } {
+  let working = raw;
+  const fenceRegex = /```(?:[a-z]+)?\s*([\s\S]*?)```/gi;
+
+  for (const match of Array.from(raw.matchAll(fenceRegex))) {
+    const body = match[1]?.trim() ?? '';
+    if (!looksLikeStructuredPayload(body)) continue;
+
+    working = working.replace(match[0], '');
+    return {
+      payload: body,
+      text: stripStructuredUiLabels(working).replace(/\n{3,}/g, '\n\n').trim(),
+    };
+  }
+
+  const bareJson = extractBareJsonObject(raw);
+  if (bareJson && looksLikeStructuredPayload(bareJson)) {
+    working = working.replace(bareJson, '');
+    return {
+      payload: bareJson,
+      text: stripStructuredUiLabels(working).replace(/\n{3,}/g, '\n\n').trim(),
+    };
+  }
+
+  return {
+    payload: null,
+    text: stripStructuredUiLabels(raw).replace(/\n{3,}/g, '\n\n').trim(),
+  };
+}
+
+export function resolveImage(data: Record<string, unknown>): string | undefined {
+  const isPlaceholder = (val: unknown) =>
+    typeof val === 'string' && (val === '...' || val === '' || val.toLowerCase() === 'null' || val.toLowerCase() === 'undefined');
+
+  for (const key of ['image', 'img', 'thumbnail', 'images', 'product_image', 'Image', 'Thumbnail']) {
+    const v = data[key];
+    if (typeof v === 'string' && v && !isPlaceholder(v)) return v;
+    if (Array.isArray(v) && typeof v[0] === 'string' && v[0] && !isPlaceholder(v[0])) return v[0];
+  }
+  return undefined;
+}
+
+export function isLikelyUiOnlyMessage(text: string): boolean {
+  const trimmed = text.trim();
+  return (
+    trimmed.length === 0 ||
+    trimmed.length < 30 ||
+    /^(chart data|table data|visualization|visual representation|product catalog|product recommendations|catalog summary|availability|inventory status|details)\s*:?\s*$/i.test(trimmed)
+  );
+}
+
+// ─── Extractors ───────────────────────────────────────────────────────────────
+
+export function extractProductsFromSources(sources: VectorMatch[] | undefined, isUser: boolean): Product[] {
+  if (isUser || !sources) return [];
+  return sources
+    .filter(s => {
+      const m = s.metadata ?? {};
+      const keys = Object.keys(m).map(k => k.toLowerCase());
+      const hasProductKey = keys.some(k =>
+        ['price', 'image', 'img', 'thumbnail', 'images', 'brand', 'product', 'sku', 'category', 'model', 'cost'].includes(k)
+      );
+      const hasPricePattern = /\$\s*\d+/.test(s.content);
+      return hasProductKey || hasPricePattern || m.type === 'product';
+    })
+    .map(s => {
+      const m = (s.metadata ?? {}) as Record<string, unknown>;
+      const name = resolveMetadataValue(m, 'name') as string;
+      const brand = resolveMetadataValue(m, 'brand') as string;
+      const price = resolveMetadataValue(m, 'price') as string | number;
+      const description = resolveMetadataValue(m, 'description') as string;
+      console.log("🚀 ~ extractProductsFromSources ~ description:", s)
+
+      return {
+        id: s.id,
+        name: name ?? s.content.split('\n')[0] ?? 'Unknown Product',
+        brand: brand,
+        price: price,
+        image: resolveImage(m),
+        link: resolveMetadataValue(m, 'link') as string,
+        description: description ? description : '' //description ?? s.content,
+      };
+    });
+}
+
+export function extractProductsFromContent(
+  content: string,
+  payloadText: string,
+  payloadCandidates: string[],
+  productsFromSources: Product[]
+): { productsFromContent: Product[]; cleanContent: string } {
+  const jsonRegex = /```(?:json|ui)?\s*([\s\S]*?)\s*```/g;
+  const products: Product[] = [];
+  let cleanContent = payloadText;
+
+  for (const candidate of payloadCandidates) {
+    try {
+      const data = JSON.parse(sanitizeJson(candidate));
+      const itemSet =
+        Array.isArray(data.data) ? data.data
+          : Array.isArray(data.items) ? data.items
+            : Array.isArray(data.products) ? data.products
+              : null;
+
+      if (
+        (data.type === 'products' || data.view === 'carousel' || data.view === 'table' || Array.isArray(data.products)) &&
+        itemSet
+      ) {
+        products.push(
+          ...itemSet.map((item: Record<string, unknown>) => ({
+            ...item,
+            image: (item.image as string) ?? resolveImage(item),
+          })) as Product[],
+        );
+      }
+    } catch {
+      // Ignore malformed product payloads
+    }
+  }
+
+  if (cleanContent.includes('"type": "products"') || cleanContent.includes('"type":"products"')) {
+    for (const match of Array.from(cleanContent.matchAll(jsonRegex))) {
+      try {
+        const data = JSON.parse(sanitizeJson(match[1]));
+        if (data.type === 'products' && Array.isArray(data.items)) {
+          products.push(
+            ...data.items.map((item: Record<string, unknown>) => ({
+              ...item,
+              image: (item.image as string) ?? resolveImage(item),
+            })) as Product[],
+          );
+          cleanContent = cleanContent.replace(match[0], '');
+        }
+      } catch {
+        // Malformed product block — skip silently
+      }
+    }
+  }
+
+  // Fallback: Scan for plain text bulleted products
+  // Requires a bullet or number at the start of the line to prevent matching plain text paragraphs.
+  const bulletRegex = /^[ \t]*(?:[•*-]|\d+\.)\s+(?:\*\*)?([^\n\-\$–—\(]+?)(?:\*\*)?(?:\s*\(?Price\s*[:\-–—]?\s*)?(?:\s*[:\-–—]\s*|\s+)\$?([\d,.]+)(?:\s*USD)?/gim;
+  const matches = Array.from(cleanContent.matchAll(bulletRegex));
+
+  if (matches.length >= 2) {
+    const normalize = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, '');
+
+    for (const match of matches) {
+      let name = match[1]?.trim() || '';
+      name = name.replace(/\s*\(?Price\s*$/i, '').replace(/[:\-–—]\s*$/g, '').trim();
+      name = name.replace(/^\*\*|\*\*$/g, '').replace(/^\*|\*$/g, '').trim();
+
+      let price = match[2]?.trim() || '';
+      price = price.replace(/[,\s]+$/, '').trim();
+
+      if (name && price) {
+        const newProduct: Product = {
+          id: `text-prod-${name}-${price}`,
+          name,
+          price: `$${price}`,
+        };
+
+        const normName = normalize(name);
+        const sourceMatch = productsFromSources.find(s => {
+          const sn = normalize(s.name);
+          return sn.includes(normName) || normName.includes(sn);
+        });
+
+        if (sourceMatch) {
+          newProduct.image = sourceMatch.image;
+          newProduct.brand = sourceMatch.brand;
+          newProduct.link = sourceMatch.link;
+          newProduct.description = sourceMatch.description;
+        }
+
+        products.push(newProduct);
+        cleanContent = cleanContent.replace(match[0], `\n**${name}** — $${price}`);
+      }
+    }
+    cleanContent = cleanContent.replace(/\n{3,}/g, '\n\n').trim();
+  }
+
+  return { productsFromContent: products, cleanContent: cleanContent.trim() };
+}
+
+export function deduplicateProducts(
+  productsFromSources: Product[],
+  productsFromContent: Product[],
+  messageContent: string
+): Product[] {
+  if (productsFromContent.length > 0) return productsFromContent;
+
+  const seen = new Set<string>();
+  const contentLower = messageContent.toLowerCase();
+
+  return productsFromSources.filter(p => {
+    const id = String(p.id ?? p.name);
+    if (seen.has(id)) return false;
+    const mentioned =
+      contentLower.includes(p.name.toLowerCase()) ||
+      (p.brand ? contentLower.includes(p.brand.toLowerCase()) : false);
+    if (mentioned) { seen.add(id); return true; }
+    return false;
+  });
+}

package/src/utils/SchemaMapper.ts

@@ -0,0 +1,129 @@
+import { ILLMProvider } from '../llm/ILLMProvider';
+
+export interface SchemaMap {
+  name?: string;
+  price?: string;
+  brand?: string;
+  image?: string;
+  stock?: string;
+  description?: string;
+  category?: string;
+}
+
+/**
+ * SchemaMapper — Uses AI to "train" the plugin on unknown database schemas.
+ * It maps arbitrary metadata keys to standard UI properties.
+ */
+export class SchemaMapper {
+  private static cache = new Map<string, SchemaMap>();
+
+  /**
+   * Descriptions of standard UI properties to help the AI map fields accurately.
+   */
+  private static readonly TARGET_PROPERTIES: Record<string, string> = {
+    name: 'The primary title, name, or label of the item',
+    price: 'The numeric cost, price, MSRP, or amount',
+    brand: 'The manufacturer, vendor, brand, or maker',
+    image: 'The URL to an image, thumbnail, photo, or picture',
+    stock: 'The availability, inventory count, quantity, or stock level',
+    description: 'The detailed text content, summary, or body info',
+    category: 'The group, department, type, or category name',
+  };
+
+  /**
+   * Trains the plugin on a set of keys.
+   * This is done once per schema and cached.
+   */
+  static async train(
+    llm: ILLMProvider,
+    projectId: string,
+    keys: string[]
+  ): Promise<SchemaMap> {
+    const cacheKey = `${projectId}:${keys.sort().join(',')}`;
+    if (this.cache.has(cacheKey)) {
+      return this.cache.get(cacheKey)!;
+    }
+
+    console.log(`[SchemaMapper] 🧠 Training AI on new schema keys: ${keys.join(', ')}`);
+
+    // Dynamically build the property list for the prompt
+    const propertyList = Object.entries(this.TARGET_PROPERTIES)
+      .map(([prop, desc]) => `- ${prop} (${desc})`)
+      .join('\n');
+
+    const prompt = `
+Given these metadata keys from a database: [${keys.join(', ')}]
+
+Identify which keys best correspond to these standard UI properties:
+${propertyList}
+
+Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys. 
+If no good match is found for a property, omit it.
+
+Example:
+{
+  "name": "Product_Title",
+  "price": "MSRP_USD",
+  "brand": "VendorName"
+}
+`;
+
+    try {
+      const response = await llm.chat(
+        [
+          { role: 'system', content: 'You are a database schema expert. You ONLY respond with valid JSON.' },
+          { role: 'user', content: prompt }
+        ],
+        ''
+      );
+
+      // Extract the first balanced JSON object from the response
+      const startIdx = response.indexOf('{');
+      if (startIdx !== -1) {
+        let braceCount = 0;
+        let endIdx = -1;
+
+        for (let i = startIdx; i < response.length; i++) {
+          if (response[i] === '{') braceCount++;
+          else if (response[i] === '}') {
+            braceCount--;
+            if (braceCount === 0) {
+              endIdx = i;
+              break;
+            }
+          }
+        }
+
+        if (endIdx !== -1) {
+          const jsonContent = response.substring(startIdx, endIdx + 1);
+          const cleanJson = this.sanitizeJson(jsonContent);
+          const mapping = JSON.parse(cleanJson) as SchemaMap;
+          this.cache.set(cacheKey, mapping);
+          return mapping;
+        }
+      }
+    } catch (error) {
+      console.warn('[SchemaMapper] AI training failed, falling back to heuristics:', error);
+    }
+
+    return {};
+  }
+
+  /**
+   * Forgiving JSON parser that fixes common AI formatting mistakes.
+   */
+  private static sanitizeJson(s: string): string {
+    return s
+      .replace(/\/\*[\s\S]*?\*\/|([^:]|^)\/\/.*$/gm, '') // Remove comments
+      .replace(/[\u201C\u201D\u2018\u2019]/g, '"') // Replace smart quotes
+      .replace(/'/g, '"') // Replace single quotes
+      .replace(/([{,]\s*)([a-zA-Z0-9_]+?)\s*:/g, '$1"$2":') // Add quotes to keys
+      .replace(/,\s*([}\]])/g, '$1') // Remove trailing commas
+      .trim();
+  }
+
+  static getCached(projectId: string, keys: string[]): SchemaMap | undefined {
+    const cacheKey = `${projectId}:${keys.sort().join(',')}`;
+    return this.cache.get(cacheKey);
+  }
+}