@retrivora-ai/rag-engine 1.8.0 → 1.8.1

package/src/utils/UITransformer.ts

@@ -1,79 +1,5 @@
-import { VectorMatch } from '../types';
-
-/**
- * Supported UI visualization types
- */
-export type VisualizationType =
-  | 'pie_chart'
-  | 'bar_chart'
-  | 'line_chart'
-  | 'table'
-  | 'product_carousel'
-  | 'text';
-
-/**
- * Base structure for all UI transformation responses
- */
-export interface UITransformationResponse {
-  type: VisualizationType;
-  title: string;
-  description?: string;
-  data: unknown;
-}
-
-/**
- * Pie chart data structure
- */
-export interface PieChartData {
-  label: string;
-  value: number;
-  inStockCount?: number;
-  outOfStockCount?: number;
-  [key: string]: unknown;
-}
-
-/**
- * Bar chart data structure
- */
-export interface BarChartData {
-  category: string;
-  value: number;
-  inStockCount?: number;
-  outOfStockCount?: number;
-  [key: string]: unknown;
-}
-
-/**
- * Line chart data point
- */
-export interface LineChartDataPoint {
-  timestamp: string | number;
-  value: number;
-  label?: string;
-  [key: string]: unknown;
-}
-
-/**
- * Table representation
- */
-export interface TableData {
-  columns: string[];
-  rows: (string | number | boolean)[][];
-}
-
-/**
- * Product carousel item
- */
-export interface CarouselProduct {
-  id: string | number;
-  name: string;
-  price?: number | string;
-  image?: string;
-  inStock?: boolean;
-  brand?: string;
-  description?: string;
-  [key: string]: unknown;
-}
+import { VectorMatch, UITransformationResponse, PieChartData, BarChartData, LineChartDataPoint, TableData, CarouselProduct, VisualizationType } from '../types';
+import { ILLMProvider } from '../llm/ILLMProvider';
 
 /**
  * UITransformer — Converts vector database results into structured UI representations
@@ -82,109 +8,82 @@ export interface CarouselProduct {
  * format based on the content and structure of the data.
  */
 export class UITransformer {
+  /**
+   * Central dictionary of common synonyms for UI properties.
+   * This allows the system to be schema-agnostic by guessing field names.
+   */
+  private static readonly SYNONYMS: Record<string, string[]> = {
+    name: ['product', 'item', 'title', 'label', 'heading', 'subject'],
+    price: ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd'],
+    brand: ['manufacturer', 'vendor', 'make', 'company', 'brand_name', 'supplier'],
+    image: ['imageUrl', 'thumbnail', 'img', 'url', 'photo', 'picture', 'media', 'image_url', 'main_image', 'product_image', 'thumb'],
+    stock: ['inventory', 'quantity', 'count', 'availability', 'stock_level', 'inStock', 'is_available'],
+    description: ['summary', 'content', 'body', 'text', 'info', 'details'],
+  };
+
   /**
    * Main transformation method
-   * Analyzes user query and retrieved data to determine the best visualization format
-   *
-   * @param userQuery - The original user question/query
-   * @param retrievedData - Vector database retrieval results
-   * @returns Structured JSON response for UI rendering
+   * Analyzes user query and retrieved data to determine if a product carousel is needed.
    */
   static transform(
     userQuery: string,
-    retrievedData: VectorMatch[]
+    retrievedData: VectorMatch[],
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap
   ): UITransformationResponse {
     if (!retrievedData || retrievedData.length === 0) {
       return this.createTextResponse('No data available', 'No relevant data found for your query.');
     }
 
-    // Analyze the retrieved data to determine best format
-    const analysis = this.analyzeData(userQuery, retrievedData);
+    const isStockRequest = this.isStockQuery(userQuery);
+    const filteredData = isStockRequest
+      ? retrievedData.filter(item => this.determineStockStatus(item))
+      : retrievedData;
 
-    // Route to appropriate transformer based on analysis
-    switch (analysis.suggestedType) {
-      case 'product_carousel':
-        return this.transformToProductCarousel(retrievedData);
-      case 'pie_chart':
-        return this.transformToPieChart(retrievedData);
-      case 'bar_chart':
-        return this.transformToBarChart(retrievedData);
-      case 'line_chart':
-        return this.transformToLineChart(retrievedData);
-      case 'table':
-        return this.transformToTable(retrievedData);
-      default:
-        return this.transformToText(retrievedData);
-    }
-  }
+    const categories = this.detectCategories(filteredData);
+    const hasProducts = filteredData.some(item => this.isProductData(item));
+    const isTimeSeries = filteredData.some(item => this.isTimeSeriesData(item));
+    const isTrendQuery = this.isTrendQuery(userQuery);
 
-  /**
-   * Analyzes the data and query to suggest the best visualization type
-   */
-  private static analyzeData(
-    query: string,
-    data: VectorMatch[]
-  ): {
-    suggestedType: VisualizationType;
-    confidence: number;
-    hasProducts: boolean;
-    hasTimeSeries: boolean;
-    hasCategories: boolean;
-  } {
-    const queryLower = query.toLowerCase();
-    const hasProducts = data.some(item =>
-      this.isProductData(item)
-    );
-    const hasTimeSeries = data.some(item =>
-      this.isTimeSeriesData(item)
-    );
-    const hasCategories = this.detectCategories(data).length > 0;
-    const isDistributionQuery =
-      queryLower.includes('distribution') ||
-      queryLower.includes('breakdown') ||
-      queryLower.includes('percentage') ||
-      queryLower.includes('proportion');
-    const isComparisonQuery =
-      queryLower.includes('compare') ||
-      queryLower.includes('vs') ||
-      queryLower.includes('difference');
-    const isTrendQuery =
-      queryLower.includes('trend') ||
-      queryLower.includes('over time') ||
-      queryLower.includes('growth') ||
-      queryLower.includes('decline');
-
-    // Decision logic
-    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
-      return { suggestedType: 'product_carousel', confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
+    // 1. High Priority: Specific visualizations for trends
+    if (isTrendQuery && isTimeSeries) {
+      return this.transformToLineChart(filteredData);
     }
 
-    if (isTrendQuery && hasTimeSeries) {
-      return { suggestedType: 'line_chart', confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
+    // 2. High Priority: Products (Always prefer carousel for products unless it's a specific breakdown request)
+    if (hasProducts && !this.shouldShowCategoryChart(userQuery, categories)) {
+      return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
 
-    if (isDistributionQuery && hasCategories) {
-      return { suggestedType: 'pie_chart', confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
+    // 3. Category breakdowns
+    if (categories.length > 1 && this.shouldShowCategoryChart(userQuery, categories)) {
+      return this.transformToPieChart(filteredData);
     }
 
-    if (isComparisonQuery && hasCategories && data.length > 2) {
-      return { suggestedType: 'bar_chart', confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
+    // 4. Secondary: Products as carousel if not already handled
+    if (hasProducts) {
+      return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
 
-    if (data.length > 5 && this.hasMultipleFields(data)) {
-      return { suggestedType: 'table', confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+    // 5. Detailed data without clear product structure -> Table
+    if (this.hasMultipleFields(filteredData)) {
+      return this.transformToTable(filteredData);
     }
 
-    return { suggestedType: 'text', confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
+    return this.transformToText(filteredData);
   }
 
   /**
    * Transform data to product carousel format
    */
-  private static transformToProductCarousel(data: VectorMatch[]): UITransformationResponse {
+  private static transformToProductCarousel(
+    data: VectorMatch[],
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap
+  ): UITransformationResponse {
     const products: CarouselProduct[] = data
       .filter(item => this.isProductData(item))
-      .map(item => this.extractProductInfo(item))
+      .map(item => this.extractProductInfo(item, config, trainedSchema))
       .filter((p): p is CarouselProduct => p !== null);
 
     return {
@@ -320,16 +219,171 @@ export class UITransformer {
     };
   }
 
+  static parseTransformationResponse(raw: string): UITransformationResponse | null {
+    const payloadText = this.extractJsonCandidate(raw);
+    if (!payloadText) return null;
+
+    try {
+      const parsed = JSON.parse(payloadText);
+      return this.normalizeTransformation(parsed);
+    } catch {
+      return null;
+    }
+  }
+
+  private static extractJsonCandidate(raw: string): string | null {
+    if (!raw) return null;
+
+    const cleaned = raw
+      .replace(/```(?:json|chart|ui)?\s*/gi, '')
+      .replace(/```/g, '')
+      .trim();
+
+    const start = cleaned.indexOf('{');
+    if (start === -1) return null;
+
+    let depth = 0;
+    let inString = false;
+    let escape = false;
+
+    for (let i = start; i < cleaned.length; i += 1) {
+      const char = cleaned[i];
+      if (escape) {
+        escape = false;
+        continue;
+      }
+      if (char === '\\') {
+        escape = true;
+        continue;
+      }
+      if (char === '"') {
+        inString = !inString;
+        continue;
+      }
+      if (inString) continue;
+
+      if (char === '{') depth += 1;
+      if (char === '}') {
+        depth -= 1;
+        if (depth === 0) {
+          return cleaned.slice(start, i + 1);
+        }
+      }
+    }
+
+    return null;
+  }
+
+  private static normalizeTransformation(payload: unknown): UITransformationResponse | null {
+    if (!payload || typeof payload !== 'object') return null;
+
+    const payloadObj = payload as Record<string, unknown>;
+    const type = this.normalizeVisualizationType(String(payloadObj.type ?? payloadObj.view ?? payloadObj.chartType ?? ''));
+    if (!type) return null;
+
+    const title = String(payloadObj.title ?? payloadObj.heading ?? 'Visualization');
+    const description = payloadObj.description ? String(payloadObj.description) : undefined;
+    const rawData = payloadObj.data ?? payloadObj.table ?? payloadObj.rows ?? payloadObj.items ?? payloadObj.content ?? null;
+
+    const data = type === 'text' && typeof rawData === 'string'
+      ? { content: rawData }
+      : rawData;
+
+    const transformation: UITransformationResponse = {
+      type,
+      title,
+      description,
+      data,
+    };
+
+    return this.validateTransformation(transformation) ? transformation : null;
+  }
+
+  private static normalizeVisualizationType(type: string): VisualizationType | null {
+    const mapping: Record<string, VisualizationType> = {
+      pie: 'pie_chart',
+      pie_chart: 'pie_chart',
+      bar: 'bar_chart',
+      bar_chart: 'bar_chart',
+      line: 'line_chart',
+      line_chart: 'line_chart',
+      radar: 'radar_chart',
+      radar_chart: 'radar_chart',
+      table: 'table',
+      text: 'text',
+      product_carousel: 'product_carousel',
+      carousel: 'carousel',
+    };
+
+    return mapping[type.toLowerCase()] ?? null;
+  }
+
+  private static validateTransformation(transformation: UITransformationResponse): boolean {
+    const { type, data } = transformation;
+
+    switch (type) {
+      case 'pie_chart':
+      case 'bar_chart':
+        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
+          item !== null && typeof item === 'object' &&
+          (typeof item.value === 'number' || !Number.isNaN(Number(item.value)))
+        );
+      case 'line_chart':
+        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
+          item !== null && typeof item === 'object' &&
+          'timestamp' in item &&
+          (typeof item.value === 'number' || !Number.isNaN(Number(item.value)))
+        );
+      case 'radar_chart':
+        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
+          item !== null && typeof item === 'object' &&
+          'attribute' in item
+        );
+      case 'table':
+        return (
+          typeof data === 'object' &&
+          data !== null &&
+          Array.isArray((data as Record<string, unknown>).columns) &&
+          Array.isArray((data as Record<string, unknown>).rows)
+        );
+      case 'text':
+        return (
+          typeof data === 'object' &&
+          data !== null &&
+          typeof (data as Record<string, unknown>).content === 'string'
+        );
+      case 'product_carousel':
+      case 'carousel':
+        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
+          item !== null && typeof item === 'object' && (('id' in item) || ('name' in item))
+        );
+      default:
+        return false;
+    }
+  }
+
   /**
    * Helper: Check if data item is product-related
    */
   private static isProductData(item: VectorMatch): boolean {
     const content = (item.content || '').toLowerCase();
-    const productKeywords = ['product', 'price', 'stock', 'item', 'sku', 'brand', 'model'];
-    return productKeywords.some(kw => content.includes(kw)) ||
-           Object.keys(item.metadata || {}).some(k =>
-             ['name', 'price', 'product', 'sku'].includes(k.toLowerCase())
-           );
+    const productKeywords = [
+      'product', 'price', 'stock', 'item', 'sku', 'brand', 'model', 'msrp', 'inventory', 'buy', 'shop',
+      'beauty', 'care', 'cosmetic', 'facial', 'cream', 'serum', 'mask', 'makeup', 'fragrance'
+    ];
+
+    // Check content for keywords
+    const hasKeywords = productKeywords.some(kw => content.includes(kw));
+
+    // Check metadata keys
+    const hasMetadataKey = Object.keys(item.metadata || {}).some(k =>
+      ['name', 'price', 'product', 'sku', 'brand', 'model', 'cost', 'item'].includes(k.toLowerCase())
+    );
+
+    // Failsafe: if we have a dollar sign followed by a number, it's likely a product
+    const hasPricePattern = /\$\s*\d+/.test(content);
+
+    return hasKeywords || hasMetadataKey || hasPricePattern;
   }
 
   /**
@@ -337,41 +391,156 @@ export class UITransformer {
    */
   private static isTimeSeriesData(item: VectorMatch): boolean {
     const content = (item.content || '').toLowerCase();
-    const timeKeywords = ['date', 'time', 'year', 'month', 'week', 'day', 'hour', 'trend', 'historical'];
-    return timeKeywords.some(kw => content.includes(kw)) ||
-           Object.keys(item.metadata || {}).some(k =>
-             ['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase())
-           );
+    const timeKeywords = ['trend', 'historical', 'growth', 'decline', 'change', 'increase', 'decrease'];
+    const hasTimeKeyword = timeKeywords.some(kw => content.includes(kw));
+
+    const metadata = item.metadata || {};
+    const maybeDateKeys = Object.keys(metadata).filter((k) =>
+      ['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase())
+    );
+    const hasValidDateValue = maybeDateKeys.some((key) => {
+      const value = metadata[key];
+      if (typeof value === 'string') {
+        return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
+      }
+      return typeof value === 'number';
+    });
+
+    return hasTimeKeyword || hasValidDateValue;
+  }
+
+  private static shouldShowCategoryChart(query: string, categories: string[]): boolean {
+    if (categories.length < 2) {
+      return false;
+    }
+
+    const normalized = query.toLowerCase();
+    const chartKeywords = [
+      'distribution',
+      'breakdown',
+      'by category',
+      'by type',
+      'compare',
+      'share',
+      'percentage',
+      'segmentation',
+      'split',
+      'category breakdown',
+      'category distribution',
+    ];
+
+    return chartKeywords.some(keyword => normalized.includes(keyword));
+  }
+
+  private static isTrendQuery(query: string): boolean {
+    const normalized = query.toLowerCase();
+    const trendKeywords = [
+      'trend',
+      'over time',
+      'historical',
+      'growth',
+      'decline',
+      'increase',
+      'decrease',
+      'year',
+      'month',
+      'week',
+      'day',
+      'comparison',
+      'compare',
+      'changes',
+      'timeline',
+    ];
+
+    return trendKeywords.some(keyword => normalized.includes(keyword));
+  }
+
+  private static isStockQuery(query: string): boolean {
+    const normalized = query.toLowerCase();
+    return (
+      normalized.includes('in stock') ||
+      normalized.includes('available') ||
+      normalized.includes('availability') ||
+      normalized.includes('inventory') ||
+      normalized.includes('stock status')
+    );
   }
 
   /**
-   * Helper: Extract product information from vector match
+   * Helper: Extract property from metadata using mapping, AI training, case-insensitivity, and synonyms.
    */
-  private static extractProductInfo(item: VectorMatch): CarouselProduct | null {
-    const meta = item.metadata || {};
+  private static getDynamicVal(
+    meta: Record<string, unknown> | undefined,
+    uiKey: string,
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap
+  ): unknown {
+    if (!meta) return undefined;
+
+    const mapping = config?.rag?.uiMapping;
+    if (mapping && mapping[uiKey]) {
+      const mappedKey = mapping[uiKey];
+      if (meta[mappedKey] !== undefined) return meta[mappedKey];
+    }
 
-    // Try to extract product info from metadata first
-    if (meta.name || meta.product) {
-      return {
-        id: item.id,
-        name: (meta.name || meta.product) as string,
-        price: meta.price as string | number | undefined,
-        image: (meta.image || meta.imageUrl) as string | undefined,
-        brand: meta.brand as string | undefined,
-        description: item.content,
-        inStock: this.determineStockStatus(item),
-      };
+    if (trainedSchema && typeof trainedSchema === 'object' && trainedSchema !== null) {
+      const trainedKey = (trainedSchema as Record<string, unknown>)[uiKey];
+      if (trainedKey && meta[trainedKey as string] !== undefined) return meta[trainedKey as string];
     }
 
-    // Fallback: try to parse from content
-    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
+    const synonyms = this.SYNONYMS[uiKey] || [];
+    const searchKeys = [uiKey, ...synonyms].map((k) => k.toLowerCase());
+
+    const foundDirectKey = Object.keys(meta).find((k) => searchKeys.includes(k.toLowerCase()));
+    if (foundDirectKey) return meta[foundDirectKey];
+
+    const foundPartialKey = Object.keys(meta).find((k) => {
+      const keyLow = k.toLowerCase();
+      return searchKeys.some((sk) => keyLow.includes(sk) || sk.includes(keyLow));
+    });
+
+    return foundPartialKey ? meta[foundPartialKey] : undefined;
+  }
+
+  private static extractProductInfo(
+    item: VectorMatch,
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap
+  ): CarouselProduct | null {
+    const meta = item.metadata || {};
+
+    const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
+    const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
+    const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
+
+    // If we have at least a name or a clear product-like structure in content
+    if (name || this.isProductData(item)) {
+      // Robust name extraction: metadata > regex > content snippet
+      let finalName = name ? String(name) : undefined;
+      if (!finalName) {
+        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+        finalName = nameMatch ? nameMatch[1].trim() : item.content.split('\n')[0].substring(0, 60);
+      }
+
+      // Robust price extraction: metadata > regex
+      let finalPrice: string | number | undefined =
+        typeof price === 'number' || typeof price === 'string'
+          ? price
+          : undefined;
+
+      if (!finalPrice) {
+        const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
+        if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
+      }
+
+      const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
 
-    if (nameMatch) {
       return {
         id: item.id,
-        name: nameMatch[1],
-        price: priceMatch ? parseFloat(priceMatch[1]) : undefined,
+        name: finalName,
+        price: finalPrice,
+        image: typeof imageValue === 'string' ? imageValue : undefined,
+        brand: brand ? String(brand) : undefined,
         description: item.content,
         inStock: this.determineStockStatus(item),
       };
@@ -401,7 +570,15 @@ export class UITransformer {
         tags.forEach(t => categories.add(String(t)));
       }
 
-      // Parse from content
+      // Parse category strings from content lines like "Shoes & Footwear: • Product"
+      const contentCategories = Array.from(new Set(
+        Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm))
+          .map(match => match[1].trim())
+          .filter(Boolean)
+      ));
+      contentCategories.forEach((category) => categories.add(category));
+
+      // Parse from structured category hints in content
       const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
       if (categoryMatch) {
         categories.add(categoryMatch[1].trim());
@@ -552,4 +729,137 @@ export class UITransformer {
 
     return fieldCount.size > 2;
   }
+
+  // ─── LLM-Driven Visualization Decision ────────────────────────────────────
+
+  /**
+   * analyzeAndDecide — sends user question + RAG data to the LLM with a
+   * structured system prompt and parses the JSON response into a
+   * UITransformationResponse.
+   *
+   * This is the recommended entry point for production use. The heuristic
+   * `transform()` method is used as a fallback if the LLM call fails.
+   *
+   * System prompt instructs the LLM to:
+   *   - Analyze the question and retrieved data
+   *   - Choose the best visualization: bar_chart | line_chart | pie_chart | table | text
+   *   - Return a strict JSON object — no prose, no markdown fences
+   *
+   * @param query   - the original user question
+   * @param sources - vector DB matches returned by RAG retrieval
+   * @param llm     - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
+   * @returns       - a validated UITransformationResponse (type + title + description + data)
+   */
+  static async analyzeAndDecide(
+    query: string,
+    sources: VectorMatch[],
+    llm: ILLMProvider,
+  ): Promise<UITransformationResponse> {
+    try {
+      const context = this.buildContextSummary(sources);
+      const systemPrompt = this.buildVisualizationSystemPrompt();
+
+      const userPrompt = [
+        `USER QUESTION: ${query}`,
+        '',
+        'RETRIEVED DATA (JSON):',
+        context,
+      ].join('\n');
+
+      const rawResponse = await llm.chat(
+        [{ role: 'user', content: userPrompt }],
+        '',
+        { systemPrompt, temperature: 0 },
+      );
+
+      const parsed = this.parseTransformationResponse(rawResponse);
+      if (parsed) {
+        console.debug('[UITransformer] LLM chose visualization type:', parsed.type);
+        return parsed;
+      }
+
+      console.warn('[UITransformer] LLM returned unparseable response; falling back to heuristic.');
+    } catch (err) {
+      console.warn('[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.', err);
+    }
+
+    // Fallback: keyword-heuristic transform
+    return this.transform(query, sources);
+  }
+
+  /**
+   * Build the system prompt that instructs the LLM to return a visualization JSON.
+   */
+  private static buildVisualizationSystemPrompt(): string {
+    return `You are a data visualization expert embedded in a RAG chat system.
+You will receive a user question and structured data retrieved from a vector database.
+Your ONLY job is to analyze this information and return a single JSON object that tells
+the frontend how to visualize it.
+
+Return ONLY a valid JSON object — no markdown code fences, no explanation, no prose.
+
+The JSON must have this exact shape:
+{
+  "type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
+  "title": "A concise, descriptive title for the visualization",
+  "description": "One sentence describing what the visualization shows",
+  "data": <structured data — see rules below>
 }
+
+DATA SHAPE per type:
+- bar_chart:  array of { "category": string, "value": number }
+- line_chart: array of { "timestamp": string, "value": number, "label": string }
+- pie_chart:  array of { "label": string, "value": number }
+- radar_chart: array of { "attribute": string, "[series1]": number, "[series2]": number, ... }
+  Example for radar_chart:
+  [
+    { "attribute": "Longevity", "Dolce Shine": 4, "CK One": 3 },
+    { "attribute": "Sillage", "Dolce Shine": 3, "CK One": 4 },
+    { "attribute": "Freshness", "Dolce Shine": 5, "CK One": 4 }
+  ]
+- table:      { "columns": string[], "rows": (string|number)[][] }
+- text:       { "content": "<prose answer>" }
+
+DECISION RULES (follow strictly):
+1. bar_chart  → comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
+2. line_chart → trends or changes over time (dates, months, years, sequential events)
+3. pie_chart  → proportional breakdown or percentage distribution
+4. radar_chart → comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
+5. table      → multi-field structured records where each item has ≥ 3 attributes
+6. text       → conversational or free-form answers where no chart adds value
+
+IMPORTANT:
+- Aggregate numeric values from the raw data — do not pass raw object arrays as data.
+- Ensure all "value" fields are numbers, not strings.
+- For bar/line/pie, keep at most 12 data points for readability.
+- Never include nested objects or arrays inside bar_chart / line_chart / pie_chart data items.`;
+  }
+
+  /**
+   * Serialize retrieved vector matches into a compact JSON context string.
+   * Limits the total character count to avoid exceeding LLM context windows.
+   */
+  private static buildContextSummary(sources: VectorMatch[], maxChars = 6000): string {
+    const items = sources.map((s, i) => ({
+      index: i + 1,
+      content: s.content?.substring(0, 400) ?? '',
+      metadata: s.metadata ?? {},
+      score: s.score ?? 0,
+    }));
+
+    const full = JSON.stringify(items, null, 2);
+    if (full.length <= maxChars) return full;
+
+    // Truncate: include as many items as fit
+    const partial: typeof items = [];
+    let chars = 2; // for []
+    for (const item of items) {
+      const chunk = JSON.stringify(item);
+      if (chars + chunk.length + 2 > maxChars) break;
+      partial.push(item);
+      chars += chunk.length + 2;
+    }
+    return JSON.stringify(partial, null, 2) + '\n// ... truncated';
+  }
+}
+