@retrivora-ai/rag-engine 1.8.9 → 1.9.1

package/src/utils/UITransformer.ts

@@ -1,80 +1,487 @@
-import { VectorMatch, UITransformationResponse, PieChartData, LineChartDataPoint, TableData, CarouselProduct, VisualizationType } from '../types';
+import {
+  VectorMatch,
+  UITransformationResponse,
+  PieChartData,
+  BarChartData,
+  LineChartDataPoint,
+  ScatterPlotDataPoint,
+  MetricCardData,
+  TableData,
+  CarouselProduct,
+  VisualizationType,
+} from '../types';
 import { ILLMProvider } from '../llm/ILLMProvider';
 import { resolveMetadataValue } from './synonyms';
 
+// ─── Intent Types ─────────────────────────────────────────────────────────────
+
+/**
+ * Structured intent extracted from the user query.
+ * Produced either by the LLM (preferred) or by heuristic fallback.
+ */
+export interface QueryIntent {
+  /** Primary display mode the user is asking for */
+  visualizationHint:
+  | 'trend'
+  | 'comparison'
+  | 'distribution'
+  | 'composition'
+  | 'correlation'
+  | 'ranking'
+  | 'kpi'
+  | 'tabular'
+  | 'geographic'
+  | 'category_breakdown'
+  | 'product_browse'
+  | 'table'
+  | 'text';
+
+  /** Chart family that best matches the analytical intent */
+  recommendedChart:
+  | 'line_chart'
+  | 'bar_chart'
+  | 'histogram'
+  | 'pie_chart'
+  | 'donut_chart'
+  | 'scatter_plot'
+  | 'horizontal_bar'
+  | 'metric_card'
+  | 'table'
+  | 'geo_map'
+  | 'text';
+
+  /** User wants only in-stock / available items */
+  filterInStockOnly: boolean;
+
+  /** User wants a tabular / list-style answer explicitly */
+  wantsExplicitTable: boolean;
+
+  /** User is asking about time-based changes */
+  isTemporal: boolean;
+
+  /** User is comparing, ranking, or contrasting entities/categories */
+  isComparison: boolean;
+
+  /** The detected language of the query (BCP-47 tag, e.g. "en", "de", "hi") */
+  language: string;
+
+  /** Free-form note the LLM can add for transparency (not used in logic) */
+  reasoning?: string;
+}
+
+type PrimitiveValue = string | number | boolean;
+type AggregationOperation = 'sum' | 'average' | 'count' | 'min' | 'max' | 'median';
+
+interface DataRecord {
+  id: string | number;
+  content: string;
+  score: number;
+  fields: Record<string, PrimitiveValue>;
+  source: VectorMatch;
+}
+
+interface FieldProfile {
+  key: string;
+  label: string;
+  kind: 'number' | 'date' | 'category' | 'boolean' | 'text';
+  values: PrimitiveValue[];
+  uniqueCount: number;
+}
+
+interface DataProfile {
+  records: DataRecord[];
+  fields: FieldProfile[];
+  numericFields: FieldProfile[];
+  dateFields: FieldProfile[];
+  categoricalFields: FieldProfile[];
+  booleanFields: FieldProfile[];
+}
+
+// ─── UITransformer ────────────────────────────────────────────────────────────
+
 /**
- * UITransformer — Converts vector database results into structured UI representations
+ * UITransformer — Converts vector database results into structured UI representations.
  *
- * This class analyzes retrieved data and automatically selects the best visualization
- * format based on the content and structure of the data.
+ * Intent detection is now delegated to the LLM via `detectIntent()`.
+ * Static keyword lists are used only as a last-resort fallback when no LLM
+ * is available (e.g. during unit tests or offline mode).
  */
 export class UITransformer {
 
+  // ─── Public Entry Points ─────────────────────────────────────────────────
+
   /**
-   * Main transformation method
-   * Analyzes user query and retrieved data to determine if a product carousel is needed.
+   * Heuristic-only transform (no LLM required).
+   * Uses the lightweight heuristic intent detector as a fallback.
+   * Prefer `analyzeAndDecide()` in production.
    */
   static transform(
     userQuery: string,
     retrievedData: VectorMatch[],
     config?: import('../config/RagConfig').RagConfig,
-    trainedSchema?: import('./SchemaMapper').SchemaMap
+    trainedSchema?: import('./SchemaMapper').SchemaMap,
+    /** Optionally pass a pre-computed intent to skip re-detection */
+    intent?: QueryIntent,
   ): UITransformationResponse {
     if (!retrievedData || retrievedData.length === 0) {
       return this.createTextResponse('No data available', 'No relevant data found for your query.');
     }
 
-    const isStockRequest = this.isStockQuery(userQuery);
-    const filteredData = isStockRequest
+    const resolvedIntent = intent ?? this.detectIntentHeuristic(userQuery);
+
+    const filteredData = resolvedIntent.filterInStockOnly
       ? retrievedData.filter(item => this.determineStockStatus(item))
       : retrievedData;
 
-    const groupByField = this.determineGroupByField(userQuery, filteredData);
-    const categories = this.detectCategories(filteredData, groupByField);
+    const profile = this.profileData(filteredData);
     const hasProducts = filteredData.some(item => this.isProductData(item));
-    const isTimeSeries = filteredData.some(item => this.isTimeSeriesData(item));
-    const isTrendQuery = this.isTrendQuery(userQuery);
+    const wantsPieLikeChart = ['pie_chart', 'donut_chart'].includes(resolvedIntent.recommendedChart);
+
+    // Priority 1 – Explicit trends
+    if (resolvedIntent.visualizationHint === 'trend' && profile.dateFields.length > 0 && profile.numericFields.length > 0) {
+      return this.transformToLineChart(profile);
+    }
 
-    // 1. High Priority: Specific visualizations for trends
-    if (isTrendQuery && isTimeSeries) {
-      return this.transformToLineChart(filteredData);
+    // Priority 2 – Explicit pie/donut or composition/category distribution
+    if (wantsPieLikeChart || ['composition', 'category_breakdown'].includes(resolvedIntent.visualizationHint)) {
+      const pieChart = this.transformToPieChart(filteredData, profile, userQuery);
+      if (pieChart) return pieChart;
     }
 
-    const explicitChartRequest = this.isExplicitChartQuery(userQuery);
+    // Priority 3 – Comparison/ranking → bar chart
+    if (['comparison', 'ranking'].includes(resolvedIntent.visualizationHint)) {
+      return this.transformToBarChart(filteredData, profile, userQuery, resolvedIntent.visualizationHint === 'ranking');
+    }
 
-    // 2. High Priority: Products (Always prefer carousel for products unless it's a specific breakdown request)
-    if (hasProducts && !explicitChartRequest && !this.shouldShowCategoryChart(userQuery)) {
-      return this.transformToProductCarousel(filteredData, config, trainedSchema);
+    // Priority 4 – Distribution/correlation/geographic/KPI fallback to supported views
+    if (resolvedIntent.visualizationHint === 'distribution') {
+      return this.transformToHistogram(profile, userQuery) ?? this.transformToBarChart(filteredData, profile, userQuery);
     }
 
-    // 3. Category breakdowns
-    if (explicitChartRequest || (categories.length > 1 && this.shouldShowCategoryChart(userQuery))) {
-      return this.transformToPieChart(filteredData, groupByField);
+    if (resolvedIntent.visualizationHint === 'correlation') {
+      return this.transformToScatterPlot(profile, userQuery) ?? this.transformToBarChart(filteredData, profile, userQuery);
     }
 
-    // 4. Secondary: Products as carousel if not already handled
-    if (hasProducts) {
-      return this.transformToProductCarousel(filteredData, config, trainedSchema);
+    if (resolvedIntent.visualizationHint === 'geographic') {
+      return this.transformToBarChart(filteredData, profile, userQuery);
+    }
+
+    // Priority 5 – Structured list/filter questions should stay tabular even if
+    // source metadata happens to contain product-like fields such as category.
+    // This must run before KPI detection because fields such as "employee count"
+    // contain the word "count" but still describe records to list/filter.
+    if (this.isStructuredListQuery(userQuery)) {
+      return this.hasMultipleFields(filteredData)
+        ? this.transformToTable(filteredData, userQuery)
+        : this.transformToText(filteredData);
+    }
+
+    if (resolvedIntent.visualizationHint === 'kpi') {
+      return this.transformToMetricCard(profile, userQuery) ?? this.transformToText(filteredData);
     }
 
-    // 5. Detailed data without clear product structure -> Table
-    if (this.hasMultipleFields(filteredData)) {
-      return this.transformToTable(filteredData);
+    // Priority 6 – Explicit tabular/detail request
+    if (
+      ['tabular', 'table'].includes(resolvedIntent.visualizationHint) ||
+      resolvedIntent.wantsExplicitTable
+    ) {
+      return this.hasMultipleFields(filteredData)
+        ? this.transformToTable(filteredData, userQuery)
+        : this.transformToText(filteredData);
     }
 
+    // Priority 7 – Product carousel only for explicit product intent.
+    if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === 'product_browse') {
+      return this.transformToProductCarousel(filteredData, config, trainedSchema);
+    }
+
+    const automatic = this.chooseAutomaticVisualization(filteredData, profile, userQuery);
+    if (automatic) return automatic;
+
+    // Fallback – plain text
     return this.transformToText(filteredData);
   }
 
   /**
-   * Transform data to product carousel format
+   * LLM-driven entry point (recommended for production).
+   *
+   * Step 1 — Detect intent via a dedicated, lightweight LLM call.
+   * Step 2 — Pass the intent + data to the visualization-selection prompt.
+   * Step 3 — Fall back to the heuristic `transform()` if either LLM call fails.
+   */
+  static async analyzeAndDecide(
+    query: string,
+    sources: VectorMatch[],
+    llm: ILLMProvider,
+  ): Promise<UITransformationResponse> {
+    // ── Step 1: Detect intent ───────────────────────────────────────────────
+    let intent: QueryIntent;
+    try {
+      intent = await this.detectIntent(query, llm);
+      console.debug('[UITransformer] Detected intent:', intent);
+    } catch (err) {
+      console.warn('[UITransformer] Intent detection failed; using heuristic.', err);
+      intent = this.detectIntentHeuristic(query);
+    }
+
+    if (this.isProductQuery(query) && ['text', 'product_browse'].includes(intent.visualizationHint)) {
+      return this.transform(
+        query,
+        sources,
+        undefined,
+        undefined,
+        { ...intent, visualizationHint: 'product_browse', recommendedChart: 'text' },
+      );
+    }
+
+    // ── Step 2: Ask the LLM to pick a visualization ─────────────────────────
+    try {
+      const context = this.buildContextSummary(sources);
+      const systemPrompt = this.buildVisualizationSystemPrompt();
+
+      const userPrompt = [
+        `USER QUESTION: ${query}`,
+        '',
+        `DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
+        '',
+        'RETRIEVED DATA (JSON):',
+        context,
+      ].join('\n');
+
+      const rawResponse = await llm.chat(
+        [{ role: 'user', content: userPrompt }],
+        '',
+        { systemPrompt, temperature: 0 },
+      );
+
+      const parsed = this.parseTransformationResponse(rawResponse);
+      if (parsed) {
+        const intentAllowsTable =
+          intent.wantsExplicitTable ||
+          ['tabular', 'table', 'geographic'].includes(intent.visualizationHint);
+        const intentWantsPieLikeChart =
+          ['pie_chart', 'donut_chart'].includes(intent.recommendedChart) ||
+          ['composition', 'category_breakdown'].includes(intent.visualizationHint);
+        // Respect the intent: never sneak in a table the user didn't ask for
+        if (parsed.type === 'table' && !intentAllowsTable) {
+          console.debug('[UITransformer] LLM chose table but intent says no. Falling back to text.');
+          return this.transform(query, sources, undefined, undefined, intent);
+        }
+        if (intentWantsPieLikeChart && parsed.type !== 'pie_chart' && this.detectCategories(sources).length > 1) {
+          console.debug('[UITransformer] LLM ignored pie/composition intent. Using deterministic pie chart.');
+          return this.transform(query, sources, undefined, undefined, intent);
+        }
+        console.debug('[UITransformer] LLM chose visualization type:', parsed.type);
+        return parsed;
+      }
+
+      console.warn('[UITransformer] LLM returned unparseable response; falling back to heuristic.');
+    } catch (err) {
+      console.warn('[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.', err);
+    }
+
+    // ── Step 3: Heuristic fallback ──────────────────────────────────────────
+    return this.transform(query, sources, undefined, undefined, intent);
+  }
+
+  // ─── Dynamic Intent Detection ─────────────────────────────────────────────
+
+  /**
+   * Calls the LLM with a compact, focused prompt to extract a structured
+   * `QueryIntent` from the user's query.
+   *
+   * Keeping this as a *separate* call from visualization selection means:
+   * - The prompt is shorter and more reliable.
+   * - The intent object can be reused across both the heuristic and LLM paths.
+   * - It is easy to unit-test intent detection in isolation.
+   */
+  static async detectIntent(query: string, llm: ILLMProvider): Promise<QueryIntent> {
+    const systemPrompt = `You are an intent classifier for a product-search RAG system.
+Given a user query, return ONLY a valid JSON object with this exact shape — no prose, no markdown:
+
+{
+  "visualizationHint": "trend" | "comparison" | "distribution" | "composition" | "correlation" | "ranking" | "kpi" | "tabular" | "geographic" | "product_browse" | "table" | "text",
+  "recommendedChart": "line_chart" | "bar_chart" | "histogram" | "pie_chart" | "donut_chart" | "scatter_plot" | "horizontal_bar" | "metric_card" | "table" | "geo_map" | "text",
+  "filterInStockOnly": boolean,
+  "wantsExplicitTable": boolean,
+  "isTemporal": boolean,
+  "isComparison": boolean,
+  "language": "<BCP-47 language tag, e.g. en, de, hi, ja>",
+  "reasoning": "<one short sentence — why you chose these values>"
+}
+
+RULES:
+- visualizationHint and recommendedChart
+    "trend"        → keywords: trend, growth, over time; recommendedChart "line_chart"
+    "comparison"   → keywords: compare, versus, top; recommendedChart "bar_chart"
+    "distribution" → keywords: distribution, spread; recommendedChart "histogram"
+    "composition"  → keywords: share, percentage, breakup, breakdown; recommendedChart "pie_chart" or "donut_chart"
+    "correlation"  → keywords: relation, correlation; recommendedChart "scatter_plot"
+    "ranking"      → keywords: highest, lowest, top 10, ranking; recommendedChart "horizontal_bar"
+    "kpi"          → keywords: total, average, count; recommendedChart "metric_card"
+    "tabular"      → keywords: detailed records, table, grid, spreadsheet; recommendedChart "table"
+    "geographic"   → keywords: region, country, map; recommendedChart "geo_map"
+    "product_browse" → user is browsing, searching, describing, viewing details for, or asking about one or more products without analytical visualization intent
+    "table"        → legacy alias for "tabular"; use only if the query literally says table
+    "text"         → conversational, factual, or other intent
+- If the user explicitly asks for a chart type, honor that chart type in recommendedChart.
+- If a query says "distribution ... in a pie chart", use visualizationHint "composition" and recommendedChart "pie_chart".
+- filterInStockOnly: true only if user mentions stock, availability, in stock, etc.
+- wantsExplicitTable: true if the user asks for a table, list, grid, spreadsheet, or detailed records.
+- isTemporal: true if time words appear (trend, historical, over time, last year, monthly, etc.)
+- isComparison: true if user compares, ranks, or contrasts entities or categories.
+- language: detect from the query text itself; default "en" if uncertain.`;
+
+    const rawResponse = await llm.chat(
+      [{ role: 'user', content: `QUERY: ${query}` }],
+      '',
+      { systemPrompt, temperature: 0 },
+    );
+
+    const parsed = this.parseIntentResponse(rawResponse);
+    if (!parsed) {
+      throw new Error(`Could not parse intent JSON from LLM response: ${rawResponse}`);
+    }
+    return parsed;
+  }
+
+  /**
+   * Parse and validate the raw LLM response into a `QueryIntent`.
+   */
+  private static parseIntentResponse(raw: string): QueryIntent | null {
+    const jsonStr = this.extractJsonCandidate(raw);
+    if (!jsonStr) return null;
+
+    try {
+      const obj = JSON.parse(jsonStr) as Partial<QueryIntent>;
+
+      const validHints = [
+        'trend', 'comparison', 'distribution', 'composition', 'correlation',
+        'ranking', 'kpi', 'tabular', 'geographic', 'category_breakdown',
+        'product_browse', 'table', 'text',
+      ] as const;
+      const validCharts = [
+        'line_chart', 'bar_chart', 'histogram', 'pie_chart', 'donut_chart',
+        'scatter_plot', 'horizontal_bar', 'metric_card', 'table', 'geo_map', 'text',
+      ] as const;
+
+      const hint = obj.visualizationHint;
+      if (!hint || !validHints.includes(hint as typeof validHints[number])) return null;
+      const normalizedHint = hint === 'category_breakdown' ? 'composition' : hint;
+      const recommendedChart =
+        typeof obj.recommendedChart === 'string' &&
+          validCharts.includes(obj.recommendedChart as typeof validCharts[number])
+          ? obj.recommendedChart as QueryIntent['recommendedChart']
+          : this.getRecommendedChartForIntent(normalizedHint as QueryIntent['visualizationHint']);
+
+      return {
+        visualizationHint: normalizedHint as QueryIntent['visualizationHint'],
+        recommendedChart,
+        filterInStockOnly: Boolean(obj.filterInStockOnly),
+        wantsExplicitTable: Boolean(obj.wantsExplicitTable),
+        isTemporal: Boolean(obj.isTemporal),
+        isComparison: Boolean(obj.isComparison),
+        language: typeof obj.language === 'string' && obj.language ? obj.language : 'en',
+        reasoning: typeof obj.reasoning === 'string' ? obj.reasoning : undefined,
+      };
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Heuristic intent detector — used when the LLM is unavailable.
+   * Intentionally minimal: it should only catch the most obvious signals.
+   * The LLM path handles everything subtle.
    */
+  static detectIntentHeuristic(query: string): QueryIntent {
+    const q = query.toLowerCase();
+
+    const isTemporal = /\b(trend|trends|over time|historical|history|growth|decline|monthly|yearly|weekly|daily|last (year|month|week)|timeline|forecast)\b/.test(q);
+
+    const isRanking = /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank(?:ing)?|best|worst|leading)\b/.test(q);
+
+    const isComparison = /\b(compare|comparison|vs\.?|versus|against|differ|difference|contrast|top)\b/.test(q) || isRanking;
+
+    const isDistribution = /\b(distribution|spread|histogram|frequency|variance|range)\b/.test(q);
+
+    const wantsPieLikeChart = /\b(pie|donut|doughnut)(?:\s+chart)?\b/.test(q);
+
+    const isComposition = wantsPieLikeChart || /\b(share|percentage|percent|breakup|breakdown|composition|split|segmentation|by category|by type|proportion)\b/.test(q);
+
+    const isCorrelation = /\b(relation|relationship|correlation|correlate|scatter|association|impact of|depend(?:s|ence)? on)\b/.test(q);
+
+    const isKpi = /\b(total|average|avg|count|sum|median|minimum|maximum|metric|kpi|how many|number of)\b/.test(q);
+
+    const isGeographic = /\b(region|country|countries|state|city|location|map|geo|geographic|territory)\b/.test(q);
+
+    const filterInStockOnly = /\b(in[- ]?stock|available|availability|inventory|stock status)\b/.test(q);
+
+    const wantsExplicitTable = /\b(table|spreadsheet|grid|detailed records|record details|list all|compare all)\b/.test(q);
+
+    let visualizationHint: QueryIntent['visualizationHint'] = 'text';
+    if (wantsExplicitTable) visualizationHint = 'tabular';
+    else if (isTemporal) visualizationHint = 'trend';
+    else if (wantsPieLikeChart) visualizationHint = 'composition';
+    else if (isRanking) visualizationHint = 'ranking';
+    else if (isComparison) visualizationHint = 'comparison';
+    else if (isDistribution) visualizationHint = 'distribution';
+    else if (isComposition) visualizationHint = 'composition';
+    else if (isCorrelation) visualizationHint = 'correlation';
+    else if (isGeographic) visualizationHint = 'geographic';
+    else if (isKpi) visualizationHint = 'kpi';
+    else if (this.isProductQuery(query)) visualizationHint = 'product_browse';
+
+    return {
+      visualizationHint,
+      recommendedChart: this.getRecommendedChartForIntent(visualizationHint),
+      filterInStockOnly,
+      wantsExplicitTable,
+      isTemporal,
+      isComparison,
+      language: 'en', // heuristic cannot reliably detect language
+    };
+  }
+
+  private static getRecommendedChartForIntent(
+    intent: QueryIntent['visualizationHint'],
+  ): QueryIntent['recommendedChart'] {
+    switch (intent) {
+      case 'trend':
+        return 'line_chart';
+      case 'comparison':
+        return 'bar_chart';
+      case 'distribution':
+        return 'histogram';
+      case 'composition':
+      case 'category_breakdown':
+        return 'pie_chart';
+      case 'correlation':
+        return 'scatter_plot';
+      case 'ranking':
+        return 'horizontal_bar';
+      case 'kpi':
+        return 'metric_card';
+      case 'tabular':
+      case 'table':
+        return 'table';
+      case 'geographic':
+        return 'geo_map';
+      case 'product_browse':
+      case 'text':
+      default:
+        return 'text';
+    }
+  }
+
+  // ─── Transform Helpers ────────────────────────────────────────────────────
+
   private static transformToProductCarousel(
     data: VectorMatch[],
     config?: import('../config/RagConfig').RagConfig,
-    trainedSchema?: import('./SchemaMapper').SchemaMap
+    trainedSchema?: import('./SchemaMapper').SchemaMap,
   ): UITransformationResponse {
     const products: CarouselProduct[] = data
-      .filter(item => this.isProductData(item))
       .map(item => this.extractProductInfo(item, config, trainedSchema))
       .filter((p): p is CarouselProduct => p !== null);
 
@@ -86,67 +493,214 @@ export class UITransformer {
     };
   }
 
-  /**
-   * Transform data to pie chart format
-   */
   private static transformToPieChart(
     data: VectorMatch[],
-    groupByField: string
-  ): UITransformationResponse {
-    let categories = this.detectCategories(data, groupByField);
-    if (categories.length === 0) categories = ['All Data'];
-    
-    const categoryData = this.aggregateByCategory(data, categories, groupByField);
+    profile?: DataProfile,
+    query = '',
+  ): UITransformationResponse | null {
+    const dimension = profile ? this.selectDimensionField(profile, query) : undefined;
+    const categories = dimension
+      ? Array.from(new Set(profile!.records.map(record => String(record.fields[dimension.key] ?? '')).filter(Boolean)))
+      : this.detectCategories(data);
+    if (categories.length === 0) return null;
+
+    const categoryData = dimension && profile
+      ? this.aggregateProfileByDimension(profile, dimension.key)
+      : this.aggregateByCategory(data, categories);
 
     const pieData: PieChartData[] = Object.entries(categoryData).map(([label, count]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data, groupByField);
-      return {
-        label,
-        value: count as number,
-        inStockCount,
-        outOfStockCount,
-      };
+      const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
+      return { label, value: count as number, inStockCount, outOfStockCount };
     });
 
     return {
       type: 'pie_chart',
-      title: `Distribution by ${groupByField.charAt(0).toUpperCase() + groupByField.slice(1)}`,
-      description: `Showing breakdown across ${categories.length} categories`,
+      title: `Distribution by ${dimension?.label ?? 'Category'}`,
+      description: `Showing breakdown across ${pieData.length} categories`,
       data: pieData,
     };
   }
 
-  /**
-   * Transform data to line chart format
-   */
-  private static transformToLineChart(data: VectorMatch[]): UITransformationResponse {
-    const timePoints = this.extractTimeSeriesData(data);
+  private static transformToLineChart(profile: DataProfile): UITransformationResponse {
+    const dateField = profile.dateFields[0];
+    const valueField = profile.numericFields[0];
+    const buckets = new Map<string, number>();
 
-    const lineData: LineChartDataPoint[] = timePoints.map(point => ({
-      timestamp: point.timestamp,
-      value: point.value,
-      label: point.label,
-    }));
+    profile.records.forEach(record => {
+      const timestamp = String(record.fields[dateField.key] ?? '');
+      const value = this.toFiniteNumber(record.fields[valueField.key]) ?? 0;
+      buckets.set(timestamp, (buckets.get(timestamp) ?? 0) + value);
+    });
+
+    const lineData: LineChartDataPoint[] = Array.from(buckets.entries())
+      .sort(([a], [b]) => Date.parse(a) - Date.parse(b))
+      .slice(0, 24)
+      .map(([timestamp, value]) => ({ timestamp, value, label: timestamp }));
 
     return {
       type: 'line_chart',
-      title: 'Trend Over Time',
+      title: `${valueField.label} Over Time`,
       description: `Showing ${lineData.length} data points`,
       data: lineData,
     };
   }
 
-  /**
-   * Transform data to table format
-   */
-  private static transformToTable(data: VectorMatch[]): UITransformationResponse {
-    const columns = this.extractTableColumns(data);
-    const rows = data.map(item => this.extractTableRow(item, columns));
+  private static transformToBarChart(
+    data: VectorMatch[],
+    profile?: DataProfile,
+    query = '',
+    horizontal = false,
+  ): UITransformationResponse {
+    const dimension = profile ? this.selectDimensionField(profile, query) : undefined;
+    const measure = profile ? this.selectNumericField(profile, query) : undefined;
+    const aggregate = dimension && profile
+      ? this.aggregateProfileByDimension(profile, dimension.key, measure?.key)
+      : this.aggregateByCategory(data, this.detectCategories(data));
+
+    const barData: BarChartData[] = Object.entries(aggregate)
+      .map(([category, value]) => ({ category, value: Number(value) }))
+      .sort((a, b) => horizontal ? b.value - a.value : 0)
+      .slice(0, 12);
+
+    const fallbackData: BarChartData[] = barData.length > 0
+      ? barData
+      : data.slice(0, 12).map((item, index) => {
+          const meta = item.metadata || {};
+          const label = String(
+            this.getDynamicVal(meta, 'name') ??
+            meta.label ??
+            meta.title ??
+            `Result ${index + 1}`,
+          );
+          const value = this.extractNumericValue(meta) ?? item.score ?? 0;
+          return { category: label, value: Number(value) };
+        });
+
+    return {
+      type: horizontal ? 'horizontal_bar' : 'bar_chart',
+      title: dimension ? `${measure?.label ?? 'Count'} by ${dimension.label}` : 'Comparison',
+      description: `Showing ${fallbackData.length} comparable values`,
+      data: fallbackData,
+    };
+  }
+
+  private static transformToHistogram(profile: DataProfile, query = ''): UITransformationResponse | null {
+    const field = this.selectNumericField(profile, query);
+    if (!field) return null;
+
+    const values = profile.records
+      .map(record => this.toFiniteNumber(record.fields[field.key]))
+      .filter((value): value is number => value !== null)
+      .sort((a, b) => a - b);
+    if (values.length === 0) return null;
+
+    const min = values[0];
+    const max = values[values.length - 1];
+    const bucketCount = Math.min(10, Math.max(3, Math.ceil(Math.sqrt(values.length))));
+    const width = max === min ? 1 : (max - min) / bucketCount;
+    const buckets = Array.from({ length: bucketCount }, (_, index) => {
+      const start = min + index * width;
+      const end = index === bucketCount - 1 ? max : start + width;
+      return { category: `${this.formatNumber(start)}-${this.formatNumber(end)}`, value: 0 };
+    });
+
+    values.forEach(value => {
+      const bucketIndex = max === min ? 0 : Math.min(bucketCount - 1, Math.floor((value - min) / width));
+      buckets[bucketIndex].value += 1;
+    });
+
+    return {
+      type: 'histogram',
+      title: `${field.label} Distribution`,
+      description: `Showing ${values.length} values across ${bucketCount} buckets`,
+      data: buckets,
+    };
+  }
+
+  private static transformToScatterPlot(profile: DataProfile, query = ''): UITransformationResponse | null {
+    const fields = this.rankFieldsByQuery(profile.numericFields, query).slice(0, 2);
+    if (fields.length < 2) return null;
+    const [xField, yField] = fields;
+
+    const points = profile.records
+      .map((record): ScatterPlotDataPoint | null => {
+        const x = this.toFiniteNumber(record.fields[xField.key]);
+        const y = this.toFiniteNumber(record.fields[yField.key]);
+        if (x === null || y === null) return null;
+        return { x, y, label: String(this.getRecordLabel(record) ?? record.id) };
+      })
+      .filter((point): point is ScatterPlotDataPoint => point !== null)
+      .slice(0, 100);
+
+    if (points.length === 0) return null;
+
+    return {
+      type: 'scatter_plot',
+      title: `${xField.label} vs ${yField.label}`,
+      description: `Showing ${points.length} paired values`,
+      data: points,
+    };
+  }
+
+  private static transformToMetricCard(profile: DataProfile, query = ''): UITransformationResponse | null {
+    const operation = this.detectAggregationOperation(query);
+    const numericField = this.selectNumericField(profile, query);
+    const values = numericField
+      ? profile.records
+          .map(record => this.toFiniteNumber(record.fields[numericField.key]))
+          .filter((value): value is number => value !== null)
+      : [];
+
+    const value = operation === 'count' || values.length === 0
+      ? profile.records.length
+      : this.calculateAggregate(values, operation);
+
+    const metric: MetricCardData = {
+      label: numericField ? `${operation} ${numericField.label}` : 'Count',
+      value,
+      operation,
+    };
+
+    return {
+      type: 'metric_card',
+      title: metric.label,
+      description: `Calculated from ${profile.records.length} result${profile.records.length === 1 ? '' : 's'}`,
+      data: metric,
+    };
+  }
+
+  private static transformToRadarChart(data: VectorMatch[]): UITransformationResponse {
+    // Aggregate attribute scores across items for radar comparison
+    const attributeMap: Record<string, Record<string, number>> = {};
+
+    data.forEach(item => {
+      const meta = item.metadata || {};
+      const seriesName = String(meta.name ?? meta.product ?? item.id ?? 'Item');
+      Object.entries(meta).forEach(([key, val]) => {
+        if (typeof val === 'number' && !['price', 'id'].includes(key.toLowerCase())) {
+          if (!attributeMap[key]) attributeMap[key] = {};
+          attributeMap[key][seriesName] = val;
+        }
+      });
+    });
 
-    const tableData: TableData = {
-      columns,
-      rows,
+    const radarData = Object.entries(attributeMap).map(([attribute, series]) => ({
+      attribute,
+      ...series,
+    }));
+
+    return {
+      type: 'radar_chart' as VisualizationType,
+      title: 'Product Comparison',
+      description: `Comparing ${data.length} items across ${radarData.length} attributes`,
+      data: radarData.length > 0 ? radarData : data.map(d => ({ attribute: d.content.substring(0, 40) })),
     };
+  }
+
+  private static transformToTable(data: VectorMatch[], query = ''): UITransformationResponse {
+    const columns = this.extractTableColumns(data, query);
+    const rows = data.map(item => this.extractTableRow(item, columns));
+    const tableData: TableData = { columns, rows };
 
     return {
       type: 'table',
@@ -156,37 +710,24 @@ export class UITransformer {
     };
   }
 
-  /**
-   * Transform data to text format (fallback)
-   */
   private static transformToText(data: VectorMatch[]): UITransformationResponse {
-    const textContent = data
-      .map(item => item.content)
-      .join('\n\n');
-
     return this.createTextResponse(
       'Search Results',
-      textContent,
-      `Found ${data.length} relevant results`
+      data.map(item => item.content).join('\n\n'),
+      `Found ${data.length} relevant results`,
     );
   }
 
-  /**
-   * Helper: Create text response
-   */
   private static createTextResponse(
     title: string,
     content: string,
-    description?: string
+    description?: string,
   ): UITransformationResponse {
-    return {
-      type: 'text',
-      title,
-      description,
-      data: { content },
-    };
+    return { type: 'text', title, description, data: { content } };
   }
 
+  // ─── LLM Response Parsing ─────────────────────────────────────────────────
+
   static parseTransformationResponse(raw: string): UITransformationResponse | null {
     const payloadText = this.extractJsonCandidate(raw);
     if (!payloadText) return null;
@@ -216,26 +757,14 @@ export class UITransformer {
 
     for (let i = start; i < cleaned.length; i += 1) {
       const char = cleaned[i];
-      if (escape) {
-        escape = false;
-        continue;
-      }
-      if (char === '\\') {
-        escape = true;
-        continue;
-      }
-      if (char === '"') {
-        inString = !inString;
-        continue;
-      }
+      if (escape) { escape = false; continue; }
+      if (char === '\\') { escape = true; continue; }
+      if (char === '"') { inString = !inString; continue; }
       if (inString) continue;
-
       if (char === '{') depth += 1;
       if (char === '}') {
         depth -= 1;
-        if (depth === 0) {
-          return cleaned.slice(start, i + 1);
-        }
+        if (depth === 0) return cleaned.slice(start, i + 1);
       }
     }
 
@@ -245,25 +774,20 @@ export class UITransformer {
   private static normalizeTransformation(payload: unknown): UITransformationResponse | null {
     if (!payload || typeof payload !== 'object') return null;
 
-    const payloadObj = payload as Record<string, unknown>;
-    const type = this.normalizeVisualizationType(String(payloadObj.type ?? payloadObj.view ?? payloadObj.chartType ?? ''));
+    const p = payload as Record<string, unknown>;
+    const type = this.normalizeVisualizationType(
+      String(p.type ?? p.view ?? p.chartType ?? ''),
+    );
     if (!type) return null;
 
-    const title = String(payloadObj.title ?? payloadObj.heading ?? 'Visualization');
-    const description = payloadObj.description ? String(payloadObj.description) : undefined;
-    const rawData = payloadObj.data ?? payloadObj.table ?? payloadObj.rows ?? payloadObj.items ?? payloadObj.content ?? null;
-
+    const title = String(p.title ?? p.heading ?? 'Visualization');
+    const description = p.description ? String(p.description) : undefined;
+    const rawData = p.data ?? p.table ?? p.rows ?? p.items ?? p.content ?? null;
     const data = type === 'text' && typeof rawData === 'string'
       ? { content: rawData }
       : rawData;
 
-    const transformation: UITransformationResponse = {
-      type,
-      title,
-      description,
-      data,
-    };
-
+    const transformation: UITransformationResponse = { type, title, description, data };
     return this.validateTransformation(transformation) ? transformation : null;
   }
 
@@ -273,571 +797,860 @@ export class UITransformer {
       pie_chart: 'pie_chart',
       bar: 'bar_chart',
       bar_chart: 'bar_chart',
+      histogram: 'histogram',
+      horizontal_bar: 'horizontal_bar',
+      horizontalbar: 'horizontal_bar',
       line: 'line_chart',
       line_chart: 'line_chart',
+      scatter: 'scatter_plot',
+      scatter_plot: 'scatter_plot',
+      scatterplot: 'scatter_plot',
       radar: 'radar_chart',
       radar_chart: 'radar_chart',
+      metric: 'metric_card',
+      metric_card: 'metric_card',
+      card: 'metric_card',
+      kpi: 'metric_card',
+      geo: 'geo_map',
+      geo_map: 'geo_map',
+      map: 'geo_map',
       table: 'table',
       text: 'text',
       product_carousel: 'product_carousel',
       carousel: 'carousel',
     };
-
     return mapping[type.toLowerCase()] ?? null;
   }
 
-  private static validateTransformation(transformation: UITransformationResponse): boolean {
-    const { type, data } = transformation;
-
+  private static validateTransformation(t: UITransformationResponse): boolean {
+    const { type, data } = t;
     switch (type) {
       case 'pie_chart':
       case 'bar_chart':
-        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
-          item !== null && typeof item === 'object' &&
-          (typeof item.value === 'number' || !Number.isNaN(Number(item.value)))
+      case 'histogram':
+      case 'horizontal_bar':
+        return Array.isArray(data) && data.every(
+          (i): i is Record<string, unknown> =>
+            i !== null && typeof i === 'object' &&
+            (typeof i.value === 'number' || !Number.isNaN(Number(i.value))),
+        );
+      case 'scatter_plot':
+        return Array.isArray(data) && data.every(
+          (i): i is Record<string, unknown> =>
+            i !== null && typeof i === 'object' &&
+            (typeof i.x === 'number' || !Number.isNaN(Number(i.x))) &&
+            (typeof i.y === 'number' || !Number.isNaN(Number(i.y))),
         );
       case 'line_chart':
-        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
-          item !== null && typeof item === 'object' &&
-          'timestamp' in item &&
-          (typeof item.value === 'number' || !Number.isNaN(Number(item.value)))
+        return Array.isArray(data) && data.every(
+          (i): i is Record<string, unknown> =>
+            i !== null && typeof i === 'object' && 'timestamp' in i &&
+            (typeof i.value === 'number' || !Number.isNaN(Number(i.value))),
+        );
+      case 'metric_card':
+        return (
+          typeof data === 'object' && data !== null &&
+          (typeof (data as Record<string, unknown>).value === 'number' ||
+            !Number.isNaN(Number((data as Record<string, unknown>).value)))
         );
+      case 'geo_map':
+        return Array.isArray(data) || (typeof data === 'object' && data !== null);
       case 'radar_chart':
-        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
-          item !== null && typeof item === 'object' &&
-          'attribute' in item
+        return Array.isArray(data) && data.every(
+          (i): i is Record<string, unknown> =>
+            i !== null && typeof i === 'object' && 'attribute' in i,
         );
       case 'table':
         return (
-          typeof data === 'object' &&
-          data !== null &&
+          typeof data === 'object' && data !== null &&
           Array.isArray((data as Record<string, unknown>).columns) &&
           Array.isArray((data as Record<string, unknown>).rows)
         );
       case 'text':
         return (
-          typeof data === 'object' &&
-          data !== null &&
+          typeof data === 'object' && data !== null &&
           typeof (data as Record<string, unknown>).content === 'string'
         );
       case 'product_carousel':
       case 'carousel':
-        return Array.isArray(data) && data.every((item): item is Record<string, unknown> =>
-          item !== null && typeof item === 'object' && (('id' in item) || ('name' in item))
+        return Array.isArray(data) && data.every(
+          (i): i is Record<string, unknown> =>
+            i !== null && typeof i === 'object' && ('id' in i || 'name' in i),
         );
       default:
         return false;
     }
   }
 
-  /**
-   * Helper: Check if data item is product-related
-   */
+  // ─── Data Inspection Helpers ──────────────────────────────────────────────
+
+  private static isStructuredListQuery(query: string): boolean {
+    const q = query.toLowerCase();
+    const asksForRecords = /\b(list|provide|show|give|get|find|which|whose|records?|details?)\b/.test(q);
+    const hasNumericPredicate = /\b(greater than|more than|above|over|less than|below|under|at least|at most|minimum|maximum|>=|<=|>|<|equals?|equal to)\b\s*\$?\d/i.test(q) ||
+      /\b\d+\b\s*(?:or more|or less|and above|and below)\b/i.test(q);
+    const hasEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
+    return asksForRecords && (hasNumericPredicate || hasEntityTerms);
+  }
+
+  private static isProductQuery(query: string): boolean {
+    const q = query.toLowerCase();
+    const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
+    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
+    const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
+    return productTerms && productAction && !nonProductEntityTerms;
+  }
+
   private static isProductData(item: VectorMatch): boolean {
     const content = (item.content || '').toLowerCase();
     const productKeywords = [
-      'product', 'price', 'stock', 'item', 'sku', 'brand', 'model', 'msrp', 'inventory', 'buy', 'shop',
-      'beauty', 'care', 'cosmetic', 'facial', 'cream', 'serum', 'mask', 'makeup', 'fragrance'
+      'product', 'price', 'stock', 'item', 'sku', 'brand', 'model', 'msrp',
+      'inventory', 'buy', 'shop', 'beauty', 'care', 'cosmetic', 'facial',
+      'cream', 'serum', 'mask', 'makeup', 'fragrance',
     ];
-
-    // Check content for keywords
     const hasKeywords = productKeywords.some(kw => content.includes(kw));
 
-    // Check metadata keys
-    const hasMetadataKey = Object.keys(item.metadata || {}).some(k =>
-      ['name', 'price', 'product', 'sku', 'brand', 'model', 'cost', 'item'].includes(k.toLowerCase())
-    );
-
-    // Failsafe: if we have a dollar sign followed by a number, it's likely a product
-    const hasPricePattern = /\$\s*\d+/.test(content);
+    const metadata = item.metadata || {};
+    const hasMetadataKey = Object.keys(metadata).some(k => {
+      const val = metadata[k];
+      return (
+        ['price', 'product', 'sku', 'brand', 'model', 'cost', 'item'].includes(k.toLowerCase()) &&
+        val !== null && val !== undefined && val !== ''
+      );
+    });
 
+    const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
     return hasKeywords || hasMetadataKey || hasPricePattern;
   }
 
-  /**
-   * Helper: Check if data contains time series
-   */
   private static isTimeSeriesData(item: VectorMatch): boolean {
-    const content = (item.content || '').toLowerCase();
-    const timeKeywords = ['trend', 'historical', 'growth', 'decline', 'change', 'increase', 'decrease'];
-    const hasTimeKeyword = timeKeywords.some(kw => content.includes(kw));
-
     const metadata = item.metadata || {};
-    const maybeDateKeys = Object.keys(metadata).filter((k) =>
-      ['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase())
+    const maybeDateKeys = Object.keys(metadata).filter(k =>
+      ['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase()),
     );
-    const hasValidDateValue = maybeDateKeys.some((key) => {
+    return maybeDateKeys.some(key => {
       const value = metadata[key];
-      if (typeof value === 'string') {
-        return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
-      }
+      if (typeof value === 'string') return !Number.isNaN(Date.parse(value)) && value.trim().length > 0;
       return typeof value === 'number';
     });
+  }
 
-    return hasTimeKeyword || hasValidDateValue;
-  }
-
-  private static isExplicitChartQuery(query: string): boolean {
-    const normalized = query.toLowerCase();
-    return normalized.includes('pie chart') || 
-           normalized.includes('piechart') || 
-           normalized.includes('bar chart') || 
-           normalized.includes('barchart') ||
-           normalized.includes('radar chart');
-  }
-
-  private static shouldShowCategoryChart(query: string): boolean {
-    const normalized = query.toLowerCase();
-    const chartKeywords = [
-      'distribution',
-      'breakdown',
-      'by category',
-      'by type',
-      'compare',
-      'share',
-      'percentage',
-      'segmentation',
-      'split',
-      'category breakdown',
-      'category distribution',
-      'pie chart',
-      'piechart',
-      'bar chart',
-      'barchart'
-    ];
+  private static hasMultipleFields(data: VectorMatch[]): boolean {
+    const fieldCount = new Set<string>();
+    data.forEach(item => Object.keys(item.metadata || {}).forEach(k => fieldCount.add(k)));
+    return fieldCount.size > 2;
+  }
 
-    return chartKeywords.some(keyword => normalized.includes(keyword));
-  }
-
-  private static isTrendQuery(query: string): boolean {
-    const normalized = query.toLowerCase();
-    const trendKeywords = [
-      'trend',
-      'over time',
-      'historical',
-      'growth',
-      'decline',
-      'increase',
-      'decrease',
-      'year',
-      'month',
-      'week',
-      'day',
-      'comparison',
-      'compare',
-      'changes',
-      'timeline',
-    ];
+  private static profileData(data: VectorMatch[]): DataProfile {
+    const records = data.map((item): DataRecord => {
+      const fields: Record<string, PrimitiveValue> = {};
+      Object.entries(item.metadata || {}).forEach(([key, value]) => {
+        const primitive = this.toPrimitive(value);
+        if (primitive !== null) fields[key] = primitive;
+      });
+      if (!fields.content && item.content) fields.content = item.content.substring(0, 500);
+      return {
+        id: item.id,
+        content: item.content,
+        score: item.score,
+        fields,
+        source: item,
+      };
+    });
 
-    return trendKeywords.some(keyword => normalized.includes(keyword));
-  }
+    const keys = Array.from(new Set(records.flatMap(record => Object.keys(record.fields))));
+    const fields = keys.map((key): FieldProfile => {
+      const values = records
+        .map(record => record.fields[key])
+        .filter((value): value is PrimitiveValue => value !== undefined && value !== null && String(value).trim() !== '');
+      const uniqueCount = new Set(values.map(value => String(value).toLowerCase())).size;
+      return {
+        key,
+        label: this.humanizeFieldName(key),
+        kind: this.inferFieldKind(key, values, records.length, uniqueCount),
+        values,
+        uniqueCount,
+      };
+    });
 
-  private static isStockQuery(query: string): boolean {
-    const normalized = query.toLowerCase();
-    return (
-      normalized.includes('in stock') ||
-      normalized.includes('available') ||
-      normalized.includes('availability') ||
-      normalized.includes('inventory') ||
-      normalized.includes('stock status')
-    );
+    return {
+      records,
+      fields,
+      numericFields: fields.filter(field => field.kind === 'number'),
+      dateFields: fields.filter(field => field.kind === 'date'),
+      categoricalFields: fields.filter(field => field.kind === 'category'),
+      booleanFields: fields.filter(field => field.kind === 'boolean'),
+    };
   }
 
-  /**
-   * Helper: Extract property from metadata using mapping, AI training, case-insensitivity, and synonyms.
-   */
-  private static getDynamicVal(
-    meta: Record<string, unknown> | undefined,
-    uiKey: string,
-    config?: import('../config/RagConfig').RagConfig,
-    trainedSchema?: import('./SchemaMapper').SchemaMap
-  ): unknown {
-    if (!meta) return undefined;
+  private static toPrimitive(value: unknown): PrimitiveValue | null {
+    if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
+    if (value === null || value === undefined) return null;
+    if (value instanceof Date) return value.toISOString();
+    return null;
+  }
 
-    const mapping = config?.rag?.uiMapping;
-    if (mapping && mapping[uiKey]) {
-      const mappedKey = mapping[uiKey];
-      if (meta[mappedKey] !== undefined) return meta[mappedKey];
+  private static inferFieldKind(
+    key: string,
+    values: PrimitiveValue[],
+    rowCount: number,
+    uniqueCount: number,
+  ): FieldProfile['kind'] {
+    const lowerKey = key.toLowerCase();
+    if (values.length === 0) return 'text';
+    if (values.every(value => typeof value === 'boolean' || /^(true|false|yes|no|in_stock|out_of_stock|available|unavailable)$/i.test(String(value)))) {
+      return 'boolean';
     }
-
-    if (trainedSchema && typeof trainedSchema === 'object' && trainedSchema !== null) {
-      const trainedKey = (trainedSchema as Record<string, unknown>)[uiKey];
-      if (trainedKey && meta[trainedKey as string] !== undefined) return meta[trainedKey as string];
+    if (/(date|time|timestamp|created|updated|month|year|period)/i.test(lowerKey) && values.some(value => this.isDateLike(value))) {
+      return 'date';
+    }
+    const numericCount = values.filter(value => this.toFiniteNumber(value) !== null).length;
+    if (numericCount / values.length >= 0.85 && !/(id|sku|ean|upc|zip|postal|phone|code)/i.test(lowerKey)) {
+      return 'number';
     }
+    if (
+      uniqueCount <= Math.max(20, Math.ceil(rowCount * 0.7)) &&
+      !/(id|sku|ean|upc|description|content|url|image|thumbnail)/i.test(lowerKey)
+    ) {
+      return 'category';
+    }
+    return 'text';
+  }
 
-    return resolveMetadataValue(meta, uiKey);
+  private static isDateLike(value: PrimitiveValue): boolean {
+    if (typeof value === 'number') return value > 1900 && value < 3000;
+    const text = String(value).trim();
+    return text.length > 3 && !Number.isNaN(Date.parse(text));
   }
 
-  private static extractProductInfo(
-    item: VectorMatch,
-    config?: import('../config/RagConfig').RagConfig,
-    trainedSchema?: import('./SchemaMapper').SchemaMap
-  ): CarouselProduct | null {
-    const meta = item.metadata || {};
+  private static chooseAutomaticVisualization(
+    data: VectorMatch[],
+    profile: DataProfile,
+    query: string,
+  ): UITransformationResponse | null {
+    if (profile.records.length === 0) return null;
+    if (profile.dateFields.length > 0 && profile.numericFields.length > 0) {
+      return this.transformToLineChart(profile);
+    }
+    if (profile.categoricalFields.length > 0 && profile.numericFields.length > 0) {
+      return this.transformToBarChart(data, profile, query);
+    }
+    if (profile.categoricalFields.length > 0) {
+      return this.transformToPieChart(data, profile, query);
+    }
+    if (profile.numericFields.length >= 2) {
+      return this.transformToScatterPlot(profile, query);
+    }
+    if (profile.numericFields.length === 1) {
+      return this.transformToMetricCard(profile, query);
+    }
+    return null;
+  }
 
-    const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
-    const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
-    const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
+  private static selectDimensionField(profile: DataProfile, query: string): FieldProfile | undefined {
+    const productCategory = profile.categoricalFields.find(field =>
+      /category|department|collection|type|group|segment|region|country|state|city/i.test(field.key),
+    );
+    const ranked = this.rankFieldsByQuery(profile.categoricalFields, query);
+    return ranked[0] ?? productCategory ?? profile.categoricalFields[0];
+  }
 
-    // If we have at least a name or a clear product-like structure in content
-    if (name || this.isProductData(item)) {
-      // Robust name extraction: metadata > regex > content snippet
-      let finalName = name ? String(name) : undefined;
-      if (!finalName) {
-        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
-        finalName = nameMatch ? nameMatch[1].trim() : item.content.split('\n')[0].substring(0, 60);
-      }
+  private static selectNumericField(profile: DataProfile, query: string): FieldProfile | undefined {
+    return this.rankFieldsByQuery(profile.numericFields, query)[0] ?? profile.numericFields[0];
+  }
 
-      // Robust price extraction: metadata > regex
-      let finalPrice: string | number | undefined =
-        typeof price === 'number' || typeof price === 'string'
-          ? price
-          : undefined;
+  private static rankFieldsByQuery(fields: FieldProfile[], query: string): FieldProfile[] {
+    const q = query.toLowerCase();
+    return [...fields].sort((a, b) => this.fieldScore(b, q) - this.fieldScore(a, q));
+  }
 
-      if (!finalPrice) {
-        const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) || item.content.match(/\$\s*([\d,.]+)/);
-        if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
-      }
+  private static fieldScore(field: FieldProfile, query: string): number {
+    const key = field.key.toLowerCase();
+    const label = field.label.toLowerCase();
+    let score = 0;
+    if (query.includes(key)) score += 4;
+    if (query.includes(label)) score += 4;
+    key.split(/[_\s-]+/).forEach(part => {
+      if (part && query.includes(part)) score += 1;
+    });
+    if (/category|department|collection|type|group|segment/.test(key)) score += 2;
+    if (/count|total|quantity|stock|inventory|sales|revenue|amount|price|value|score/.test(key)) score += 2;
+    if (/id|sku|ean|upc|code/.test(key)) score -= 5;
+    return score;
+  }
 
-      const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
+  private static normalizeComparableField(field: string): string {
+    return field
+      .toLowerCase()
+      .replace(/&/g, 'and')
+      .replace(/[^a-z0-9]+/g, '')
+      .trim();
+  }
 
-      return {
-        id: item.id,
-        name: finalName,
-        price: finalPrice,
-        image: typeof imageValue === 'string' ? imageValue : undefined,
-        brand: brand ? String(brand) : undefined,
-        description: item.content,
-        inStock: this.determineStockStatus(item),
-      };
-    }
+  private static fieldTokens(field: string): string[] {
+    return field
+      .toLowerCase()
+      .replace(/[_-]+/g, ' ')
+      .split(/\s+/)
+      .map(token => token.replace(/[^a-z0-9]/g, ''))
+      .filter(Boolean);
+  }
 
-    return null;
+  private static aggregateProfileByDimension(
+    profile: DataProfile,
+    dimensionKey: string,
+    measureKey?: string,
+  ): Record<string, number> {
+    const result: Record<string, number> = {};
+    profile.records.forEach(record => {
+      const category = String(record.fields[dimensionKey] ?? 'Other').trim() || 'Other';
+      const value = measureKey ? this.toFiniteNumber(record.fields[measureKey]) ?? 0 : 1;
+      result[category] = (result[category] ?? 0) + value;
+    });
+    return result;
   }
 
-  /**
-   * Helper: Dynamically determine what field to group by based on user query
-   */
-  private static determineGroupByField(query: string, data: VectorMatch[]): string {
-    const normalized = query.toLowerCase();
-    const match = normalized.match(/(?:by|across|per|based on)\s+([a-zA-Z]+)/i);
-    
-    let targetField = 'category';
-    if (match && match[1]) {
-      let field = match[1].toLowerCase();
-      // Un-pluralize common words to match schema keys
-      if (field.endsWith('ies')) field = field.replace(/ies$/, 'y');
-      else if (field.endsWith('s') && field !== 'status') field = field.slice(0, -1);
-      
-      const stopWords = ['the', 'all', 'different', 'various', 'some', 'these', 'those'];
-      if (!stopWords.includes(field)) {
-         targetField = field;
+  private static getRecordLabel(record: DataRecord): PrimitiveValue | undefined {
+    const labelKeys = ['name', 'title', 'label', 'product', 'item', 'brand'];
+    const key = Object.keys(record.fields).find(fieldKey =>
+      labelKeys.some(labelKey => fieldKey.toLowerCase().includes(labelKey)),
+    );
+    return key ? record.fields[key] : undefined;
+  }
+
+  private static detectAggregationOperation(query: string): AggregationOperation {
+    const q = query.toLowerCase();
+    if (/\b(avg|average|mean)\b/.test(q)) return 'average';
+    if (/\b(count|how many|number of)\b/.test(q)) return 'count';
+    if (/\b(min|minimum|lowest)\b/.test(q)) return 'min';
+    if (/\b(max|maximum|highest)\b/.test(q)) return 'max';
+    if (/\bmedian\b/.test(q)) return 'median';
+    return 'sum';
+  }
+
+  private static calculateAggregate(values: number[], operation: AggregationOperation): number {
+    if (values.length === 0) return 0;
+    switch (operation) {
+      case 'average':
+        return values.reduce((sum, value) => sum + value, 0) / values.length;
+      case 'count':
+        return values.length;
+      case 'min':
+        return Math.min(...values);
+      case 'max':
+        return Math.max(...values);
+      case 'median': {
+        const sorted = [...values].sort((a, b) => a - b);
+        const middle = Math.floor(sorted.length / 2);
+        return sorted.length % 2 === 0 ? (sorted[middle - 1] + sorted[middle]) / 2 : sorted[middle];
       }
+      case 'sum':
+      default:
+        return values.reduce((sum, value) => sum + value, 0);
     }
+  }
 
-    // Verify field exists, or fallback
-    const hasTargetField = data.some(d => d.metadata && d.metadata[targetField] !== undefined);
-    if (!hasTargetField && targetField === 'category') {
-       if (data.some(d => d.metadata && d.metadata['type'] !== undefined)) return 'type';
-       if (data.some(d => d.metadata && d.metadata['brand'] !== undefined)) return 'brand';
-    }
+  private static humanizeFieldName(key: string): string {
+    return key
+      .replace(/[_-]+/g, ' ')
+      .replace(/([a-z])([A-Z])/g, '$1 $2')
+      .replace(/\s+/g, ' ')
+      .trim()
+      .replace(/\b\w/g, char => char.toUpperCase());
+  }
 
-    return targetField;
+  private static formatNumber(value: number): string {
+    return Number.isInteger(value) ? String(value) : value.toFixed(1);
   }
 
-  /**
-   * Helper: Detect grouping values dynamically
-   */
-  private static detectCategories(data: VectorMatch[], groupByField: string): string[] {
+  private static detectCategories(data: VectorMatch[]): string[] {
     const categories = new Set<string>();
-
     data.forEach(item => {
-      const meta = item.metadata || {};
-
-      if (meta[groupByField] !== undefined) {
-        const val = meta[groupByField];
-        if (Array.isArray(val)) val.forEach(v => categories.add(String(v)));
-        else categories.add(String(val));
-      } else if (groupByField === 'category') {
-        // Fallbacks for legacy 'category' extraction
-        if (meta.type) categories.add(String(meta.type));
-        if (meta.tag) {
-          const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
-          tags.forEach(t => categories.add(String(t)));
-        }
-        const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
-        if (categoryMatch) categories.add(categoryMatch[1].trim());
-      }
+      const category = this.getProductCategory(item);
+      if (category) categories.add(category);
     });
-
     return Array.from(categories);
   }
 
-  /**
-   * Helper: Aggregate data dynamically
-   */
+  private static getProductCategory(item: VectorMatch): string | null {
+    const meta = item.metadata || {};
+    const metadataCategory = resolveMetadataValue(meta, 'category');
+    if (this.isUsableCategory(metadataCategory)) return String(metadataCategory).trim();
+
+    const content = item.content || '';
+    const explicitCategoryMatch = content.match(
+      /(?:^|\n)\s*(?:product\s*)?(?:category|department|collection)\s*[:\-–]\s*([^\n]+)/i,
+    );
+    if (explicitCategoryMatch && this.isUsableCategory(explicitCategoryMatch[1])) {
+      return explicitCategoryMatch[1].trim();
+    }
+
+    return null;
+  }
+
+  private static isUsableCategory(value: unknown): value is string | number {
+    if (value === null || value === undefined) return false;
+    const category = String(value).trim();
+    if (!category) return false;
+    return !/^(category|type|tag|price|cost|stock|availability|description|product|item|name|brand|sku|id|ean|currency|color|size|index|internal id|other|\[?type\]?|\[?products?_?\d*\]?)$/i.test(category);
+  }
+
   private static aggregateByCategory(
     data: VectorMatch[],
     categories: string[],
-    groupByField: string
   ): Record<string, number> {
-    const result: Record<string, number> = {};
-
-    categories.forEach(cat => {
-      result[cat] = 0;
-    });
-
+    const result: Record<string, number> = Object.fromEntries(categories.map(c => [c, 0]));
     data.forEach(item => {
-      const meta = item.metadata || {};
-      let itemCategory = 'Other';
-
-      if (meta[groupByField] !== undefined) {
-        itemCategory = String(meta[groupByField]);
-      } else if (groupByField === 'category' && meta.type) {
-        itemCategory = String(meta.type);
-      }
-
-      if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
-        result[itemCategory]++;
-      } else {
-        result['Other'] = (result['Other'] || 0) + 1;
-      }
+      const cat = this.getProductCategory(item) ?? 'Other';
+      if (Object.prototype.hasOwnProperty.call(result, cat)) result[cat]++;
+      else result['Other'] = (result['Other'] || 0) + 1;
     });
-
     return result;
   }
 
-  /**
-   * Helper: Extract time series data
-   */
   private static extractTimeSeriesData(
-    data: VectorMatch[]
+    data: VectorMatch[],
   ): Array<{ timestamp: string | number; value: number; label?: string }> {
     return data.map(item => {
       const meta = item.metadata || {};
       return {
-        timestamp: (meta.timestamp || meta.date || new Date().toISOString()) as string | number,
-        value: (meta.value || item.score || 0) as number,
-        label: (meta.label || item.content.substring(0, 50)) as string,
+        timestamp: (meta.timestamp ?? meta.date ?? new Date().toISOString()) as string | number,
+        value: (meta.value ?? item.score ?? 0) as number,
+        label: (meta.label ?? item.content.substring(0, 50)) as string,
       };
     });
   }
 
-  /**
-   * Helper: Extract table columns
-   */
-  private static extractTableColumns(data: VectorMatch[]): string[] {
-    const columnSet = new Set<string>();
-    columnSet.add('Content');
+  private static extractNumericValue(meta: Record<string, unknown>): number | null {
+    const preferredKeys = [
+      'value', 'count', 'total', 'average', 'avg', 'amount', 'sales',
+      'revenue', 'score', 'quantity', 'price',
+    ];
+
+    for (const key of preferredKeys) {
+      const raw = resolveMetadataValue(meta, key) ?? meta[key];
+      const value = typeof raw === 'number' ? raw : Number(String(raw ?? '').replace(/[$,% ,]/g, ''));
+      if (Number.isFinite(value)) return value;
+    }
+
+    for (const value of Object.values(meta)) {
+      const numeric = typeof value === 'number' ? value : Number(String(value ?? '').replace(/[$,% ,]/g, ''));
+      if (Number.isFinite(numeric)) return numeric;
+    }
+
+    return null;
+  }
+
+  private static extractTableColumns(data: VectorMatch[], query = ''): string[] {
+    const q = query.toLowerCase();
+    const availableFields = this.extractAvailableTableFields(data);
+
+    if (/\b(organization|organisations?|organizations?|company|companies)\b/.test(q) &&
+      /\b(employee|employees|staff|headcount|workforce)\b/.test(q)) {
+      const nameField = this.pickTableField(availableFields, [
+        'company name',
+        'organization name',
+        'organisation name',
+        'name',
+        'company',
+        'organization',
+        'organisation',
+      ]);
+      const employeeField = this.pickTableField(availableFields, [
+        'number of employees',
+        'employee count',
+        'employees',
+        'employee_count',
+        'number_employees',
+        'num employees',
+        'staff count',
+        'headcount',
+        'workforce',
+      ]);
+
+      const columns = [nameField, employeeField].filter((field): field is string => Boolean(field));
+      if (columns.length > 0) return columns;
+    }
+
+    const requestedFields = availableFields
+      .map(field => ({
+        field,
+        score: this.tableFieldQueryScore(field, q),
+      }))
+      .filter(item => item.score > 0)
+      .sort((a, b) => b.score - a.score)
+      .map(item => item.field);
+
+    if (requestedFields.length > 0) {
+      return requestedFields.slice(0, Math.min(6, requestedFields.length));
+    }
+
+    const metadataFields = Array.from(new Set(
+      data.flatMap(item => Object.keys(item.metadata || {}).map(k => this.humanizeFieldName(k))),
+    ));
+
+    return metadataFields.length > 0 ? metadataFields : ['Content'];
+  }
+
+  private static extractTableRow(
+    item: VectorMatch,
+    columns: string[],
+  ): (string | number | boolean)[] {
+    return columns.map(col => this.resolveTableCellValue(item, col));
+  }
+
+  private static extractAvailableTableFields(data: VectorMatch[]): string[] {
+    const fields = new Map<string, string>();
+    const addField = (field: string) => {
+      const clean = this.humanizeFieldName(field);
+      if (!clean || /^(content|\[?type\]?)$/i.test(clean)) return;
+      const normalized = this.normalizeComparableField(clean);
+      if (!fields.has(normalized)) fields.set(normalized, clean);
+    };
 
     data.forEach(item => {
-      Object.keys(item.metadata || {}).forEach(key => {
-        columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
-      });
+      Object.keys(item.metadata || {}).forEach(addField);
+      for (const match of (item.content || '').matchAll(/(?:^|\n)\s*([A-Za-z][A-Za-z0-9_ /-]{1,50})\s*[:\-–]\s*([^\n]+)/g)) {
+        addField(match[1]);
+      }
     });
 
-    return Array.from(columnSet);
+    return Array.from(fields.values());
   }
 
-  /**
-   * Helper: Extract table row
-   */
-  private static extractTableRow(item: VectorMatch, columns: string[]): (string | number | boolean)[] {
+  private static pickTableField(fields: string[], aliases: string[]): string | undefined {
+    const normalizedAliases = aliases.map(alias => this.normalizeComparableField(alias));
+    for (const alias of normalizedAliases) {
+      const exact = fields.find(field => this.normalizeComparableField(field) === alias);
+      if (exact) return exact;
+    }
+
+    for (const alias of normalizedAliases) {
+      const fuzzy = fields.find(field => {
+        const normalizedField = this.normalizeComparableField(field);
+        if (/(id|code|uuid)$/.test(normalizedField) && !/(id|code|uuid)$/.test(alias)) return false;
+        return normalizedField.includes(alias) || alias.includes(normalizedField);
+      });
+      if (fuzzy) return fuzzy;
+    }
+
+    return undefined;
+  }
+
+  private static tableFieldQueryScore(field: string, query: string): number {
+    const normalizedField = this.normalizeComparableField(field);
+    if (!normalizedField) return 0;
+    const fieldTokens = this.fieldTokens(field);
+    return fieldTokens.reduce((score, token) => {
+      if (token.length < 3) return score;
+      return query.includes(token) ? score + 1 : score;
+    }, query.includes(normalizedField) ? 3 : 0);
+  }
+
+  private static resolveTableCellValue(item: VectorMatch, column: string): string | number | boolean {
+    if (column === 'Content') return item.content.substring(0, 100);
+
     const meta = item.metadata || {};
-    return columns.map(col => {
-      if (col === 'Content') {
-        return item.content.substring(0, 100);
-      }
+    const normalizedColumn = this.normalizeComparableField(column);
+    const exactMetadata = Object.entries(meta).find(([key]) =>
+      this.normalizeComparableField(key) === normalizedColumn ||
+      this.normalizeComparableField(this.humanizeFieldName(key)) === normalizedColumn,
+    );
+    if (exactMetadata && exactMetadata[1] !== undefined && exactMetadata[1] !== null) {
+      return this.toDisplayValue(exactMetadata[1]);
+    }
 
-      const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
-      const value = meta[metaKey];
-      return value !== undefined ? String(value) : '';
-    });
+    const aliasValue = this.resolveAliasedTableCell(meta, column);
+    if (aliasValue !== undefined) return this.toDisplayValue(aliasValue);
+
+    const contentValue = this.extractContentFieldValue(item.content, column);
+    if (contentValue !== null) return contentValue;
+
+    const aliasContentValue = this.extractAliasedContentFieldValue(item.content, column);
+    return aliasContentValue ?? '';
   }
 
-  /**
-   * Helper: Calculate stock counts dynamically
-   */
-  private static calculateStockCounts(category: string, data: VectorMatch[], groupByField: string): { inStockCount: number; outOfStockCount: number } {
+  private static resolveAliasedTableCell(meta: Record<string, unknown>, column: string): unknown {
+    const normalizedColumn = this.normalizeComparableField(column);
+    const aliases = normalizedColumn.includes('employee')
+      ? ['number of employees', 'employee count', 'employees', 'headcount', 'staff count', 'workforce']
+      : normalizedColumn.includes('name')
+        ? ['company name', 'organization name', 'organisation name', 'name', 'company', 'organization', 'organisation']
+        : [];
+
+    for (const alias of aliases) {
+      const normalizedAlias = this.normalizeComparableField(alias);
+      const match = Object.entries(meta).find(([key]) => {
+        const normalizedKey = this.normalizeComparableField(key);
+        return normalizedKey === normalizedAlias ||
+          normalizedKey.includes(normalizedAlias) ||
+          normalizedAlias.includes(normalizedKey);
+      });
+      if (match && match[1] !== undefined && match[1] !== null) return match[1];
+    }
+
+    return undefined;
+  }
+
+  private static extractContentFieldValue(content: string, column: string): string | null {
+    const escaped = column.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '[\\s_ -]+');
+    const pattern = new RegExp(`(?:^|\\n)\\s*${escaped}\\s*[:\\-–]\\s*([^\\n]+)`, 'i');
+    const match = content.match(pattern);
+    return match ? match[1].trim() : null;
+  }
+
+  private static extractAliasedContentFieldValue(content: string, column: string): string | null {
+    const normalizedColumn = this.normalizeComparableField(column);
+    const aliases = normalizedColumn.includes('employee')
+      ? ['Number Of Employees', 'Number of Employees', 'Employee Count', 'Employees', 'Headcount', 'Staff Count', 'Workforce']
+      : normalizedColumn.includes('name')
+        ? ['Company Name', 'Organization Name', 'Organisation Name', 'Name', 'Company', 'Organization', 'Organisation']
+        : [];
+
+    for (const alias of aliases) {
+      const value = this.extractContentFieldValue(content, alias);
+      if (value !== null) return value;
+    }
+
+    return null;
+  }
+
+  private static toDisplayValue(value: unknown): string | number | boolean {
+    if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') return value;
+    return String(value ?? '');
+  }
+
+  private static calculateStockCounts(
+    category: string,
+    data: VectorMatch[],
+  ): { inStockCount: number; outOfStockCount: number } {
     let inStock = 0;
     let outOfStock = 0;
-
     data.forEach(d => {
-      const meta = d.metadata || {};
-      let itemCategory = 'Other';
-      if (meta[groupByField] !== undefined) {
-        itemCategory = String(meta[groupByField]);
-      } else if (groupByField === 'category' && meta.type) {
-        itemCategory = String(meta.type);
-      }
-
-      if (itemCategory === category) {
-        if (this.determineStockStatus(d)) {
-          inStock++;
-        } else {
-          outOfStock++;
-        }
+      const cat = this.getProductCategory(d) ?? 'Other';
+      if (cat === category) {
+        const quantity = this.extractStockQuantity(d) ?? 1;
+        if (this.determineStockStatus(d)) inStock += quantity;
+        else outOfStock += quantity;
       }
     });
-
     return { inStockCount: inStock, outOfStockCount: outOfStock };
   }
 
-  /**
-   * Helper: Determine if item is in stock
-   */
   private static determineStockStatus(item: VectorMatch): boolean {
     const meta = item.metadata || {};
-
-    // Check metadata for stock status
-    if (meta.inStock !== undefined) {
-      return Boolean(meta.inStock);
-    }
-    if (meta.stock !== undefined) {
-      return Boolean(meta.stock);
-    }
-    if (meta.available !== undefined) {
-      return Boolean(meta.available);
+    const stockValue = resolveMetadataValue(meta, 'stock');
+    if (stockValue !== undefined) {
+      const normalized = String(stockValue).toLowerCase();
+      if (/out[_\s-]?of[_\s-]?stock|unavailable|false|no|sold out|0/.test(normalized)) return false;
+      if (/in[_\s-]?stock|available|true|yes/.test(normalized)) return true;
+      const numeric = this.toFiniteNumber(stockValue);
+      if (numeric !== null) return numeric > 0;
     }
+    if (meta.inStock !== undefined) return Boolean(meta.inStock);
+    if (meta.stock !== undefined) return Boolean(meta.stock);
+    if (meta.available !== undefined) return Boolean(meta.available);
 
-    // Check content
     const content = (item.content || '').toLowerCase();
-    if (content.includes('out of stock') || content.includes('unavailable')) {
-      return false;
-    }
-    if (content.includes('in stock') || content.includes('available')) {
-      return true;
-    }
-
-    // Default to true if not specified
+    if (/out[_\s-]?of[_\s-]?stock|unavailable|sold out/.test(content)) return false;
+    if (/in[_\s-]?stock|available/.test(content)) return true;
     return true;
   }
 
-  /**
-   * Helper: Check if data has multiple fields
-   */
-  private static hasMultipleFields(data: VectorMatch[]): boolean {
-    const fieldCount = new Set<string>();
+  private static extractStockQuantity(item: VectorMatch): number | null {
+    const meta = item.metadata || {};
+    const stockValue = resolveMetadataValue(meta, 'stock');
+    const numericStock = this.toFiniteNumber(stockValue);
+    if (numericStock !== null) return numericStock;
 
-    data.forEach(item => {
-      Object.keys(item.metadata || {}).forEach(key => {
-        fieldCount.add(key);
-      });
-    });
+    const content = item.content || '';
+    const stockMatch = content.match(/\b(?:stock|inventory|quantity|count)\s*[:\-–]?\s*([\d,]+)/i);
+    if (!stockMatch) return null;
 
-    return fieldCount.size > 2;
+    return this.toFiniteNumber(stockMatch[1]);
   }
 
-  // ─── LLM-Driven Visualization Decision ────────────────────────────────────
+  private static toFiniteNumber(value: unknown): number | null {
+    if (typeof value === 'number') return Number.isFinite(value) ? value : null;
+    const numeric = Number(String(value ?? '').replace(/[$,% ,]/g, ''));
+    return Number.isFinite(numeric) ? numeric : null;
+  }
 
-  /**
-   * analyzeAndDecide — sends user question + RAG data to the LLM with a
-   * structured system prompt and parses the JSON response into a
-   * UITransformationResponse.
-   *
-   * This is the recommended entry point for production use. The heuristic
-   * `transform()` method is used as a fallback if the LLM call fails.
-   *
-   * System prompt instructs the LLM to:
-   *   - Analyze the question and retrieved data
-   *   - Choose the best visualization: bar_chart | line_chart | pie_chart | table | text
-   *   - Return a strict JSON object — no prose, no markdown fences
-   *
-   * @param query   - the original user question
-   * @param sources - vector DB matches returned by RAG retrieval
-   * @param llm     - any ILLMProvider instance (OpenAI, Anthropic, Ollama, Gemini, REST…)
-   * @returns       - a validated UITransformationResponse (type + title + description + data)
-   */
-  static async analyzeAndDecide(
-    query: string,
-    sources: VectorMatch[],
-    llm: ILLMProvider,
-  ): Promise<UITransformationResponse> {
-    try {
-      const context = this.buildContextSummary(sources);
-      const systemPrompt = this.buildVisualizationSystemPrompt();
+  // ─── Product Extraction ───────────────────────────────────────────────────
 
-      const userPrompt = [
-        `USER QUESTION: ${query}`,
-        '',
-        'RETRIEVED DATA (JSON):',
-        context,
-      ].join('\n');
+  private static getDynamicVal(
+    meta: Record<string, unknown> | undefined,
+    uiKey: string,
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap,
+  ): unknown {
+    if (!meta) return undefined;
 
-      const rawResponse = await llm.chat(
-        [{ role: 'user', content: userPrompt }],
-        '',
-        { systemPrompt, temperature: 0 },
-      );
+    const mapping = config?.rag?.uiMapping;
+    if (mapping?.[uiKey] && meta[mapping[uiKey]] !== undefined) return meta[mapping[uiKey]];
 
-      const parsed = this.parseTransformationResponse(rawResponse);
-      if (parsed) {
-        console.debug('[UITransformer] LLM chose visualization type:', parsed.type);
-        return parsed;
+    if (trainedSchema && typeof trainedSchema === 'object') {
+      const trainedKey = (trainedSchema as Record<string, unknown>)[uiKey];
+      if (trainedKey && meta[trainedKey as string] !== undefined) return meta[trainedKey as string];
+    }
+
+    return resolveMetadataValue(meta, uiKey);
+  }
+
+  private static extractProductInfo(
+    item: VectorMatch,
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap,
+  ): CarouselProduct | null {
+    const meta = item.metadata || {};
+    const name = this.getDynamicVal(meta, 'name', config, trainedSchema);
+    const price = this.getDynamicVal(meta, 'price', config, trainedSchema);
+    const brand = this.getDynamicVal(meta, 'brand', config, trainedSchema);
+    const description = this.cleanProductDescription(
+      this.extractProductDescriptionFromContent(item.content) ??
+      this.getProductDescriptionValue(meta, config, trainedSchema),
+    );
+
+    if (name || this.isProductData(item)) {
+      let finalName = name ? String(name) : undefined;
+      if (!finalName) {
+        const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+        finalName = nameMatch ? nameMatch[1].trim() : item.content.split('\n')[0].substring(0, 60);
       }
 
-      console.warn('[UITransformer] LLM returned unparseable response; falling back to heuristic.');
-    } catch (err) {
-      console.warn('[UITransformer] analyzeAndDecide LLM call failed; falling back to heuristic.', err);
+      let finalPrice: string | number | undefined =
+        typeof price === 'number' || typeof price === 'string' ? price : undefined;
+      if (!finalPrice) {
+        const priceMatch =
+          item.content.match(/(?:Price|Cost):\s*\$?([\d,.]+)/i) ||
+          item.content.match(/\$\s*([\d,.]+)/);
+        if (priceMatch) finalPrice = priceMatch[1].replace(/,/g, '');
+      }
+
+      const imageValue = this.getDynamicVal(meta, 'image', config, trainedSchema);
+
+      return {
+        id: item.id,
+        name: finalName,
+        price: finalPrice,
+        image: typeof imageValue === 'string' ? imageValue : undefined,
+        brand: brand ? String(brand) : undefined,
+        description,
+        inStock: this.determineStockStatus(item),
+      };
     }
 
-    // Fallback: keyword-heuristic transform
-    return this.transform(query, sources);
+    return null;
   }
 
-  /**
-   * Build the system prompt that instructs the LLM to return a visualization JSON.
-   */
+  private static extractProductDescriptionFromContent(content: string): string | null {
+    const bodyMatch = content.match(
+      /\bBody\s*\(HTML\)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i,
+    );
+    if (bodyMatch?.[1]) return bodyMatch[1];
+
+    const descriptionMatch = content.match(
+      /\b(?:Description|Summary|Details|Body|Content)\s*:\s*([\s\S]*?)(?=\s*[,.]?\s+\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\b\s*:|$)/i,
+    );
+    if (descriptionMatch?.[1]) return descriptionMatch[1];
+
+    return null;
+  }
+
+  private static getProductDescriptionValue(
+    meta: Record<string, unknown>,
+    config?: import('../config/RagConfig').RagConfig,
+    trainedSchema?: import('./SchemaMapper').SchemaMap,
+  ): unknown {
+    const mapped = this.getDynamicVal(meta, 'description', config, trainedSchema);
+    if (mapped !== undefined && mapped !== meta.content && mapped !== meta.text) return mapped;
+
+    const preferredKeys = [
+      'body_html',
+      'body html',
+      'bodyHtml',
+      'description',
+      'summary',
+      'details',
+      'body',
+    ];
+
+    for (const key of preferredKeys) {
+      const match = Object.keys(meta).find(candidate =>
+        this.normalizeComparableField(candidate) === this.normalizeComparableField(key),
+      );
+      if (match && meta[match] !== undefined && meta[match] !== null) return meta[match];
+    }
+
+    return undefined;
+  }
+
+  private static cleanProductDescription(raw: unknown): string | undefined {
+    if (raw === null || raw === undefined) return undefined;
+
+    const extracted = this.extractProductDescriptionFromContent(String(raw));
+    if (extracted && extracted !== String(raw)) return this.cleanProductDescription(extracted);
+
+    const text = String(raw)
+      .replace(/<[^>]*>/g, ' ')
+      .replace(/&nbsp;/gi, ' ')
+      .replace(/&amp;/gi, '&')
+      .replace(/&quot;/gi, '"')
+      .replace(/&#39;/gi, "'")
+      .replace(/\[[^\]]*TYPE[^\]]*\]/gi, ' ')
+      .replace(/\b(?:Handle|Title|Vendor|Type|Tags|Published|Option\d*\s+Name|Option\d*\s+Value|Option|Variant|Image|SKU|Price|Status|Category)\s*:\s*[^,.\n:]+[,.]?/gi, ' ')
+      .replace(/\bBody\s*\(HTML\)\s*:\s*/gi, ' ')
+      .replace(/\b(?:Description|Summary|Details|Body|Content)\s*:\s*/gi, ' ')
+      .replace(/\s+/g, ' ')
+      .trim();
+
+    return text || undefined;
+  }
+
+  // ─── Visualization Prompt ─────────────────────────────────────────────────
+
   private static buildVisualizationSystemPrompt(): string {
     return `You are a data visualization expert embedded in a RAG chat system.
-You will receive a user question and structured data retrieved from a vector database.
-Your ONLY job is to analyze this information and return a single JSON object that tells
-the frontend how to visualize it.
+You will receive a user question, a pre-computed INTENT object, and structured data from a vector database.
+Your ONLY job is to return a single JSON object describing how to visualize the data.
+The INTENT object is authoritative — honor it. For example:
+  - If intent.wantsExplicitTable is false, never return type "table".
+  - If intent.isTemporal is true, prefer line_chart.
+  - If intent.visualizationHint is "comparison" or "ranking", prefer bar_chart.
+  - If intent.visualizationHint is "composition", prefer pie_chart.
+  - If intent.recommendedChart is unsupported by the response schema, use the nearest supported type.
+  - Always respond in the language specified by intent.language.
+
+Return ONLY a valid JSON object — no markdown fences, no prose.
 
-Return ONLY a valid JSON object — no markdown code fences, no explanation, no prose.
-
-The JSON must have this exact shape:
 {
-  "type": "bar_chart" | "line_chart" | "pie_chart" | "radar_chart" | "table" | "text",
-  "title": "A concise, descriptive title for the visualization",
-  "description": "One sentence describing what the visualization shows",
-  "data": <structured data — see rules below>
+  "type": "bar_chart" | "horizontal_bar" | "line_chart" | "pie_chart" | "histogram" | "scatter_plot" | "radar_chart" | "metric_card" | "geo_map" | "table" | "text",
+  "title": "concise descriptive title",
+  "description": "one sentence describing the visualization",
+  "data": <structured data per type below>
 }
 
-DATA SHAPE per type:
-- bar_chart:  array of { "category": string, "value": number }
-- line_chart: array of { "timestamp": string, "value": number, "label": string }
-- pie_chart:  array of { "label": string, "value": number }
-- radar_chart: array of { "attribute": string, "[series1]": number, "[series2]": number, ... }
-  Example for radar_chart:
-  [
-    { "attribute": "Longevity", "Dolce Shine": 4, "CK One": 3 },
-    { "attribute": "Sillage", "Dolce Shine": 3, "CK One": 4 },
-    { "attribute": "Freshness", "Dolce Shine": 5, "CK One": 4 }
-  ]
-- table:      { "columns": string[], "rows": (string|number)[][] }
-- text:       { "content": "<prose answer>" }
-
-DECISION RULES (follow strictly):
-1. IF the user explicitly asks for a "pie chart", "bar chart", or "table", you MUST output that exact type. Do not deviate from their request.
-2. bar_chart  → comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
-3. line_chart → trends or changes over time (dates, months, years, sequential events)
-4. pie_chart  → proportional breakdown or percentage distribution
-5. radar_chart → comparing 2 or more products across MULTIPLE attributes.
-6. table      → multi-field structured records where each item has ≥ 3 attributes
-7. text       → conversational or free-form answers where no chart adds value
-
-IMPORTANT:
-- Aggregate numeric values from the raw data — do not pass raw object arrays as data.
-- Ensure all "value" fields are numbers, not strings.
-- For bar/line/pie, keep at most 12 data points for readability.
-- Never include nested objects or arrays inside bar_chart / line_chart / pie_chart data items.`;
+DATA SHAPES:
+- bar_chart:   [{ "category": string, "value": number }]
+- horizontal_bar: [{ "category": string, "value": number }]
+- histogram:   [{ "category": string, "value": number }]
+- line_chart:  [{ "timestamp": string, "value": number, "label": string }]
+- pie_chart:   [{ "label": string, "value": number }]
+- scatter_plot:[{ "x": number, "y": number, "label": string }]
+- radar_chart: [{ "attribute": string, "<series1>": number, "<series2>": number, ... }]
+- metric_card: { "label": string, "value": number, "operation": "sum"|"average"|"count"|"min"|"max"|"median" }
+- geo_map:     [{ "category": string, "value": number }]
+- table:       { "columns": string[], "rows": (string|number)[][] }
+- text:        { "content": "A concise plain-language answer." }
+
+RULES:
+1. Aggregate values — never pass raw nested objects in chart arrays.
+2. All "value" fields must be numbers.
+3. Cap bar/line/pie at 12 data points.
+4. Use dollar signs only for monetary prices, not for stock quantities or years.
+5. If no relevant data is found, return text: "I cannot answer this question based on the available information."`;
   }
 
-  /**
-   * Serialize retrieved vector matches into a compact JSON context string.
-   * Limits the total character count to avoid exceeding LLM context windows.
-   */
   private static buildContextSummary(sources: VectorMatch[], maxChars = 6000): string {
     const items = sources.map((s, i) => ({
       index: i + 1,
@@ -849,9 +1662,8 @@ IMPORTANT:
     const full = JSON.stringify(items, null, 2);
     if (full.length <= maxChars) return full;
 
-    // Truncate: include as many items as fit
     const partial: typeof items = [];
-    let chars = 2; // for []
+    let chars = 2;
     for (const item of items) {
       const chunk = JSON.stringify(item);
       if (chars + chunk.length + 2 > maxChars) break;
@@ -861,4 +1673,3 @@ IMPORTANT:
     return JSON.stringify(partial, null, 2) + '\n// ... truncated';
   }
 }
-