@retrivora-ai/rag-engine 1.7.0 → 1.7.2

package/src/handlers/index.ts

@@ -1,8 +1,9 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { VectorPlugin } from '../core/VectorPlugin';
 import { RagConfig } from '../config/RagConfig';
-import { ChatMessage } from '../types';
+import { ChatMessage, ChatResponse } from '../types';
 import { DocumentParser } from '../utils/DocumentParser';
+import { UITransformer } from '../utils/UITransformer';
 
 // ─── SSE helpers ──────────────────────────────────────────────────────────────
 
@@ -19,11 +20,16 @@ function sseTextFrame(text: string): string {
   return `data: ${JSON.stringify({ type: 'text', text })}\n\n`;
 }
 
-/** Encode the retrieval metadata as an SSE frame. */
+/** Encode the retrieval metadata with UI transformation as an SSE frame. */
 function sseMetaFrame(meta: unknown): string {
   return `data: ${JSON.stringify({ type: 'metadata', ...((meta as object) ?? {}) })}\n\n`;
 }
 
+/** Encode the UI transformation result as an SSE frame. */
+function sseUIFrame(uiTransformation: unknown): string {
+  return `data: ${JSON.stringify({ type: 'ui_transformation', ...((uiTransformation as object) ?? {}) })}\n\n`;
+}
+
 /** Encode a stream error as an SSE frame. */
 function sseErrorFrame(message: string): string {
   return `data: ${JSON.stringify({ type: 'error', error: message })}\n\n`;
@@ -145,6 +151,18 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
             } else {
               // Retrieval metadata object — always the final frame
               enqueue(sseMetaFrame(chunk));
+
+              // Transform retrieved sources to UI representation
+              const sources = (chunk as ChatResponse)?.sources || [];
+              if (sources && sources.length > 0) {
+                try {
+                  const uiTransformation = UITransformer.transform(message, sources);
+                  enqueue(sseUIFrame(uiTransformation));
+                } catch (transformError) {
+                  console.warn('[createStreamHandler] UI transformation warning:', transformError);
+                  // Don't fail the stream if transformation fails, just skip it
+                }
+              }
             }
           }
         } catch (streamError) {
@@ -292,4 +310,4 @@ export function createSuggestionsHandler(configOrPlugin?: Partial<RagConfig> | V
 }
 
 // Re-export SSE helper so host apps can use it in custom handlers
-export { sseFrame, sseTextFrame, sseMetaFrame, sseErrorFrame };
+export { sseFrame, sseTextFrame, sseMetaFrame, sseErrorFrame, sseUIFrame };

package/src/hooks/useUITransform.ts

@@ -0,0 +1,54 @@
+import { useState, useCallback } from 'react';
+import { UITransformer, UITransformationResponse } from '@/utils/UITransformer';
+import { VectorMatch } from '@/types';
+
+/**
+ * useUITransform hook — Transforms vector database results into UI components
+ *
+ * @example
+ * const { transform, result, loading } = useUITransform();
+ *
+ * const handleSearch = async (query: string) => {
+ *   const results = await fetchSearchResults(query);
+ *   transform(query, results);
+ * };
+ *
+ * return (
+ *   <>
+ *     {result && <DynamicChart data={result} />}
+ *   </>
+ * );
+ */
+export function useUITransform() {
+  const [result, setResult] = useState<UITransformationResponse | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  const transform = useCallback(
+    (query: string, data: VectorMatch[]) => {
+      try {
+        setLoading(true);
+        setError(null);
+
+        const transformed = UITransformer.transform(query, data);
+        setResult(transformed);
+
+        return transformed;
+      } catch (err) {
+        const message = err instanceof Error ? err.message : 'Transformation error';
+        setError(message);
+        return null;
+      } finally {
+        setLoading(false);
+      }
+    },
+    []
+  );
+
+  const reset = useCallback(() => {
+    setResult(null);
+    setError(null);
+  }, []);
+
+  return { transform, result, loading, error, reset };
+}

package/src/utils/UITransformer.ts

@@ -0,0 +1,535 @@
+import { VectorMatch } from '../types';
+
+/**
+ * Supported UI visualization types
+ */
+export type VisualizationType =
+  | 'pie_chart'
+  | 'bar_chart'
+  | 'line_chart'
+  | 'table'
+  | 'product_carousel'
+  | 'text';
+
+/**
+ * Base structure for all UI transformation responses
+ */
+export interface UITransformationResponse {
+  type: VisualizationType;
+  title: string;
+  description?: string;
+  data: unknown;
+}
+
+/**
+ * Pie chart data structure
+ */
+export interface PieChartData {
+  label: string;
+  value: number;
+  inStock?: boolean;
+  [key: string]: unknown;
+}
+
+/**
+ * Bar chart data structure
+ */
+export interface BarChartData {
+  category: string;
+  value: number;
+  inStock?: boolean;
+  [key: string]: unknown;
+}
+
+/**
+ * Line chart data point
+ */
+export interface LineChartDataPoint {
+  timestamp: string | number;
+  value: number;
+  label?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * Table representation
+ */
+export interface TableData {
+  columns: string[];
+  rows: (string | number | boolean)[][];
+}
+
+/**
+ * Product carousel item
+ */
+export interface CarouselProduct {
+  id: string | number;
+  name: string;
+  price?: number | string;
+  image?: string;
+  inStock?: boolean;
+  brand?: string;
+  description?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * UITransformer — Converts vector database results into structured UI representations
+ *
+ * This class analyzes retrieved data and automatically selects the best visualization
+ * format based on the content and structure of the data.
+ */
+export class UITransformer {
+  /**
+   * Main transformation method
+   * Analyzes user query and retrieved data to determine the best visualization format
+   *
+   * @param userQuery - The original user question/query
+   * @param retrievedData - Vector database retrieval results
+   * @returns Structured JSON response for UI rendering
+   */
+  static transform(
+    userQuery: string,
+    retrievedData: VectorMatch[]
+  ): UITransformationResponse {
+    if (!retrievedData || retrievedData.length === 0) {
+      return this.createTextResponse('No data available', 'No relevant data found for your query.');
+    }
+
+    // Analyze the retrieved data to determine best format
+    const analysis = this.analyzeData(userQuery, retrievedData);
+
+    // Route to appropriate transformer based on analysis
+    switch (analysis.suggestedType) {
+      case 'product_carousel':
+        return this.transformToProductCarousel(retrievedData);
+      case 'pie_chart':
+        return this.transformToPieChart(retrievedData);
+      case 'bar_chart':
+        return this.transformToBarChart(retrievedData);
+      case 'line_chart':
+        return this.transformToLineChart(retrievedData);
+      case 'table':
+        return this.transformToTable(retrievedData);
+      default:
+        return this.transformToText(retrievedData);
+    }
+  }
+
+  /**
+   * Analyzes the data and query to suggest the best visualization type
+   */
+  private static analyzeData(
+    query: string,
+    data: VectorMatch[]
+  ): {
+    suggestedType: VisualizationType;
+    confidence: number;
+    hasProducts: boolean;
+    hasTimeSeries: boolean;
+    hasCategories: boolean;
+  } {
+    const queryLower = query.toLowerCase();
+    const hasProducts = data.some(item =>
+      this.isProductData(item)
+    );
+    const hasTimeSeries = data.some(item =>
+      this.isTimeSeriesData(item)
+    );
+    const hasCategories = this.detectCategories(data).length > 0;
+    const isDistributionQuery =
+      queryLower.includes('distribution') ||
+      queryLower.includes('breakdown') ||
+      queryLower.includes('percentage') ||
+      queryLower.includes('proportion');
+    const isComparisonQuery =
+      queryLower.includes('compare') ||
+      queryLower.includes('vs') ||
+      queryLower.includes('difference');
+    const isTrendQuery =
+      queryLower.includes('trend') ||
+      queryLower.includes('over time') ||
+      queryLower.includes('growth') ||
+      queryLower.includes('decline');
+
+    // Decision logic
+    if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
+      return { suggestedType: 'product_carousel', confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
+    }
+
+    if (isTrendQuery && hasTimeSeries) {
+      return { suggestedType: 'line_chart', confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
+    }
+
+    if (isDistributionQuery && hasCategories) {
+      return { suggestedType: 'pie_chart', confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
+    }
+
+    if (isComparisonQuery && hasCategories && data.length > 2) {
+      return { suggestedType: 'bar_chart', confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
+    }
+
+    if (data.length > 5 && this.hasMultipleFields(data)) {
+      return { suggestedType: 'table', confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
+    }
+
+    return { suggestedType: 'text', confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
+  }
+
+  /**
+   * Transform data to product carousel format
+   */
+  private static transformToProductCarousel(data: VectorMatch[]): UITransformationResponse {
+    const products: CarouselProduct[] = data
+      .filter(item => this.isProductData(item))
+      .map(item => this.extractProductInfo(item))
+      .filter((p): p is CarouselProduct => p !== null);
+
+    return {
+      type: 'product_carousel',
+      title: 'Recommended Products',
+      description: `Found ${products.length} relevant products`,
+      data: products,
+    };
+  }
+
+  /**
+   * Transform data to pie chart format
+   */
+  private static transformToPieChart(
+    data: VectorMatch[]
+  ): UITransformationResponse {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+
+    const pieData: PieChartData[] = Object.entries(categoryData).map(([label, count]) => ({
+      label,
+      value: count as number,
+      inStock: this.checkStockStatus(label, data),
+    }));
+
+    return {
+      type: 'pie_chart',
+      title: 'Distribution by Category',
+      description: `Showing breakdown across ${categories.length} categories`,
+      data: pieData,
+    };
+  }
+
+  /**
+   * Transform data to bar chart format
+   */
+  private static transformToBarChart(
+    data: VectorMatch[]
+  ): UITransformationResponse {
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
+
+    const barData: BarChartData[] = Object.entries(categoryData).map(([category, value]) => ({
+      category,
+      value: value as number,
+      inStock: this.checkStockStatus(category, data),
+    }));
+
+    return {
+      type: 'bar_chart',
+      title: 'Comparison by Category',
+      description: `Comparing ${categories.length} categories`,
+      data: barData,
+    };
+  }
+
+  /**
+   * Transform data to line chart format
+   */
+  private static transformToLineChart(data: VectorMatch[]): UITransformationResponse {
+    const timePoints = this.extractTimeSeriesData(data);
+
+    const lineData: LineChartDataPoint[] = timePoints.map(point => ({
+      timestamp: point.timestamp,
+      value: point.value,
+      label: point.label,
+    }));
+
+    return {
+      type: 'line_chart',
+      title: 'Trend Over Time',
+      description: `Showing ${lineData.length} data points`,
+      data: lineData,
+    };
+  }
+
+  /**
+   * Transform data to table format
+   */
+  private static transformToTable(data: VectorMatch[]): UITransformationResponse {
+    const columns = this.extractTableColumns(data);
+    const rows = data.map(item => this.extractTableRow(item, columns));
+
+    const tableData: TableData = {
+      columns,
+      rows,
+    };
+
+    return {
+      type: 'table',
+      title: 'Detailed Results',
+      description: `Showing ${data.length} results`,
+      data: tableData,
+    };
+  }
+
+  /**
+   * Transform data to text format (fallback)
+   */
+  private static transformToText(data: VectorMatch[]): UITransformationResponse {
+    const textContent = data
+      .map(item => item.content)
+      .join('\n\n');
+
+    return this.createTextResponse(
+      'Search Results',
+      textContent,
+      `Found ${data.length} relevant results`
+    );
+  }
+
+  /**
+   * Helper: Create text response
+   */
+  private static createTextResponse(
+    title: string,
+    content: string,
+    description?: string
+  ): UITransformationResponse {
+    return {
+      type: 'text',
+      title,
+      description,
+      data: { content },
+    };
+  }
+
+  /**
+   * Helper: Check if data item is product-related
+   */
+  private static isProductData(item: VectorMatch): boolean {
+    const content = (item.content || '').toLowerCase();
+    const productKeywords = ['product', 'price', 'stock', 'item', 'sku', 'brand', 'model'];
+    return productKeywords.some(kw => content.includes(kw)) ||
+           Object.keys(item.metadata || {}).some(k =>
+             ['name', 'price', 'product', 'sku'].includes(k.toLowerCase())
+           );
+  }
+
+  /**
+   * Helper: Check if data contains time series
+   */
+  private static isTimeSeriesData(item: VectorMatch): boolean {
+    const content = (item.content || '').toLowerCase();
+    const timeKeywords = ['date', 'time', 'year', 'month', 'week', 'day', 'hour', 'trend', 'historical'];
+    return timeKeywords.some(kw => content.includes(kw)) ||
+           Object.keys(item.metadata || {}).some(k =>
+             ['date', 'timestamp', 'time', 'period'].includes(k.toLowerCase())
+           );
+  }
+
+  /**
+   * Helper: Extract product information from vector match
+   */
+  private static extractProductInfo(item: VectorMatch): CarouselProduct | null {
+    const meta = item.metadata || {};
+
+    // Try to extract product info from metadata first
+    if (meta.name || meta.product) {
+      return {
+        id: item.id,
+        name: (meta.name || meta.product) as string,
+        price: meta.price as string | number | undefined,
+        image: (meta.image || meta.imageUrl) as string | undefined,
+        brand: meta.brand as string | undefined,
+        description: item.content,
+        inStock: this.determineStockStatus(item),
+      };
+    }
+
+    // Fallback: try to parse from content
+    const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
+    const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
+
+    if (nameMatch) {
+      return {
+        id: item.id,
+        name: nameMatch[1],
+        price: priceMatch ? parseFloat(priceMatch[1]) : undefined,
+        description: item.content,
+        inStock: this.determineStockStatus(item),
+      };
+    }
+
+    return null;
+  }
+
+  /**
+   * Helper: Detect categories in data
+   */
+  private static detectCategories(data: VectorMatch[]): string[] {
+    const categories = new Set<string>();
+
+    data.forEach(item => {
+      const meta = item.metadata || {};
+
+      // Check for category field
+      if (meta.category) {
+        categories.add(String(meta.category));
+      }
+      if (meta.type) {
+        categories.add(String(meta.type));
+      }
+      if (meta.tag) {
+        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
+        tags.forEach(t => categories.add(String(t)));
+      }
+
+      // Parse from content
+      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
+      if (categoryMatch) {
+        categories.add(categoryMatch[1].trim());
+      }
+    });
+
+    return Array.from(categories);
+  }
+
+  /**
+   * Helper: Aggregate data by category
+   */
+  private static aggregateByCategory(
+    data: VectorMatch[],
+    categories: string[]
+  ): Record<string, number> {
+    const result: Record<string, number> = {};
+
+    categories.forEach(cat => {
+      result[cat] = 0;
+    });
+
+    data.forEach(item => {
+      const meta = item.metadata || {};
+      const itemCategory = (meta.category || meta.type || 'Other') as string;
+
+      if (result.hasOwnProperty(itemCategory)) {
+        result[itemCategory]++;
+      } else {
+        result['Other'] = (result['Other'] || 0) + 1;
+      }
+    });
+
+    return result;
+  }
+
+  /**
+   * Helper: Extract time series data
+   */
+  private static extractTimeSeriesData(
+    data: VectorMatch[]
+  ): Array<{ timestamp: string | number; value: number; label?: string }> {
+    return data.map(item => {
+      const meta = item.metadata || {};
+      return {
+        timestamp: (meta.timestamp || meta.date || new Date().toISOString()) as string | number,
+        value: (meta.value || item.score || 0) as number,
+        label: (meta.label || item.content.substring(0, 50)) as string,
+      };
+    });
+  }
+
+  /**
+   * Helper: Extract table columns
+   */
+  private static extractTableColumns(data: VectorMatch[]): string[] {
+    const columnSet = new Set<string>();
+    columnSet.add('Content');
+
+    data.forEach(item => {
+      Object.keys(item.metadata || {}).forEach(key => {
+        columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
+      });
+    });
+
+    return Array.from(columnSet);
+  }
+
+  /**
+   * Helper: Extract table row
+   */
+  private static extractTableRow(item: VectorMatch, columns: string[]): (string | number | boolean)[] {
+    const meta = item.metadata || {};
+    return columns.map(col => {
+      if (col === 'Content') {
+        return item.content.substring(0, 100);
+      }
+
+      const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
+      const value = meta[metaKey];
+      return value !== undefined ? String(value) : '';
+    });
+  }
+
+  /**
+   * Helper: Check stock status
+   */
+  private static checkStockStatus(category: string, data: VectorMatch[]): boolean {
+    const item = data.find(d => {
+      const meta = d.metadata || {};
+      return meta.category === category || meta.type === category;
+    });
+
+    return this.determineStockStatus(item || ({} as VectorMatch));
+  }
+
+  /**
+   * Helper: Determine if item is in stock
+   */
+  private static determineStockStatus(item: VectorMatch): boolean {
+    const meta = item.metadata || {};
+
+    // Check metadata for stock status
+    if (meta.inStock !== undefined) {
+      return Boolean(meta.inStock);
+    }
+    if (meta.stock !== undefined) {
+      return Boolean(meta.stock);
+    }
+    if (meta.available !== undefined) {
+      return Boolean(meta.available);
+    }
+
+    // Check content
+    const content = (item.content || '').toLowerCase();
+    if (content.includes('out of stock') || content.includes('unavailable')) {
+      return false;
+    }
+    if (content.includes('in stock') || content.includes('available')) {
+      return true;
+    }
+
+    // Default to true if not specified
+    return true;
+  }
+
+  /**
+   * Helper: Check if data has multiple fields
+   */
+  private static hasMultipleFields(data: VectorMatch[]): boolean {
+    const fieldCount = new Set<string>();
+
+    data.forEach(item => {
+      Object.keys(item.metadata || {}).forEach(key => {
+        fieldCount.add(key);
+      });
+    });
+
+    return fieldCount.size > 2;
+  }
+}