@retrivora-ai/rag-engine 1.8.3 → 1.8.5

package/src/providers/vectordb/QdrantProvider.ts

@@ -120,11 +120,14 @@ export class QdrantProvider extends BaseVectorProvider {
         const opts = this.config.options as Record<string, unknown>;
         const dimensionsForCreate = (opts.dimensions as number) || 1536;
 
+        const metric = this.config.distanceMetric || 'cosine';
+        const distance = metric === 'euclidean' ? 'Euclid' : metric === 'dotproduct' ? 'Dot' : 'Cosine';
+
         console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
         await this.http.put(`/collections/${this.indexName}`, {
           vectors: {
             size: dimensionsForCreate,
-            distance: 'Cosine',
+            distance: distance,
           },
         });
         console.log(`[QdrantProvider] ✅ Created collection "${this.indexName}"`);

package/src/tailwind.css

@@ -0,0 +1,2 @@
+@import "tailwindcss";
+@source "./**/*.{ts,tsx,js,jsx}";

package/src/utils/UITransformer.ts

@@ -29,7 +29,8 @@ export class UITransformer {
       ? retrievedData.filter(item => this.determineStockStatus(item))
       : retrievedData;
 
-    const categories = this.detectCategories(filteredData);
+    const groupByField = this.determineGroupByField(userQuery, filteredData);
+    const categories = this.detectCategories(filteredData, groupByField);
     const hasProducts = filteredData.some(item => this.isProductData(item));
     const isTimeSeries = filteredData.some(item => this.isTimeSeriesData(item));
     const isTrendQuery = this.isTrendQuery(userQuery);
@@ -39,14 +40,16 @@ export class UITransformer {
       return this.transformToLineChart(filteredData);
     }
 
+    const explicitChartRequest = this.isExplicitChartQuery(userQuery);
+
     // 2. High Priority: Products (Always prefer carousel for products unless it's a specific breakdown request)
-    if (hasProducts && !this.shouldShowCategoryChart(userQuery, categories)) {
+    if (hasProducts && !explicitChartRequest && !this.shouldShowCategoryChart(userQuery)) {
       return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
 
     // 3. Category breakdowns
-    if (categories.length > 1 && this.shouldShowCategoryChart(userQuery, categories)) {
-      return this.transformToPieChart(filteredData);
+    if (explicitChartRequest || (categories.length > 1 && this.shouldShowCategoryChart(userQuery))) {
+      return this.transformToPieChart(filteredData, groupByField);
     }
 
     // 4. Secondary: Products as carousel if not already handled
@@ -87,13 +90,16 @@ export class UITransformer {
    * Transform data to pie chart format
    */
   private static transformToPieChart(
-    data: VectorMatch[]
+    data: VectorMatch[],
+    groupByField: string
   ): UITransformationResponse {
-    const categories = this.detectCategories(data);
-    const categoryData = this.aggregateByCategory(data, categories);
+    let categories = this.detectCategories(data, groupByField);
+    if (categories.length === 0) categories = ['All Data'];
+    
+    const categoryData = this.aggregateByCategory(data, categories, groupByField);
 
     const pieData: PieChartData[] = Object.entries(categoryData).map(([label, count]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
+      const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data, groupByField);
       return {
         label,
         value: count as number,
@@ -104,7 +110,7 @@ export class UITransformer {
 
     return {
       type: 'pie_chart',
-      title: 'Distribution by Category',
+      title: `Distribution by ${groupByField.charAt(0).toUpperCase() + groupByField.slice(1)}`,
       description: `Showing breakdown across ${categories.length} categories`,
       data: pieData,
     };
@@ -371,11 +377,16 @@ export class UITransformer {
     return hasTimeKeyword || hasValidDateValue;
   }
 
-  private static shouldShowCategoryChart(query: string, categories: string[]): boolean {
-    if (categories.length < 2) {
-      return false;
-    }
+  private static isExplicitChartQuery(query: string): boolean {
+    const normalized = query.toLowerCase();
+    return normalized.includes('pie chart') || 
+           normalized.includes('piechart') || 
+           normalized.includes('bar chart') || 
+           normalized.includes('barchart') ||
+           normalized.includes('radar chart');
+  }
 
+  private static shouldShowCategoryChart(query: string): boolean {
     const normalized = query.toLowerCase();
     const chartKeywords = [
       'distribution',
@@ -389,6 +400,10 @@ export class UITransformer {
       'split',
       'category breakdown',
       'category distribution',
+      'pie chart',
+      'piechart',
+      'bar chart',
+      'barchart'
     ];
 
     return chartKeywords.some(keyword => normalized.includes(keyword));
@@ -501,38 +516,57 @@ export class UITransformer {
   }
 
   /**
-   * Helper: Detect categories in data
+   * Helper: Dynamically determine what field to group by based on user query
+   */
+  private static determineGroupByField(query: string, data: VectorMatch[]): string {
+    const normalized = query.toLowerCase();
+    const match = normalized.match(/(?:by|across|per|based on)\s+([a-zA-Z]+)/i);
+    
+    let targetField = 'category';
+    if (match && match[1]) {
+      let field = match[1].toLowerCase();
+      // Un-pluralize common words to match schema keys
+      if (field.endsWith('ies')) field = field.replace(/ies$/, 'y');
+      else if (field.endsWith('s') && field !== 'status') field = field.slice(0, -1);
+      
+      const stopWords = ['the', 'all', 'different', 'various', 'some', 'these', 'those'];
+      if (!stopWords.includes(field)) {
+         targetField = field;
+      }
+    }
+
+    // Verify field exists, or fallback
+    const hasTargetField = data.some(d => d.metadata && d.metadata[targetField] !== undefined);
+    if (!hasTargetField && targetField === 'category') {
+       if (data.some(d => d.metadata && d.metadata['type'] !== undefined)) return 'type';
+       if (data.some(d => d.metadata && d.metadata['brand'] !== undefined)) return 'brand';
+    }
+
+    return targetField;
+  }
+
+  /**
+   * Helper: Detect grouping values dynamically
    */
-  private static detectCategories(data: VectorMatch[]): string[] {
+  private static detectCategories(data: VectorMatch[], groupByField: string): string[] {
     const categories = new Set<string>();
 
     data.forEach(item => {
       const meta = item.metadata || {};
 
-      // Check for category field
-      if (meta.category) {
-        categories.add(String(meta.category));
-      }
-      if (meta.type) {
-        categories.add(String(meta.type));
-      }
-      if (meta.tag) {
-        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
-        tags.forEach(t => categories.add(String(t)));
-      }
-
-      // Parse category strings from content lines like "Shoes & Footwear: • Product"
-      const contentCategories = Array.from(new Set(
-        Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm))
-          .map(match => match[1].trim())
-          .filter(Boolean)
-      ));
-      contentCategories.forEach((category) => categories.add(category));
-
-      // Parse from structured category hints in content
-      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
-      if (categoryMatch) {
-        categories.add(categoryMatch[1].trim());
+      if (meta[groupByField] !== undefined) {
+        const val = meta[groupByField];
+        if (Array.isArray(val)) val.forEach(v => categories.add(String(v)));
+        else categories.add(String(val));
+      } else if (groupByField === 'category') {
+        // Fallbacks for legacy 'category' extraction
+        if (meta.type) categories.add(String(meta.type));
+        if (meta.tag) {
+          const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
+          tags.forEach(t => categories.add(String(t)));
+        }
+        const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
+        if (categoryMatch) categories.add(categoryMatch[1].trim());
       }
     });
 
@@ -540,11 +574,12 @@ export class UITransformer {
   }
 
   /**
-   * Helper: Aggregate data by category
+   * Helper: Aggregate data dynamically
    */
   private static aggregateByCategory(
     data: VectorMatch[],
-    categories: string[]
+    categories: string[],
+    groupByField: string
   ): Record<string, number> {
     const result: Record<string, number> = {};
 
@@ -554,7 +589,13 @@ export class UITransformer {
 
     data.forEach(item => {
       const meta = item.metadata || {};
-      const itemCategory = (meta.category || meta.type || 'Other') as string;
+      let itemCategory = 'Other';
+
+      if (meta[groupByField] !== undefined) {
+        itemCategory = String(meta[groupByField]);
+      } else if (groupByField === 'category' && meta.type) {
+        itemCategory = String(meta.type);
+      }
 
       if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
@@ -615,15 +656,21 @@ export class UITransformer {
   }
 
   /**
-   * Helper: Calculate stock counts by category
+   * Helper: Calculate stock counts dynamically
    */
-  private static calculateStockCounts(category: string, data: VectorMatch[]): { inStockCount: number; outOfStockCount: number } {
+  private static calculateStockCounts(category: string, data: VectorMatch[], groupByField: string): { inStockCount: number; outOfStockCount: number } {
     let inStock = 0;
     let outOfStock = 0;
 
     data.forEach(d => {
       const meta = d.metadata || {};
-      const itemCategory = (meta.category || meta.type || 'Other') as string;
+      let itemCategory = 'Other';
+      if (meta[groupByField] !== undefined) {
+        itemCategory = String(meta[groupByField]);
+      } else if (groupByField === 'category' && meta.type) {
+        itemCategory = String(meta.type);
+      }
+
       if (itemCategory === category) {
         if (this.determineStockStatus(d)) {
           inStock++;
@@ -772,12 +819,13 @@ DATA SHAPE per type:
 - text:       { "content": "<prose answer>" }
 
 DECISION RULES (follow strictly):
-1. bar_chart  → comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
-2. line_chart → trends or changes over time (dates, months, years, sequential events)
-3. pie_chart  → proportional breakdown or percentage distribution
-4. radar_chart → comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
-5. table      → multi-field structured records where each item has ≥ 3 attributes
-6. text       → conversational or free-form answers where no chart adds value
+1. IF the user explicitly asks for a "pie chart", "bar chart", or "table", you MUST output that exact type. Do not deviate from their request.
+2. bar_chart  → comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
+3. line_chart → trends or changes over time (dates, months, years, sequential events)
+4. pie_chart  → proportional breakdown or percentage distribution
+5. radar_chart → comparing 2 or more products across MULTIPLE attributes.
+6. table      → multi-field structured records where each item has ≥ 3 attributes
+7. text       → conversational or free-form answers where no chart adds value
 
 IMPORTANT:
 - Aggregate numeric values from the raw data — do not pass raw object arrays as data.