@retrivora-ai/rag-engine 1.8.8 → 1.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "1.8.8",
+  "version": "1.9.0",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",
@@ -74,7 +74,7 @@
   "scripts": {
     "dev": "next dev",
     "build": "next build",
-    "build:pkg": "npx @tailwindcss/cli -i src/tailwind.css -o src/compiled.css && tsup src/index.ts src/handlers/index.ts src/server.ts --format cjs,esm --dts --clean --no-splitting --inject-style --tsconfig tsconfig.build.json --external react,react-dom,next,mongodb,pg,openai,@anthropic-ai/sdk,@pinecone-database/pinecone,axios,lucide-react,mammoth,pdf-parse,react-markdown,remark-gfm,next-themes,langchain,@langchain/core,@langchain/openai,llamaindex && cp src/compiled.css dist/index.css && rm src/compiled.css",
+    "build:pkg": "tsup src/index.ts src/handlers/index.ts src/server.ts --format cjs,esm --dts --clean --no-splitting --tsconfig tsconfig.build.json --external react,react-dom,next,mongodb,pg,openai,@anthropic-ai/sdk,@pinecone-database/pinecone,axios,lucide-react,mammoth,pdf-parse,react-markdown,remark-gfm,next-themes,langchain,@langchain/core,@langchain/openai,llamaindex,./index.css && npx @tailwindcss/cli -i src/tailwind.css -o dist/index.css",
     "start": "next start",
     "lint": "eslint",
     "prepublishOnly": "npm run build:pkg"

package/src/config/RagConfig.ts

@@ -44,8 +44,6 @@ export interface VectorDBConfig {
    * - searchFields?: string[] | string // optional override for which columns receive exact-match boosts
    */
   options: Record<string, unknown>;
-  /** Optional distance metric (defaults to cosine if not specified and supported by provider) */
-  distanceMetric?: 'cosine' | 'euclidean' | 'dotproduct';
 }
 
 // ---------------------------------------------------------------------------

package/src/index.ts

@@ -3,7 +3,7 @@
  * This entry point is CLIENT-SAFE and can be imported in React Client Components.
  */
 
-import './compiled.css';
+import './index.css';
 
 // ── React Components ─────────────────────────────────────────
 export { ChatWidget } from './components/ChatWidget';

package/src/providers/vectordb/ChromaDBProvider.ts

@@ -41,13 +41,9 @@ export class ChromaDBProvider extends BaseVectorProvider {
     } catch (err) {
       if (axios.isAxiosError(err) && err.response?.status === 404) {
         // Create the collection if it doesn't exist
-        const metric = this.config.distanceMetric || 'cosine';
-        const space = metric === 'euclidean' ? 'l2' : metric === 'dotproduct' ? 'ip' : 'cosine';
-
         console.log(`[ChromaDBProvider] ⏳ Collection "${this.indexName}" not found. Creating...`);
         const { data } = await this.http.post('/api/v1/collections', {
           name: this.indexName,
-          metadata: { "hnsw:space": space }
         });
         this.collectionId = data.id;
         console.log(`[ChromaDBProvider] ✅ Created collection "${this.indexName}" (id: ${this.collectionId})`);

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -66,13 +66,10 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           embedding   VECTOR(${this.dimensions})
         )
       `);
-      const metric = this.config.distanceMetric || 'cosine';
-      const indexOps = metric === 'euclidean' ? 'vector_l2_ops' : metric === 'dotproduct' ? 'vector_ip_ops' : 'vector_cosine_ops';
-
       await client.query(`
         CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
         ON ${this.uploadTable}
-        USING hnsw (embedding ${indexOps})
+        USING hnsw (embedding vector_cosine_ops)
       `);
 
       const res = await client.query(`
@@ -150,13 +147,10 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         await client.query(createTableSql);
         
         // Create Index
-        const metric = this.config.distanceMetric || 'cosine';
-        const indexOps = metric === 'euclidean' ? 'vector_l2_ops' : metric === 'dotproduct' ? 'vector_ip_ops' : 'vector_cosine_ops';
-
         await client.query(`
           CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
           ON "${tableName}"
-          USING hnsw (embedding ${indexOps})
+          USING hnsw (embedding vector_cosine_ops)
         `);
         
         // Ensure this new table is added to our search scope
@@ -280,13 +274,6 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         let sqlQuery = '';
         let params: unknown[] = [];
 
-        const metric = this.config.distanceMetric || 'cosine';
-        const scoreExpr = metric === 'euclidean' 
-          ? `1 / (1 + (embedding <-> $1::vector))` 
-          : metric === 'dotproduct' 
-          ? `(embedding <#> $1::vector) * -1` 
-          : `1 - (embedding <=> $1::vector)`;
-
         if (queryText) {
           const hasEntityHints = entityHints.length > 0;
           const exactNameScoreExpr = hasEntityHints
@@ -303,9 +290,9 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           // Actual Hybrid Search: Semantic Vector Score + Full-Text Keyword Rank
           sqlQuery = `
             SELECT *, 
-             (${scoreExpr}) AS vector_score,
+             (1 - (embedding <=> $1::vector)) AS vector_score,
              COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
-             ((${scoreExpr}) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50
@@ -315,7 +302,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           // Fallback to Vector-only Search
           sqlQuery = `
             SELECT *, 
-             (${scoreExpr}) AS hybrid_score
+             (1 - (embedding <=> $1::vector)) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -97,13 +97,10 @@ export class PostgreSQLProvider extends BaseVectorProvider {
           embedding   VECTOR(${this.dimensions})
         )
       `);
-      const metric = this.config.distanceMetric || 'cosine';
-      const indexOps = metric === 'euclidean' ? 'vector_l2_ops' : metric === 'dotproduct' ? 'vector_ip_ops' : 'vector_cosine_ops';
-      
       await client.query(`
         CREATE INDEX IF NOT EXISTS ${this.tableName}_embedding_idx
         ON ${this.tableName}
-        USING hnsw (embedding ${indexOps})
+        USING hnsw (embedding vector_cosine_ops)
       `);
     } finally {
       client.release();
@@ -185,19 +182,11 @@ export class PostgreSQLProvider extends BaseVectorProvider {
       const efSearch = (this.config.options.efSearch as number) || Math.max(topK * 10, 40);
       await client.query(`SET LOCAL hnsw.ef_search = ${efSearch}`);
 
-      const metric = this.config.distanceMetric || 'cosine';
-      const queryOp = metric === 'euclidean' ? '<->' : metric === 'dotproduct' ? '<#>' : '<=>';
-      const scoreExpr = metric === 'euclidean' 
-        ? `1 / (1 + (embedding <-> $1::vector))` 
-        : metric === 'dotproduct' 
-        ? `(embedding <#> $1::vector) * -1` 
-        : `1 - (embedding <=> $1::vector)`;
-
       const result = await client.query(
-        `SELECT id, content, metadata, ${scoreExpr} AS score
+        `SELECT id, content, metadata, 1 - (embedding <=> $1::vector) AS score
          FROM ${this.tableName}
          ${whereClause}
-         ORDER BY embedding ${queryOp} $1::vector
+         ORDER BY embedding <=> $1::vector
          LIMIT $2`,
         params
       );

package/src/providers/vectordb/QdrantProvider.ts

@@ -120,14 +120,11 @@ export class QdrantProvider extends BaseVectorProvider {
         const opts = this.config.options as Record<string, unknown>;
         const dimensionsForCreate = (opts.dimensions as number) || 1536;
 
-        const metric = this.config.distanceMetric || 'cosine';
-        const distance = metric === 'euclidean' ? 'Euclid' : metric === 'dotproduct' ? 'Dot' : 'Cosine';
-
         console.log(`[QdrantProvider] ⏳ Creating collection "${this.indexName}" (dimensions: ${dimensionsForCreate})...`);
         await this.http.put(`/collections/${this.indexName}`, {
           vectors: {
             size: dimensionsForCreate,
-            distance: distance,
+            distance: 'Cosine',
           },
         });
         console.log(`[QdrantProvider] ✅ Created collection "${this.indexName}"`);

package/src/utils/UITransformer.ts

@@ -29,8 +29,7 @@ export class UITransformer {
       ? retrievedData.filter(item => this.determineStockStatus(item))
       : retrievedData;
 
-    const groupByField = this.determineGroupByField(userQuery, filteredData);
-    const categories = this.detectCategories(filteredData, groupByField);
+    const categories = this.detectCategories(filteredData);
     const hasProducts = filteredData.some(item => this.isProductData(item));
     const isTimeSeries = filteredData.some(item => this.isTimeSeriesData(item));
     const isTrendQuery = this.isTrendQuery(userQuery);
@@ -40,16 +39,14 @@ export class UITransformer {
       return this.transformToLineChart(filteredData);
     }
 
-    const explicitChartRequest = this.isExplicitChartQuery(userQuery);
-
     // 2. High Priority: Products (Always prefer carousel for products unless it's a specific breakdown request)
-    if (hasProducts && !explicitChartRequest && !this.shouldShowCategoryChart(userQuery)) {
+    if (hasProducts && !this.shouldShowCategoryChart(userQuery, categories)) {
       return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
 
     // 3. Category breakdowns
-    if (explicitChartRequest || (categories.length > 1 && this.shouldShowCategoryChart(userQuery))) {
-      return this.transformToPieChart(filteredData, groupByField);
+    if (categories.length > 1 && this.shouldShowCategoryChart(userQuery, categories)) {
+      return this.transformToPieChart(filteredData);
     }
 
     // 4. Secondary: Products as carousel if not already handled
@@ -90,16 +87,13 @@ export class UITransformer {
    * Transform data to pie chart format
    */
   private static transformToPieChart(
-    data: VectorMatch[],
-    groupByField: string
+    data: VectorMatch[]
   ): UITransformationResponse {
-    let categories = this.detectCategories(data, groupByField);
-    if (categories.length === 0) categories = ['All Data'];
-    
-    const categoryData = this.aggregateByCategory(data, categories, groupByField);
+    const categories = this.detectCategories(data);
+    const categoryData = this.aggregateByCategory(data, categories);
 
     const pieData: PieChartData[] = Object.entries(categoryData).map(([label, count]) => {
-      const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data, groupByField);
+      const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
       return {
         label,
         value: count as number,
@@ -110,7 +104,7 @@ export class UITransformer {
 
     return {
       type: 'pie_chart',
-      title: `Distribution by ${groupByField.charAt(0).toUpperCase() + groupByField.slice(1)}`,
+      title: 'Distribution by Category',
       description: `Showing breakdown across ${categories.length} categories`,
       data: pieData,
     };
@@ -377,16 +371,11 @@ export class UITransformer {
     return hasTimeKeyword || hasValidDateValue;
   }
 
-  private static isExplicitChartQuery(query: string): boolean {
-    const normalized = query.toLowerCase();
-    return normalized.includes('pie chart') || 
-           normalized.includes('piechart') || 
-           normalized.includes('bar chart') || 
-           normalized.includes('barchart') ||
-           normalized.includes('radar chart');
-  }
+  private static shouldShowCategoryChart(query: string, categories: string[]): boolean {
+    if (categories.length < 2) {
+      return false;
+    }
 
-  private static shouldShowCategoryChart(query: string): boolean {
     const normalized = query.toLowerCase();
     const chartKeywords = [
       'distribution',
@@ -400,10 +389,6 @@ export class UITransformer {
       'split',
       'category breakdown',
       'category distribution',
-      'pie chart',
-      'piechart',
-      'bar chart',
-      'barchart'
     ];
 
     return chartKeywords.some(keyword => normalized.includes(keyword));
@@ -516,57 +501,38 @@ export class UITransformer {
   }
 
   /**
-   * Helper: Dynamically determine what field to group by based on user query
-   */
-  private static determineGroupByField(query: string, data: VectorMatch[]): string {
-    const normalized = query.toLowerCase();
-    const match = normalized.match(/(?:by|across|per|based on)\s+([a-zA-Z]+)/i);
-    
-    let targetField = 'category';
-    if (match && match[1]) {
-      let field = match[1].toLowerCase();
-      // Un-pluralize common words to match schema keys
-      if (field.endsWith('ies')) field = field.replace(/ies$/, 'y');
-      else if (field.endsWith('s') && field !== 'status') field = field.slice(0, -1);
-      
-      const stopWords = ['the', 'all', 'different', 'various', 'some', 'these', 'those'];
-      if (!stopWords.includes(field)) {
-         targetField = field;
-      }
-    }
-
-    // Verify field exists, or fallback
-    const hasTargetField = data.some(d => d.metadata && d.metadata[targetField] !== undefined);
-    if (!hasTargetField && targetField === 'category') {
-       if (data.some(d => d.metadata && d.metadata['type'] !== undefined)) return 'type';
-       if (data.some(d => d.metadata && d.metadata['brand'] !== undefined)) return 'brand';
-    }
-
-    return targetField;
-  }
-
-  /**
-   * Helper: Detect grouping values dynamically
+   * Helper: Detect categories in data
    */
-  private static detectCategories(data: VectorMatch[], groupByField: string): string[] {
+  private static detectCategories(data: VectorMatch[]): string[] {
     const categories = new Set<string>();
 
     data.forEach(item => {
       const meta = item.metadata || {};
 
-      if (meta[groupByField] !== undefined) {
-        const val = meta[groupByField];
-        if (Array.isArray(val)) val.forEach(v => categories.add(String(v)));
-        else categories.add(String(val));
-      } else if (groupByField === 'category') {
-        // Fallbacks for legacy 'category' extraction
-        if (meta.type) categories.add(String(meta.type));
-        if (meta.tag) {
-          const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
-          tags.forEach(t => categories.add(String(t)));
-        }
-        const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
-        if (categoryMatch) categories.add(categoryMatch[1].trim());
+      // Check for category field
+      if (meta.category) {
+        categories.add(String(meta.category));
+      }
+      if (meta.type) {
+        categories.add(String(meta.type));
+      }
+      if (meta.tag) {
+        const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
+        tags.forEach(t => categories.add(String(t)));
+      }
+
+      // Parse category strings from content lines like "Shoes & Footwear: • Product"
+      const contentCategories = Array.from(new Set(
+        Array.from(item.content.matchAll(/^\s*([^:\n]+):\s*(?:•|\*|-)*/gm))
+          .map(match => match[1].trim())
+          .filter(Boolean)
+      ));
+      contentCategories.forEach((category) => categories.add(category));
+
+      // Parse from structured category hints in content
+      const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
+      if (categoryMatch) {
+        categories.add(categoryMatch[1].trim());
       }
     });
 
@@ -574,12 +540,11 @@ export class UITransformer {
   }
 
   /**
-   * Helper: Aggregate data dynamically
+   * Helper: Aggregate data by category
    */
   private static aggregateByCategory(
     data: VectorMatch[],
-    categories: string[],
-    groupByField: string
+    categories: string[]
   ): Record<string, number> {
     const result: Record<string, number> = {};
 
@@ -589,13 +554,7 @@ export class UITransformer {
 
     data.forEach(item => {
       const meta = item.metadata || {};
-      let itemCategory = 'Other';
-
-      if (meta[groupByField] !== undefined) {
-        itemCategory = String(meta[groupByField]);
-      } else if (groupByField === 'category' && meta.type) {
-        itemCategory = String(meta.type);
-      }
+      const itemCategory = (meta.category || meta.type || 'Other') as string;
 
       if (Object.prototype.hasOwnProperty.call(result, itemCategory)) {
         result[itemCategory]++;
@@ -656,21 +615,15 @@ export class UITransformer {
   }
 
   /**
-   * Helper: Calculate stock counts dynamically
+   * Helper: Calculate stock counts by category
    */
-  private static calculateStockCounts(category: string, data: VectorMatch[], groupByField: string): { inStockCount: number; outOfStockCount: number } {
+  private static calculateStockCounts(category: string, data: VectorMatch[]): { inStockCount: number; outOfStockCount: number } {
     let inStock = 0;
     let outOfStock = 0;
 
     data.forEach(d => {
       const meta = d.metadata || {};
-      let itemCategory = 'Other';
-      if (meta[groupByField] !== undefined) {
-        itemCategory = String(meta[groupByField]);
-      } else if (groupByField === 'category' && meta.type) {
-        itemCategory = String(meta.type);
-      }
-
+      const itemCategory = (meta.category || meta.type || 'Other') as string;
       if (itemCategory === category) {
         if (this.determineStockStatus(d)) {
           inStock++;
@@ -819,13 +772,12 @@ DATA SHAPE per type:
 - text:       { "content": "<prose answer>" }
 
 DECISION RULES (follow strictly):
-1. IF the user explicitly asks for a "pie chart", "bar chart", or "table", you MUST output that exact type. Do not deviate from their request.
-2. bar_chart  → comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
-3. line_chart → trends or changes over time (dates, months, years, sequential events)
-4. pie_chart  → proportional breakdown or percentage distribution
-5. radar_chart → comparing 2 or more products across MULTIPLE attributes.
-6. table      → multi-field structured records where each item has ≥ 3 attributes
-7. text       → conversational or free-form answers where no chart adds value
+1. bar_chart  → comparing quantities across categories (e.g. sales by region, price by product). Use this when there is only ONE value per category.
+2. line_chart → trends or changes over time (dates, months, years, sequential events)
+3. pie_chart  → proportional breakdown or percentage distribution
+4. radar_chart → comparing 2 or more products across MULTIPLE attributes (e.g. comparing features, ratings, or characteristics of 2 specific products). If the user asks to "compare" products and the data contains multiple dimensions or ratings for them, you MUST use this.
+5. table      → multi-field structured records where each item has ≥ 3 attributes
+6. text       → conversational or free-form answers where no chart adds value
 
 IMPORTANT:
 - Aggregate numeric values from the raw data — do not pass raw object arrays as data.