@retrivora-ai/rag-engine 2.0.2 → 2.0.3

package/src/rendering/rules/Rule2ComparisonRule.ts

@@ -0,0 +1,24 @@
+import { DecisionContext, IntentCategory, IRenderRule, RenderDecision } from '../types';
+
+export class Rule2ComparisonRule implements IRenderRule {
+  readonly name = 'Rule2Comparison';
+  readonly priority = 90;
+
+  evaluate(context: DecisionContext, intent: IntentCategory): RenderDecision | null {
+    const q = (context.userQuery || '').toLowerCase().trim();
+
+    const isComparison =
+      intent === 'comparison' ||
+      /\b(compare|comparison|vs\.?|versus|difference between|differ|contrast|against)\b/i.test(q);
+
+    if (isComparison) {
+      return {
+        renderType: 'table',
+        confidence: 0.90,
+        reason: 'Rule 2: Comparison query contrasting multiple entities requires structured table rendering.',
+      };
+    }
+
+    return null;
+  }
+}

package/src/rendering/rules/Rule3ProductDiscoveryRule.ts

@@ -0,0 +1,28 @@
+import { DecisionContext, IntentCategory, IRenderRule, RenderDecision } from '../types';
+
+export class Rule3ProductDiscoveryRule implements IRenderRule {
+  readonly name = 'Rule3ProductDiscovery';
+  readonly priority = 85;
+
+  evaluate(context: DecisionContext, intent: IntentCategory): RenderDecision | null {
+    const q = (context.userQuery || '').toLowerCase().trim();
+
+    const isDiscovery =
+      intent === 'product_search' ||
+      intent === 'recommendation' ||
+      /\b(best|recommend|top|buy|shopping|under \$|cheap|shoes|laptops|chairs|headphones|sneakers|apparel|products|show available)\b/i.test(q);
+
+    const isNotChart = !/\b(chart|sales|revenue|monthly|growth|trend|over time)\b/i.test(q);
+    const isAnalyticalRanking = /\b(top performing|top selling|best performing|performing|sales|revenue|ranking)\b/i.test(q);
+
+    if (isDiscovery && isNotChart && !isAnalyticalRanking) {
+      return {
+        renderType: 'carousel',
+        confidence: 0.90,
+        reason: 'Rule 3: Product discovery/recommendation query requires product carousel rendering.',
+      };
+    }
+
+    return null;
+  }
+}

package/src/rendering/rules/Rule4AnalyticalRule.ts

@@ -0,0 +1,38 @@
+import { ChartType, DecisionContext, IntentCategory, IRenderRule, RenderDecision } from '../types';
+
+export class Rule4AnalyticalRule implements IRenderRule {
+  readonly name = 'Rule4Analytical';
+  readonly priority = 80;
+
+  evaluate(context: DecisionContext, intent: IntentCategory): RenderDecision | null {
+    const q = (context.userQuery || '').toLowerCase().trim();
+
+    const isAnalytical =
+      intent === 'analytics' ||
+      intent === 'trend_analysis' ||
+      intent === 'ranking' ||
+      /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
+
+    if (!isAnalytical) return null;
+
+    // Determine specific chart type based on query intent signals
+    let chartType: ChartType = 'bar'; // Default category comparison
+
+    if (intent === 'trend_analysis' || /\b(monthly|yearly|weekly|growth|over time|trend|timeline|historical)\b/i.test(q)) {
+      chartType = 'line';
+    } else if (/\b(share|percentage|percent|breakup|breakdown|composition|pie|donut|proportion)\b/i.test(q)) {
+      chartType = 'pie';
+    } else if (/\b(correlation|scatter|relationship|relation|impact of|depends on)\b/i.test(q)) {
+      chartType = 'scatter';
+    } else {
+      chartType = 'bar';
+    }
+
+    return {
+      renderType: 'chart',
+      chartType,
+      confidence: 0.85,
+      reason: `Rule 4: Analytical query with numerical insights selects ${chartType} chart.`,
+    };
+  }
+}

package/src/rendering/rules/Rule5MixedResponseRule.ts

@@ -0,0 +1,29 @@
+import { DecisionContext, IntentCategory, IRenderRule, RenderDecision } from '../types';
+
+export class Rule5MixedResponseRule implements IRenderRule {
+  readonly name = 'Rule5MixedResponse';
+  readonly priority = 95;
+
+  evaluate(context: DecisionContext, _intent: IntentCategory): RenderDecision | null {
+    const q = (context.userQuery || '').toLowerCase().trim();
+
+    // Specific mixed intent match: "top selling products this month" / multi-section requests
+    const isTopSellingThisMonth = /\btop\s*(selling|performing)?\s*products?\s*(this|last)?\s*(month|year|week)\b/i.test(q);
+    const hasMultipleSectionsRequest = /\b(summary|overview)\b/i.test(q) && /\b(carousel|products?|items?)\b/i.test(q) && /\b(chart|sales|trend|revenue)\b/i.test(q);
+
+    if (isTopSellingThisMonth || hasMultipleSectionsRequest) {
+      return {
+        renderType: 'mixed',
+        confidence: 0.95,
+        reason: 'Rule 5: Mixed response query requires multi-section composition (text, carousel, chart).',
+        sections: [
+          { type: 'text', title: 'Summary' },
+          { type: 'carousel', title: 'Top Products' },
+          { type: 'chart', chartType: 'bar', title: 'Sales Performance' },
+        ],
+      };
+    }
+
+    return null;
+  }
+}

package/src/rendering/rules/Rule6SmallResultSetRule.ts

@@ -0,0 +1,24 @@
+import { DecisionContext, IntentCategory, IRenderRule, RenderDecision } from '../types';
+
+export class Rule6SmallResultSetRule implements IRenderRule {
+  readonly name = 'Rule6SmallResultSet';
+  readonly priority = 110;
+
+  evaluate(context: DecisionContext, _intent: IntentCategory): RenderDecision | null {
+    const docs = context.retrievedDocuments || [];
+
+    // Only apply if documents are present and fewer than 3
+    if (docs.length > 0 && docs.length < 3) {
+      const q = (context.userQuery || '').toLowerCase().trim();
+      const isExplicitComparison = /\b(compare|versus|vs\.?)\b/i.test(q);
+
+      return {
+        renderType: isExplicitComparison ? 'table' : 'text',
+        confidence: 0.98,
+        reason: `Rule 6: Small result set (${docs.length} rows < 3). Charts avoided to prevent sparse or uninformative visualizations.`,
+      };
+    }
+
+    return null;
+  }
+}

package/src/rendering/rules/Rule7LargeTableRule.ts

@@ -0,0 +1,35 @@
+import { DecisionContext, IntentCategory, IRenderRule, RenderDecision } from '../types';
+
+export class Rule7LargeTableRule implements IRenderRule {
+  readonly name = 'Rule7LargeTable';
+  readonly priority = 75;
+
+  evaluate(context: DecisionContext, intent: IntentCategory): RenderDecision | null {
+    const docs = context.retrievedDocuments || [];
+
+    if (docs.length > 5) {
+      const q = (context.userQuery || '').toLowerCase().trim();
+      const hasNumericAgg = intent === 'analytics' || intent === 'ranking' || /\b(sales|revenue|growth|count|total|average)\b/i.test(q);
+
+      if (hasNumericAgg) {
+        return {
+          renderType: 'mixed',
+          confidence: 0.88,
+          reason: `Rule 7: Large result set (${docs.length} rows > 5) with numeric aggregation rendered as Table + Chart.`,
+          sections: [
+            { type: 'table', title: 'Detailed Data Table' },
+            { type: 'chart', chartType: 'bar', title: 'Numerical Breakdown' },
+          ],
+        };
+      }
+
+      return {
+        renderType: 'table',
+        confidence: 0.85,
+        reason: `Rule 7: Large result set (${docs.length} rows > 5) rendered as structured table for readability.`,
+      };
+    }
+
+    return null;
+  }
+}

package/src/rendering/types.ts

@@ -0,0 +1,96 @@
+import { VectorMatch, UITransformationResponse } from '../types';
+
+/**
+ * Primary rendering outcome types supported by the decision engine.
+ */
+export type RenderType =
+  | 'text'
+  | 'table'
+  | 'carousel'
+  | 'chart'
+  | 'mixed'
+  | 'card'
+  | 'none';
+
+/**
+ * Chart sub-types available when renderType === 'chart'.
+ */
+export type ChartType =
+  | 'bar'
+  | 'line'
+  | 'pie'
+  | 'scatter';
+
+/**
+ * High-level intent classification categories.
+ */
+export type IntentCategory =
+  | 'information_lookup'
+  | 'product_search'
+  | 'recommendation'
+  | 'comparison'
+  | 'analytics'
+  | 'trend_analysis'
+  | 'ranking'
+  | 'summarization';
+
+/**
+ * Individual section specifier for multi-section (mixed) responses.
+ */
+export interface RenderSectionDecision {
+  type: RenderType;
+  chartType?: ChartType;
+  title?: string;
+}
+
+/**
+ * Final decision produced by the Visualization Decision Engine.
+ */
+export interface RenderDecision {
+  /** Primary rendering format selected */
+  renderType: RenderType;
+  /** Specific chart sub-type if renderType === 'chart' */
+  chartType?: ChartType;
+  /** Confidence score between 0.0 and 1.0 */
+  confidence: number;
+  /** Human-readable explanation of why this decision was reached */
+  reason: string;
+  /** Sub-sections if renderType === 'mixed' */
+  sections?: RenderSectionDecision[];
+}
+
+/**
+ * Context provided to the intent classifier & decision engine rules.
+ */
+export interface DecisionContext {
+  /** Original user prompt/query string */
+  userQuery: string;
+  /** Vector DB retrieved documents / context records */
+  retrievedDocuments?: VectorMatch[];
+  /** Optional LLM text response if already available */
+  llmResponse?: string;
+  /** Additional custom metadata or config settings */
+  metadata?: Record<string, unknown>;
+}
+
+/**
+ * Interface implemented by rule strategies in the Rule Engine.
+ */
+export interface IRenderRule {
+  /** Unique name of the decision rule */
+  name: string;
+  /** Precedence score (higher priority rules run first) */
+  priority: number;
+  /** Evaluate the rule against current query context and classified intent */
+  evaluate(context: DecisionContext, intent: IntentCategory): RenderDecision | null;
+}
+
+/**
+ * Interface implemented by pluggable Renderer Strategies in RendererRegistry.
+ */
+export interface IRendererStrategy {
+  /** Render type identifier handled by this strategy */
+  type: RenderType | string;
+  /** Transform input data into a standardized UITransformationResponse payload */
+  render(data: unknown, options?: Record<string, unknown>): UITransformationResponse;
+}

package/src/server.ts

@@ -20,7 +20,7 @@ export type {
   LLMProvider,
   EmbeddingProvider,
 } from './config/RagConfig';
-export type { VectorMatch, UpsertDocument, IngestDocument, ChatResponse, ChatMessage, ChatOptions } from './types';
+export type * from './types';
 export type { ILLMProvider, EmbedOptions } from './llm/ILLMProvider';
 export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
 export type { HealthCheckResult, IProviderValidator, IProviderHealthChecker } from './core/ProviderInterfaces';
@@ -45,9 +45,10 @@ export { ConfigBuilder, PRESETS, createFromPreset } from './config/ConfigBuilder
 export { EmbeddingStrategyResolver, EmbeddingStrategy } from './config/EmbeddingStrategy';
 export { LLM_PROFILES, VECTOR_PROFILES } from './config/UniversalProfiles';
 
-// ── RAG Utilities ──────────────────────────────────────────────
+// ── RAG Utilities & Visualization ──────────────────────────────────────────────
 export { DocumentChunker } from './rag/DocumentChunker';
 export { DocumentParser } from './utils/DocumentParser';
+export * from './rendering';
 
 // ── Vector DB Providers ────────────────────────────────────────
 export { BaseVectorProvider } from './providers/vectordb/BaseVectorProvider';

package/src/types/index.ts

@@ -226,3 +226,51 @@ export interface CarouselProduct {
 
 export * from './props';
 export * from './chat';
+
+// ─── Global Retrivora Metadata Schema ───────────────────────────────────────
+
+/**
+ * Standardized Retrivora Metadata Schema for document vectors and storage.
+ * Guarantees uniform metadata structure across vector DB backends (Pinecone, MongoDB, etc.).
+ */
+export interface RetrivoraChunkMetadata {
+  /** Document unique identifier */
+  docId: string;
+  /** Original file name (e.g. report.pdf, data.json, sheets.xlsx) */
+  fileName: string;
+  /** File format type identifier */
+  fileType: 'txt' | 'json' | 'pdf' | 'excel' | 'word' | 'md' | 'csv' | string;
+  /** MIME type string */
+  mimeType: string;
+  /** 0-indexed position of chunk within the parent document */
+  chunkIndex: number;
+  /** Total count of chunks in the document */
+  totalChunks: number;
+  /** Extracted text content of the chunk */
+  content: string;
+  /** Character length of the chunk content */
+  characterCount: number;
+  /** Workspace or Project ID */
+  workspaceId: string;
+  /** Isolated vector database namespace (e.g. Pinecone namespace) */
+  pineconeNamespace: string;
+  /** Retrivora Tier level */
+  tier: 'free' | 'enterprise';
+  /** ISO 8601 timestamp when the document was processed */
+  ingestedAt: string;
+  /** Additional custom metadata Key-Value pairs */
+  customMetadata?: Record<string, unknown>;
+}
+
+export interface RetrivoraIngestedDocument {
+  docId: string;
+  fileName: string;
+  fileType: string;
+  mimeType: string;
+  fileSizeBytes?: number;
+  totalChunks: number;
+  workspaceId: string;
+  pineconeNamespace: string;
+  ingestedAt: string;
+  chunks: RetrivoraChunkMetadata[];
+}

package/src/utils/DocumentParser.ts

@@ -1,6 +1,6 @@
 /**
  * DocumentParser — handles text extraction from various file formats.
- * Supported: .txt, .md, .json, .csv, .pdf (requires pdf-parse), .docx (requires mammoth)
+ * Supported: .txt, .md, .json, .csv, .pdf, .docx, .doc, .xlsx, .xls
  */
 
 export class DocumentParser {
@@ -8,14 +8,14 @@ export class DocumentParser {
    * Extract text from a File or Buffer based on its type.
    */
   static async parse(file: File | Buffer, fileName: string, mimeType: string): Promise<string> {
-    const extension = fileName.split('.').pop()?.toLowerCase();
+    const extension = fileName.split('.').pop()?.toLowerCase() || '';
 
-    // 1. Basic text formats
+    // 1. Basic text & Markdown formats
     if (extension === 'txt' || extension === 'md' || mimeType === 'text/plain' || mimeType === 'text/markdown') {
       return this.readAsText(file);
     }
 
-    // 2. JSON
+    // 2. JSON files
     if (extension === 'json' || mimeType === 'application/json') {
       const text = await this.readAsText(file);
       try {
@@ -26,39 +26,75 @@ export class DocumentParser {
       }
     }
 
-    // 3. CSV
+    // 3. CSV files
     if (extension === 'csv' || mimeType === 'text/csv') {
       return this.readAsText(file);
     }
 
-    // 4. PDF (Requires pdf-parse)
+    // 4. Excel spreadsheets (.xlsx, .xls)
+    if (
+      extension === 'xlsx' ||
+      extension === 'xls' ||
+      mimeType.includes('spreadsheet') ||
+      mimeType.includes('excel')
+    ) {
+      try {
+        // Dynamic import with ts-ignore to allow optional resolution
+        const xlsx = await import('xlsx');
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await (file as File).arrayBuffer());
+        const workbook = xlsx.read(buffer, { type: 'buffer' });
+        const sheetsText: string[] = [];
+
+        for (const sheetName of workbook.SheetNames) {
+          const sheet = workbook.Sheets[sheetName];
+          const csv = xlsx.utils.sheet_to_csv(sheet);
+          if (csv && csv.trim()) {
+            sheetsText.push(`--- Sheet: ${sheetName} ---\n${csv.trim()}`);
+          }
+        }
+        return sheetsText.join('\n\n') || `[Empty Excel Workbook: ${fileName}]`;
+      } catch {
+        // Fall back to reading text lines if xlsx module is not present
+        try {
+          return await this.readAsText(file);
+        } catch {
+          return `[Excel File Content: ${fileName}]`;
+        }
+      }
+    }
+
+    // 5. PDF Documents (Requires pdf-parse)
     if (extension === 'pdf' || mimeType === 'application/pdf') {
-       try {
-         // Dynamic import to avoid errors if not installed
-         const pdf = await import('pdf-parse');
-         const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await (file as File).arrayBuffer());
-         const data = await pdf.default(buffer);
-         return data.text;
-       } catch (err) {
-         console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
-         return `[PDF Parsing Error for ${fileName}]`;
-       }
+      try {
+        const pdf = await import('pdf-parse');
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await (file as File).arrayBuffer());
+        const data = await pdf.default(buffer);
+        return data.text;
+      } catch (err) {
+        console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
+        return `[PDF Parsing Error for ${fileName}]`;
+      }
     }
 
-    // 5. DOCX (Requires mammoth)
-    if (extension === 'docx' || mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document') {
-       try {
-         const mammoth = await import('mammoth');
-         const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await (file as File).arrayBuffer());
-         const result = await mammoth.extractRawText({ buffer });
-         return result.value;
-       } catch (err) {
-         console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
-         return `[DOCX Parsing Error for ${fileName}]`;
-       }
+    // 6. Word Documents (.docx, .doc) (Requires mammoth)
+    if (
+      extension === 'docx' ||
+      extension === 'doc' ||
+      mimeType === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' ||
+      mimeType === 'application/msword'
+    ) {
+      try {
+        const mammoth = await import('mammoth');
+        const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await (file as File).arrayBuffer());
+        const result = await mammoth.extractRawText({ buffer });
+        return result.value;
+      } catch (err) {
+        console.warn('[DocumentParser] Word parsing failed. Make sure "mammoth" is installed.', err);
+        return `[Word Parsing Error for ${fileName}]`;
+      }
     }
 
-    // Fallback: try reading as text
+    // Fallback: try reading as plain UTF-8 text
     try {
       return await this.readAsText(file);
     } catch {

package/src/utils/UITransformer.ts

@@ -12,6 +12,7 @@ import {
 } from '../types';
 import { ILLMProvider } from '../llm/ILLMProvider';
 import { resolveMetadataValue } from './synonyms';
+import { VisualizationDecisionEngine } from '../rendering/VisualizationDecisionEngine';
 
 // ─── Intent Types ─────────────────────────────────────────────────────────────
 
@@ -192,6 +193,9 @@ export class UITransformer {
 
     // Priority 7 – Product carousel only for explicit product intent.
     if ((hasProducts || this.isProductQuery(userQuery)) && resolvedIntent.visualizationHint === 'product_browse') {
+      if (!this.isProductQuery(userQuery)) {
+        return this.transformToText(filteredData);
+      }
       return this.transformToProductCarousel(filteredData, config, trainedSchema);
     }
 
@@ -214,6 +218,13 @@ export class UITransformer {
     sources: VectorMatch[],
     llm: ILLMProvider,
   ): Promise<UITransformationResponse> {
+    // ── Step 0: Fast Visualization Decision Engine check ────────────────────
+    const decision = VisualizationDecisionEngine.decideVisualization(query, sources);
+    if (decision.renderType === 'text') {
+      console.debug('[UITransformer] Decision Engine selected plain text rendering — skipping visual LLM calls.');
+      return this.transformToText(sources);
+    }
+
     // ── Step 1: Detect intent ───────────────────────────────────────────────
     let intent: QueryIntent;
     try {
@@ -768,7 +779,7 @@ RULES:
 
   private static transformToText(data: VectorMatch[]): UITransformationResponse {
     return this.createTextResponse(
-      'Search Results',
+      'Retrieved Context',
       data.map(item => item.content).join('\n\n'),
       `Found ${data.length} relevant results`,
     );
@@ -974,8 +985,8 @@ RULES:
   private static isProductData(item: VectorMatch): boolean {
     const content = (item.content || '').toLowerCase();
     const productKeywords = [
-      'product', 'price', 'stock', 'item', 'sku', 'brand', 'model', 'msrp',
-      'inventory', 'buy', 'shop', 'beauty', 'care', 'cosmetic', 'facial',
+      'product', 'price', 'stock', 'sku', 'brand', 'msrp',
+      'inventory', 'buy', 'shop', 'beauty', 'cosmetic', 'facial',
       'cream', 'serum', 'mask', 'makeup', 'fragrance',
     ];
     const hasKeywords = productKeywords.some(kw => content.includes(kw));
@@ -984,7 +995,7 @@ RULES:
     const hasMetadataKey = Object.keys(metadata).some(k => {
       const val = metadata[k];
       return (
-        ['price', 'product', 'sku', 'brand', 'model', 'cost', 'item'].includes(k.toLowerCase()) &&
+        ['price', 'product', 'sku', 'brand', 'cost'].includes(k.toLowerCase()) &&
         val !== null && val !== undefined && val !== ''
       );
     });