@retrivora-ai/rag-engine 1.9.3 → 1.9.6

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -72,6 +72,11 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         ON ${this.uploadTable}
         USING hnsw (embedding vector_cosine_ops)
       `);
+      await client.query(`
+        CREATE INDEX IF NOT EXISTS "${this.uploadTable}_fts_idx"
+        ON "${this.uploadTable}"
+        USING gin (to_tsvector('english', content))
+      `);
 
       const res = await client.query(`
         SELECT table_name 
@@ -158,6 +163,11 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           ON "${tableName}"
           USING hnsw (embedding vector_cosine_ops)
         `);
+        await client.query(`
+          CREATE INDEX IF NOT EXISTS "${tableName}_fts_idx"
+          ON "${tableName}"
+          USING gin (to_tsvector('english', content))
+        `);
         
         // Ensure this new table is added to our search scope
         if (!this.tables.includes(tableName)) {
@@ -297,9 +307,12 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
               const synonyms = FIELD_SYNONYMS[key] ?? [];
               const keysToCheck = [key, ...synonyms];
 
-              // Map each candidate key to LOWER(metadata->>'candidate')
-              const coalesceExprs = keysToCheck.map(k => `LOWER(metadata->>'${k}')`);
-              return `COALESCE(${coalesceExprs.join(', ')}) = LOWER($${paramIdx})`;
+              // Map each candidate key to LOWER(coalesce(metadata->>'key', to_jsonb(t)->>'key'))
+              const coalesceExprs = keysToCheck.flatMap(k => [
+                `metadata->>'${k}'`,
+                `to_jsonb(t)->>'${k}'`
+              ]);
+              return `COALESCE(${coalesceExprs.map(expr => `LOWER(${expr})`).join(', ')}) = LOWER($${paramIdx})`;
             });
           whereClause = `WHERE ${conditions.join(' AND ')}`;
         }
@@ -312,31 +325,50 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
                    CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(', ')})
                    THEN 1.0 ELSE 0.0 END
                  ), 0)
-                 FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
+                 FROM jsonb_each_text(COALESCE(v.metadata, k.metadata)) AS kv(key, val)
                  WHERE key IN (${this.searchFields.map(f => `'${f}'`).join(', ')})
                ) * 3.0`
             : '';
 
-          // Actual Hybrid Search: Semantic Vector Score + Full-Text Keyword Rank
+          // Optimized Subquery CTE Hybrid Search (Uses HNSW vector index & GIN fulltext index)
           sqlQuery = `
-            SELECT *, 
-             (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
-            FROM "${table}" t
-            ${whereClause}
+            WITH vector_search AS (
+              SELECT *, (1 - (embedding <=> $1::vector)) AS vector_score
+              FROM "${table}" t
+              ${whereClause}
+              ORDER BY embedding <=> $1::vector ASC
+              LIMIT ${tableLimit}
+            ),
+            keyword_search AS (
+              SELECT *, COALESCE(ts_rank(to_tsvector('english', content), to_tsquery('english', $2)), 0) AS keyword_score
+              FROM "${table}" t
+              WHERE ${whereClause ? whereClause.replace('WHERE', '') : 'TRUE'}
+                AND to_tsvector('english', content) @@ to_tsquery('english', $2)
+              ORDER BY keyword_score DESC
+              LIMIT ${tableLimit}
+            )
+            SELECT 
+              COALESCE(v.id, k.id) AS id,
+              COALESCE(v.namespace, k.namespace) AS namespace,
+              COALESCE(v.content, k.content) AS content,
+              COALESCE(v.metadata, k.metadata) AS metadata,
+              COALESCE(v.vector_score, 0) AS vector_score,
+              COALESCE(k.keyword_score, 0) AS keyword_score,
+              (COALESCE(v.vector_score, 0) + COALESCE(k.keyword_score, 0) * 2.0 ${exactNameScoreExpr}) AS hybrid_score
+            FROM vector_search v
+            FULL OUTER JOIN keyword_search k ON v.id = k.id
             ORDER BY hybrid_score DESC
             LIMIT ${tableLimit}
           `;
           params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
         } else {
-          // Fallback to Vector-only Search
+          // Fallback to Vector-only Search (ordered ASC to use HNSW index)
           sqlQuery = `
             SELECT *, 
             (1 - (embedding <=> $1::vector)) AS hybrid_score
             FROM "${table}" t
             ${whereClause}
-            ORDER BY hybrid_score DESC
+            ORDER BY embedding <=> $1::vector ASC
             LIMIT ${tableLimit}
           `;
           params = [vectorLiteral, ...filterParams];

package/src/rag/EntityExtractor.ts

@@ -41,7 +41,7 @@ Text to extract from:
       // 1. Try to find the JSON block using a regex
       const jsonMatch = response.match(/\{[\s\S]*\}/);
       const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
-      
+
       return JSON.parse(cleanJson);
     } catch {
       // 2. If parsing fails, try to repair truncated JSON
@@ -62,7 +62,7 @@ Text to extract from:
    */
   private repairTruncatedJson(json: string): string | null {
     let text = json.trim();
-    
+
     // Find the first { and work from there
     const startIdx = text.indexOf('{');
     if (startIdx === -1) return null;

package/src/rag/Reranker.ts

@@ -6,8 +6,8 @@ export class Reranker {
    * Re-ranks matches based on a secondary relevance score using an LLM.
    */
   async rerank(
-    matches: VectorMatch[], 
-    query: string, 
+    matches: VectorMatch[],
+    query: string,
     limit: number = 5,
     llm?: ILLMProvider
   ): Promise<VectorMatch[]> {
@@ -27,7 +27,7 @@ export class Reranker {
       const scoredMatches = matches.map(match => {
         const contentLower = match.content.toLowerCase();
         let keywordScore = 0;
-        
+
         keywords.forEach(keyword => {
           if (contentLower.includes(keyword)) {
             keywordScore += 1;
@@ -55,22 +55,15 @@ export class Reranker {
     try {
       // Take top 10 matches to avoid large context and high latency
       const topN = matches.slice(0, 10);
-      
-      const prompt = `You are a relevance ranking expert.
-Given the following user query and a list of retrieved document chunks, rank the chunks by relevance to the query.
-Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant.
-Use the indices provided in brackets like [0], [1], etc.
-Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.
-
-Query: "${query}"
-
-Documents:
-${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, ' ')}`).join('\n')}`;
 
       const response = await llm.chat(
-        [{ role: 'user', content: prompt }],
+        [{ role: 'user', content: `Query: "${query}"\n\nDocuments:\n${topN.map((m, i) => `[${i}] ${m.content.replace(/\n/g, ' ')}`).join('\n')}` }],
         '',
-        { temperature: 0, maxTokens: 50 }
+        {
+          systemPrompt: 'You are a relevance ranking expert. Given the user query and a list of retrieved document chunks, rank the chunks by relevance to the query. Return the result as a comma-separated list of document indices in order of relevance, from most relevant to least relevant. Use the indices provided in brackets like [0], [1], etc. Only return the indices (e.g., "2,0,1"), nothing else. Do not include brackets in the output.',
+          temperature: 0,
+          maxTokens: 50
+        }
       );
 
       const cleanedResponse = response.trim().replace(/[\[\]\s]/g, '');

package/src/server.ts

@@ -6,7 +6,20 @@
  */
 
 // ── Shared Types ──────────────────────────────────────────────
-export type { RagConfig, VectorDBConfig, LLMConfig, EmbeddingConfig, UIConfig, RAGConfig, VectorDBProvider, LLMProvider, EmbeddingProvider } from './config/RagConfig';
+export type {
+  RagConfig,
+  UniversalRagConfig,
+  RetrievalConfig,
+  WorkflowConfig,
+  VectorDBConfig,
+  LLMConfig,
+  EmbeddingConfig,
+  UIConfig,
+  RAGConfig,
+  VectorDBProvider,
+  LLMProvider,
+  EmbeddingProvider,
+} from './config/RagConfig';
 export type { VectorMatch, UpsertDocument, IngestDocument, ChatResponse, ChatMessage, ChatOptions } from './types';
 export type { ILLMProvider, EmbedOptions } from './llm/ILLMProvider';
 export type { Chunk, ChunkOptions } from './rag/DocumentChunker';
@@ -16,6 +29,7 @@ export type { BatchOptions, BatchResult } from './core/BatchProcessor';
 
 // ── Core Orchestration ─────────────────────────────────────────
 export { VectorPlugin } from './core/VectorPlugin';
+export { Retrivora } from './core/Retrivora';
 export { Pipeline } from './core/Pipeline';
 export { ConfigResolver } from './core/ConfigResolver';
 export { ProviderRegistry } from './core/ProviderRegistry';
@@ -23,6 +37,18 @@ export { ConfigValidator } from './core/ConfigValidator';
 export { ProviderHealthCheck } from './core/ProviderHealthCheck';
 export { BatchProcessor } from './core/BatchProcessor';
 
+// ── Exceptions ────────────────────────────────────────────────
+export {
+  RetrivoraError,
+  ProviderNotFoundException,
+  EmbeddingFailedException,
+  RetrievalException,
+  RateLimitException,
+  ConfigurationException,
+  AuthenticationException,
+} from './exceptions';
+export type { RetrivoraErrorCode } from './exceptions';
+
 // ── Configuration Helpers ──────────────────────────────────────
 export { getRagConfig } from './config/serverConfig';
 export { ConfigBuilder, PRESETS, createFromPreset } from './config/ConfigBuilder';

package/src/types/chat.ts

@@ -14,6 +14,13 @@ export interface RagMessage extends ChatMessage {
   /** Full observability trace emitted by the backend. Only present on assistant messages. */
   trace?: import('./index').ObservabilityTrace;
   createdAt: string;
+  /**
+   * Chain-of-thought reasoning from the LLM.
+   * Populated by native Anthropic extended thinking or simulated <think> tag stripping.
+   */
+  thinking?: string;
+  /** Wall-clock ms the model spent in the thinking phase. */
+  thinkingMs?: number;
 }
 
 export interface ChatOptions {

package/src/utils/UITransformer.ts

@@ -217,7 +217,7 @@ export class UITransformer {
     // ── Step 1: Detect intent ───────────────────────────────────────────────
     let intent: QueryIntent;
     try {
-      intent = await this.detectIntent(query, llm);
+      intent = await this.detectIntent(query, llm, sources);
       console.debug('[UITransformer] Detected intent:', intent);
     } catch (err) {
       console.warn('[UITransformer] Intent detection failed; using heuristic.', err);
@@ -239,11 +239,24 @@ export class UITransformer {
       const context = this.buildContextSummary(sources);
       const systemPrompt = this.buildVisualizationSystemPrompt();
 
+      const profile = this.profileData(sources);
+      const schemaContext = `
+RETRIEVED DATA SCHEMA:
+- Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Text/Other fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
+`;
+
       const userPrompt = [
         `USER QUESTION: ${query}`,
         '',
         `DETECTED INTENT (JSON): ${JSON.stringify(intent)}`,
         '',
+        `RETRIEVED DATA SCHEMA:`,
+        schemaContext,
+        '',
         'RETRIEVED DATA (JSON):',
         context,
       ].join('\n');
@@ -256,6 +269,26 @@ export class UITransformer {
 
       const parsed = this.parseTransformationResponse(rawResponse);
       if (parsed) {
+        // Validate suitability of selected chart type based on schema
+        let isCompatible = true;
+        if (parsed.type === 'line_chart' && (profile.dateFields.length === 0 || profile.numericFields.length === 0)) {
+          isCompatible = false;
+        } else if (
+          ['bar_chart', 'horizontal_bar', 'pie_chart', 'donut_chart'].includes(parsed.type) &&
+          (profile.categoricalFields.length === 0 && profile.numericFields.length === 0)
+        ) {
+          isCompatible = false;
+        } else if (parsed.type === 'scatter_plot' && profile.numericFields.length < 2) {
+          isCompatible = false;
+        } else if (parsed.type === 'metric_card' && profile.numericFields.length === 0) {
+          isCompatible = false;
+        }
+
+        if (!isCompatible) {
+          console.warn(`[UITransformer] LLM chose incompatible visualization type "${parsed.type}" for retrieved schema. Falling back to heuristic.`);
+          return this.transform(query, sources, undefined, undefined, intent);
+        }
+
         const intentAllowsTable =
           intent.wantsExplicitTable ||
           ['tabular', 'table', 'geographic'].includes(intent.visualizationHint);
@@ -295,7 +328,27 @@ export class UITransformer {
    * - The intent object can be reused across both the heuristic and LLM paths.
    * - It is easy to unit-test intent detection in isolation.
    */
-  static async detectIntent(query: string, llm: ILLMProvider): Promise<QueryIntent> {
+  static async detectIntent(query: string, llm: ILLMProvider, sources?: VectorMatch[]): Promise<QueryIntent> {
+    let schemaProfileText = '';
+    if (sources && sources.length > 0) {
+      const profile = this.profileData(sources);
+      const hasProducts = sources.some(item => this.isProductData(item));
+      schemaProfileText = `
+RETRIEVED DATA SCHEMA:
+- Numeric fields (measures): ${profile.numericFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Category fields (dimensions): ${profile.categoricalFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Date fields (temporal): ${profile.dateFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Boolean fields: ${profile.booleanFields.map(f => `${f.key} (unique values: ${f.uniqueCount})`).join(', ') || 'None'}
+- Text fields: ${profile.fields.filter(f => f.kind === 'text').map(f => f.key).join(', ') || 'None'}
+
+Please choose visualizationHint and recommendedChart dynamically based on the available schema.
+- Do NOT recommend "trend" or "line_chart" if there are no Date fields.
+- Do NOT recommend numeric-dependent visualizations (like bar_chart, line_chart, pie_chart, scatter_plot, histogram, metric_card) if there are no Numeric fields. Fall back to tabular/table or text.
+- If there are multiple metadata fields but no clear categories or numbers, "tabular"/"table" is a highly accurate default.
+${hasProducts ? `- Note: The retrieved data matches product/catalog structure. If the user query is looking for, exploring, searching, browsing, or listing products/items, you MUST recommend visualizationHint: "product_browse" and recommendedChart: "text".` : ''}
+`;
+    }
+
     const systemPrompt = `You are an intent classifier for a product-search RAG system.
 Given a user query, return ONLY a valid JSON object with this exact shape — no prose, no markdown:
 
@@ -332,8 +385,10 @@ RULES:
 - isComparison: true if user compares, ranks, or contrasts entities or categories.
 - language: detect from the query text itself; default "en" if uncertain.`;
 
+    const userPrompt = `QUERY: ${query}${schemaProfileText ? `\n\n${schemaProfileText}` : ''}`;
+
     const rawResponse = await llm.chat(
-      [{ role: 'user', content: `QUERY: ${query}` }],
+      [{ role: 'user', content: userPrompt }],
       '',
       { systemPrompt, temperature: 0 },
     );
@@ -894,14 +949,28 @@ RULES:
     return asksForRecords && (hasNumericPredicate || hasEntityTerms);
   }
 
+  /** @internal kept private for transform() internal logic */
   private static isProductQuery(query: string): boolean {
+    return UITransformer._productQueryTest(query);
+  }
+
+  /**
+   * Public entry-point for external callers (e.g. Pipeline)
+   * to check whether a query maps to product_browse without triggering an LLM call.
+   */
+  static isProductQueryPublic(query: string): boolean {
+    return UITransformer._productQueryTest(query);
+  }
+
+  private static _productQueryTest(query: string): boolean {
     const q = query.toLowerCase();
     const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
-    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list)\b/.test(q);
+    const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
     const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
     return productTerms && productAction && !nonProductEntityTerms;
   }
 
+
   private static isProductData(item: VectorMatch): boolean {
     const content = (item.content || '').toLowerCase();
     const productKeywords = [

package/dist/DocumentChunker-Dh9TvmGG.d.mts

@@ -1,45 +0,0 @@
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string | number;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-    /** Characters used to split text, in order of priority */
-    separators?: string[];
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    private readonly separators;
-    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
-    /**
-     * Split a single text string into overlapping chunks using a recursive strategy.
-     * Preserves structural boundaries (Markdown headers) where possible.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    private recursiveSplit;
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string | number;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { type Chunk as C, DocumentChunker as D, type ChunkOptions as a };

package/dist/DocumentChunker-Dh9TvmGG.d.ts

@@ -1,45 +0,0 @@
-/**
- * DocumentChunker — splits long text into overlapping chunks
- * suitable for vector embedding and retrieval.
- *
- * Strategy: sentence-aware sliding window
- *   1. Split on sentence boundaries
- *   2. Accumulate sentences into chunks up to `chunkSize` characters
- *   3. Carry over `chunkOverlap` characters from the previous chunk
- */
-interface Chunk {
-    id: string;
-    content: string;
-    metadata?: Record<string, unknown>;
-}
-interface ChunkOptions {
-    /** Target chunk size in characters (default 1000) */
-    chunkSize?: number;
-    /** Overlap between consecutive chunks in characters (default 200) */
-    chunkOverlap?: number;
-    /** Source document identifier used as ID prefix */
-    docId?: string | number;
-    /** Extra metadata to attach to every chunk */
-    metadata?: Record<string, unknown>;
-    /** Characters used to split text, in order of priority */
-    separators?: string[];
-}
-declare class DocumentChunker {
-    private readonly chunkSize;
-    private readonly chunkOverlap;
-    private readonly separators;
-    constructor(chunkSize?: number, chunkOverlap?: number, separators?: string[]);
-    /**
-     * Split a single text string into overlapping chunks using a recursive strategy.
-     * Preserves structural boundaries (Markdown headers) where possible.
-     */
-    chunk(text: string, options?: ChunkOptions): Chunk[];
-    private recursiveSplit;
-    chunkMany(documents: Array<{
-        content: string;
-        docId?: string | number;
-        metadata?: Record<string, unknown>;
-    }>): Chunk[];
-}
-
-export { type Chunk as C, DocumentChunker as D, type ChunkOptions as a };