@retrivora-ai/rag-engine 1.9.1 → 1.9.3

package/src/core/Pipeline.ts

@@ -354,7 +354,10 @@ You are a helpful assistant. Use the provided context to answer questions accura
     );
 
     if (upsertResult.errors.length > 0) {
-      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
+      console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed. Error details:`);
+      upsertResult.errors.forEach((err, idx) => {
+        console.warn(`  Batch ${idx + 1} Error: ${err.error.message || String(err.error)}`);
+      });
     }
 
     return upsertResult.totalProcessed;
@@ -482,8 +485,9 @@ You are a helpful assistant. Use the provided context to answer questions accura
         ? await this.graphDB.query(searchQuery)
         : undefined;
 
+      const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
       const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
-      const wantsExhaustiveList = hasNumericPredicates || /\b(list|all|show|provide|give|get)\b/i.test(question);
+      const wantsExhaustiveList = hasNumericPredicates || hasMetadataFilter || /\b(list|all|show|provide|give|get|browse|find|search|display)\b/i.test(question);
       const retrievalLimit = wantsExhaustiveList ? Math.max(topK * 20, 100) : topK * 2;
 
       const rawSources = (strategyResult === 'vector' || strategyResult === 'both') && queryVector && queryVector.length > 0
@@ -497,21 +501,44 @@ You are a helpful assistant. Use the provided context to answer questions accura
       // 3. Reranking
       const rerankStart = performance.now();
       const structuredSources = this.applyStructuredFilters(rawSources, filter);
-      let sources = hasNumericPredicates
+      let fullSources = hasNumericPredicates
         ? structuredSources
         : structuredSources.filter((m) => m.score >= scoreThreshold);
+      const rerankLimit = wantsExhaustiveList ? retrievalLimit : topK;
       if (!hasNumericPredicates && this.config.rag?.useReranking) {
-        sources = await this.reranker.rerank(sources, question, topK);
+        fullSources = await this.reranker.rerank(fullSources, question, rerankLimit);
       } else if (!wantsExhaustiveList) {
-        sources = sources.slice(0, topK);
+        fullSources = fullSources.slice(0, topK);
       }
       const rerankMs = performance.now() - rerankStart;
 
-      // 4. Context Augmentation
-      let context = sources.length
-        ? sources.map((m, i) => `[Source ${i + 1}]\n${m.content}`).join('\n\n---\n\n')
+      // 4. Context Augmentation - Review all retrieved/reranked sources for full comprehension
+      let context = fullSources.length
+        ? fullSources.map((m, i) => `[Source ${i + 1}]\n${m.content}`).join('\n\n---\n\n')
         : 'No relevant context found.';
 
+      // Count the records of relevant sources to pass into the slice dynamically
+      let displayCount = 15; // default fallback
+
+      if (hasMetadataFilter) {
+        // If a metadata filter matches (e.g. brand, category), all of those matches are highly relevant
+        displayCount = fullSources.length;
+      } else {
+        // For general semantic queries, count the number of matches meeting high relevance threshold
+        const baseThreshold = Math.max(scoreThreshold, 0.4);
+        const highlyRelevant = fullSources.filter(m => m.score >= baseThreshold);
+        displayCount = Math.max(highlyRelevant.length, topK);
+        // Cap non-metadata semantic searches to a maximum of 15 to keep the UI clean
+        if (displayCount > 15) {
+          displayCount = 15;
+        }
+      }
+
+      // Ensure they are sorted from highest to lowest score, then slice using our dynamic count
+      const sources = [...fullSources]
+        .sort((a, b) => b.score - a.score)
+        .slice(0, displayCount);
+
       if (graphData && graphData.nodes.length > 0) {
         const graphContext = graphData.nodes.map(n =>
           `Entity: ${n.label} (${n.id})${n.properties ? ' - ' + JSON.stringify(n.properties) : ''}`
@@ -525,8 +552,21 @@ You are a helpful assistant. Use the provided context to answer questions accura
         ? SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => undefined)
         : Promise.resolve(undefined);
 
+      // 4.6 Start UI Transformation concurrently with generation to drastically reduce latency
+      const uiTransformationPromise = trainedSchemaPromise.then(trainedSchema =>
+        this.generateUiTransformation(
+          question,
+          sources,
+          trainedSchema,
+          hasNumericPredicates,
+        )
+      ).catch(uiError => {
+        console.warn('[Pipeline] UI transformation failed concurrently:', uiError);
+        return UITransformer.transform(question, sources, this.config);
+      });
+
       // 5. Generation (Streaming)
-      const restrictionSuffix = '\n\n(IMPORTANT: Use plain text only. NEVER generate tables, HTML figures, or text charts. The UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization. If listing products, use simple bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)';
+      const restrictionSuffix = '\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)';
       const hardenedHistory = [...history];
       const userQuestion = { role: 'user', content: question + restrictionSuffix } as ChatMessage;
       const messages: ChatMessage[] = [...hardenedHistory, userQuestion];
@@ -592,19 +632,7 @@ You are a helpful assistant. Use the provided context to answer questions accura
         timestamp: new Date().toISOString(),
       };
 
-      let uiTransformation: unknown;
-      try {
-        const trainedSchema = await trainedSchemaPromise;
-        uiTransformation = await this.generateUiTransformation(
-          question,
-          sources,
-          trainedSchema,
-          hasNumericPredicates,
-        );
-      } catch (uiError) {
-        console.warn('[Pipeline] UI transformation failed before metadata yield:', uiError);
-        uiTransformation = UITransformer.transform(question, sources, this.config);
-      }
+      const uiTransformation = await uiTransformationPromise;
 
       yield {
         reply: '',
@@ -804,9 +832,9 @@ You are a helpful assistant. Use the provided context to answer questions accura
 
     const sources = (strategy === 'vector' || strategy === 'both') && queryVector && queryVector.length > 0
       ? this.applyStructuredFilters(
-          await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
-          baseFilter,
-        )
+        await this.vectorDB.query(queryVector, retrievalLimit, ns, baseFilter),
+        baseFilter,
+      )
       : [];
 
     // Multi-Vector Support: Check for parent_id in metadata

package/src/core/QueryProcessor.ts

@@ -156,6 +156,7 @@ export class QueryProcessor {
       new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
       new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
       new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
+      new RegExp(`\\b(?:under|in|for|from|of)\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
     ];
 
     for (const pattern of fieldValuePatterns) {
@@ -301,12 +302,12 @@ export class QueryProcessor {
    * Determines the retrieval strategy based on the query content.
    */
   static async determineRetrievalStrategy(
-    query: string, 
+    query: string,
     llm?: import('../llm/ILLMProvider').ILLMProvider,
     customGraphKeywords?: string[],
     customVectorKeywords?: string[]
   ): Promise<'vector' | 'graph' | 'both'> {
-    
+
     if (llm) {
       try {
         const prompt = `You are a routing expert in a RAG system.
@@ -336,29 +337,29 @@ Return ONLY 'vector', 'graph', or 'both'. No explanation.`;
 
     // Fallback to keywords
     const normalized = query.toLowerCase();
-    
+
     // Graph keywords
     const graphKeywords = customGraphKeywords || [
-      'relationship', 'connect', 'link', 'network', 'friend', 'colleague', 
+      'relationship', 'connect', 'link', 'network', 'friend', 'colleague',
       'manager', 'hierarchy', 'multi-hop', 'shortest path', 'between',
       'related to', 'associated with', 'belongs to', 'part of'
     ];
-    
+
     const hasGraphKeyword = graphKeywords.some(kw => normalized.includes(kw));
-    
+
     // Vector keywords (usually generic search or content specific)
     const vectorKeywords = customVectorKeywords || [
       'find', 'search', 'tell me about', 'what is', 'how to', 'documents about'
     ];
-    
+
     const hasVectorKeyword = vectorKeywords.some(kw => normalized.includes(kw));
-    
+
     if (hasGraphKeyword && !hasVectorKeyword) {
       return 'graph';
     } else if (hasGraphKeyword && hasVectorKeyword) {
       return 'both';
     }
-    
+
     return 'vector';
   }
 }

package/src/handlers/index.ts

@@ -142,15 +142,24 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
     }
 
     const encoder = new TextEncoder();
+    let isActive = true;
 
     const stream = new ReadableStream({
       async start(controller) {
-        const enqueue = (text: string) => controller.enqueue(encoder.encode(text));
+        const enqueue = (text: string) => {
+          if (!isActive) return;
+          try {
+            controller.enqueue(encoder.encode(text));
+          } catch (err) {
+            console.warn('[createStreamHandler] Failed to enqueue (stream already closed):', err);
+          }
+        };
 
         try {
           const pipelineStream = plugin.chatStream(message, history, namespace);
 
           for await (const chunk of pipelineStream) {
+            if (!isActive) break;
             if (typeof chunk === 'string') {
               enqueue(sseTextFrame(chunk));
             } else {
@@ -166,7 +175,6 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
                 enqueue(sseObservabilityFrame(responseChunk.trace));
               }
 
-
               if (sources.length > 0) {
                 try {
                   // The pipeline already computed ui_transformation inside askStream().
@@ -187,18 +195,31 @@ export function createStreamHandler(configOrPlugin?: Partial<RagConfig> | Vector
                   } catch { /* silent */ }
                 }
               }
-
             }
           }
         } catch (streamError) {
-          const errorMessage =
-            streamError instanceof Error ? streamError.message : String(streamError);
-          console.error('[createStreamHandler] Stream error:', streamError);
-          enqueue(sseErrorFrame(errorMessage));
+          // Do not log or enqueue errors if the client intentionally cancelled/aborted
+          if (isActive) {
+            const errorMessage =
+              streamError instanceof Error ? streamError.message : String(streamError);
+            console.error('[createStreamHandler] Stream error:', streamError);
+            try {
+              enqueue(sseErrorFrame(errorMessage));
+            } catch { /* silent */ }
+          }
         } finally {
-          controller.close();
+          if (isActive) {
+            isActive = false;
+            try {
+              controller.close();
+            } catch { /* silent */ }
+          }
         }
       },
+      cancel(reason) {
+        isActive = false;
+        console.log('[createStreamHandler] Stream connection closed by client:', reason);
+      }
     });
 
     return new Response(stream, { headers: SSE_HEADERS });
@@ -292,6 +313,7 @@ export function createUploadHandler(configOrPlugin?: Partial<RagConfig> | Vector
           if (parsed.data && parsed.data.length > 0) {
             let i = 0;
             let lastRowData: Record<string, string> | null = null;
+            const csvCols = Object.keys(parsed.data[0] as Record<string, string>);
             
             for (const row of parsed.data) {
               i++;
@@ -328,6 +350,7 @@ export function createUploadHandler(configOrPlugin?: Partial<RagConfig> | Vector
                   fileType: file.type,
                   uploadedAt: new Date().toISOString(),
                   ...(dimension ? { dimension } : {}),
+                  csvHeaders: csvCols,
                   ...rowData
                 }
               });

package/src/hooks/useRagChat.ts

@@ -74,6 +74,7 @@ export function useRagChat(
   const [error, setError] = useState<string | null>(null);
   const lastInputRef = useRef<string>('');
   const messagesRef = useRef<RagMessage[]>(messages);
+  const abortControllerRef = useRef<AbortController | null>(null);
 
   useEffect(() => {
     messagesRef.current = messages;
@@ -100,12 +101,21 @@ export function useRagChat(
       setError(null);
       setIsLoading(true);
 
+      // Abort any existing call
+      if (abortControllerRef.current) {
+        abortControllerRef.current.abort();
+      }
+
+      const controller = new AbortController();
+      abortControllerRef.current = controller;
+
       try {
         const history = messagesRef.current.map(({ role, content }) => ({ role, content }));
 
         const response = await fetch(apiUrl, {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
+          signal: controller.signal,
           body: JSON.stringify({
             message: trimmed,
             history,
@@ -245,11 +255,16 @@ export function useRagChat(
         );
         onReply?.(finalMsg);
       } catch (err) {
+        if (err instanceof Error && err.name === 'AbortError') {
+          console.log('[useRagChat] Streaming stopped by user.');
+          return;
+        }
         const msg = err instanceof Error ? err.message : 'Something went wrong. Please try again.';
         setError(msg);
         onError?.(msg);
       } finally {
         setIsLoading(false);
+        abortControllerRef.current = null;
       }
     },
     [apiUrl, isLoading, namespace, onError, onReply, projectId, setMessages]
@@ -270,6 +285,14 @@ export function useRagChat(
     }
   }, [sendMessage]);
 
+  const stop = useCallback(() => {
+    if (abortControllerRef.current) {
+      abortControllerRef.current.abort();
+      abortControllerRef.current = null;
+    }
+    setIsLoading(false);
+  }, []);
+
   return {
     messages,
     isLoading,
@@ -278,5 +301,6 @@ export function useRagChat(
     clear,
     retry,
     setMessages,
+    stop,
   };
 }

package/src/providers/vectordb/MongoDBProvider.ts

@@ -191,12 +191,19 @@ export class MongoDBProvider extends BaseVectorProvider {
 
     const results = await this.collection!.aggregate(pipeline).toArray();
 
-    return (results as unknown as Array<{ _id: string; score: number; [key: string]: unknown }>).map((res) => ({
-      id: res._id,
-      content: res[this.contentKey] as string,
-      metadata: (res[this.metadataKey] as Record<string, unknown>) || {},
-      score: res.score,
-    }));
+    return (results as unknown as Array<{ _id: string; score: number; [key: string]: unknown }>).map((res) => {
+      // Normalize MongoDB's vectorSearchScore (which is shifted to [0, 1] e.g. (1+cos)/2)
+      // back to the standard [-1, 1] cosine space. This ensures the global relevance 
+      // threshold (e.g. 0.4) properly filters out random orthogonal matches.
+      const normalizedScore = (res.score * 2) - 1;
+
+      return {
+        id: res._id,
+        content: res[this.contentKey] as string,
+        metadata: (res[this.metadataKey] as Record<string, unknown>) || {},
+        score: normalizedScore,
+      };
+    });
   }
 
   async delete(id: string | number, namespace?: string): Promise<void> {

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -5,6 +5,7 @@ import { Pool, PoolClient } from 'pg';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
 import { VectorDBConfig } from '../../config/RagConfig';
+import { FIELD_SYNONYMS } from '../../utils/synonyms';
 
 
 /**
@@ -129,8 +130,13 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         
         // Extract headers from the first doc's metadata
         const firstMeta = fileDocs[0].metadata || {};
-        const systemKeys = ['fileName', 'fileSize', 'fileType', 'uploadedAt', 'dimension', 'chunkIndex', 'id', 'namespace', 'content', 'metadata', 'embedding'];
-        const csvHeaders = Object.keys(firstMeta).filter(k => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        let csvHeaders: string[] = [];
+        if (Array.isArray(firstMeta.csvHeaders)) {
+          csvHeaders = firstMeta.csvHeaders;
+        } else {
+          const systemKeys = ['fileName', 'fileSize', 'fileType', 'uploadedAt', 'dimension', 'chunkIndex', 'id', 'namespace', 'content', 'metadata', 'embedding', 'docId', 'docid', 'chunkId', 'chunkid', 'csvHeaders', 'csvheaders'];
+          csvHeaders = Object.keys(firstMeta).filter(k => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        }
         
         // 1. Create Table Dynamically
         const columnDefs = csvHeaders.map(h => `"${h}" TEXT`).join(',\n          ');
@@ -269,10 +275,34 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     const dynamicKeywordQuery = getDynamicKeywordQuery();
     const tableLimit = Math.max(topK, 50);
 
+    const metadataFilters = _filter?.metadata as Record<string, string> | undefined;
+
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = '';
         let params: unknown[] = [];
+        let whereClause = '';
+        const filterParams: unknown[] = [];
+
+        // Build case-insensitive SQL metadata filter conditions with synonym-awareness
+        if (metadataFilters && Object.keys(metadataFilters).length > 0) {
+          const conditions = Object.entries(metadataFilters)
+            .map(([key, val]) => {
+              filterParams.push(String(val));
+              // Offset based on whether it is vector + keyword or vector only
+              const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
+              const paramIdx = baseOffset + filterParams.length;
+
+              // Check if we have synonyms for this key (e.g. 'brand' -> 'vendor', 'manufacturer')
+              const synonyms = FIELD_SYNONYMS[key] ?? [];
+              const keysToCheck = [key, ...synonyms];
+
+              // Map each candidate key to LOWER(metadata->>'candidate')
+              const coalesceExprs = keysToCheck.map(k => `LOWER(metadata->>'${k}')`);
+              return `COALESCE(${coalesceExprs.join(', ')}) = LOWER($${paramIdx})`;
+            });
+          whereClause = `WHERE ${conditions.join(' AND ')}`;
+        }
 
         if (queryText && dynamicKeywordQuery) {
           const hasEntityHints = entityHints.length > 0;
@@ -294,20 +324,22 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
              COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
              ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
+            ${whereClause}
             ORDER BY hybrid_score DESC
             LIMIT ${tableLimit}
           `;
-          params = [vectorLiteral, dynamicKeywordQuery];
+          params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
         } else {
           // Fallback to Vector-only Search
           sqlQuery = `
             SELECT *, 
             (1 - (embedding <=> $1::vector)) AS hybrid_score
             FROM "${table}" t
+            ${whereClause}
             ORDER BY hybrid_score DESC
             LIMIT ${tableLimit}
           `;
-          params = [vectorLiteral];
+          params = [vectorLiteral, ...filterParams];
         }
 
         const result = await this.pool.query(sqlQuery, params);

package/src/types/chat.ts

@@ -59,4 +59,6 @@ export interface UseRagChatReturn {
   retry: () => Promise<void>;
   /** Programmatically set the conversation (e.g. to restore from a DB) */
   setMessages: React.Dispatch<React.SetStateAction<RagMessage[]>>;
+  /** Abort the current active stream request */
+  stop: () => void;
 }

package/src/utils/SchemaMapper.ts

@@ -57,14 +57,16 @@ Given these metadata keys from a database: [${keys.join(', ')}]
 Identify which keys best correspond to these standard UI properties:
 ${propertyList}
 
-Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys. 
+Return ONLY a valid JSON object where the keys are the UI properties and the values are the EXACT matching database keys from the list above. 
 If no good match is found for a property, omit it.
 
 Example:
 {
-  "name": "Product_Title",
-  "price": "MSRP_USD",
-  "brand": "VendorName"
+  "name": "Title",
+  "price": "Variant Price",
+  "brand": "Vendor",
+  "image": "Image Src",
+  "stock": "Variant Inventory Qty"
 }
 `;
 

package/src/utils/UITransformer.ts

@@ -483,7 +483,8 @@ RULES:
   ): UITransformationResponse {
     const products: CarouselProduct[] = data
       .map(item => this.extractProductInfo(item, config, trainedSchema))
-      .filter((p): p is CarouselProduct => p !== null);
+      .filter((p): p is CarouselProduct => p !== null)
+      .slice(0, 15);
 
     return {
       type: 'product_carousel',

package/src/utils/synonyms.ts

@@ -4,15 +4,15 @@
  */
 export const FIELD_SYNONYMS: Record<string, string[]> = {
   name:        ['product', 'item', 'title', 'label', 'heading', 'subject', 'product name', 'item name'],
-  price:       ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd'],
+  price:       ['cost', 'amount', 'msrp', 'price', 'rate', 'value', 'price_usd', 'variant price'],
   brand:       ['manufacturer', 'vendor', 'make', 'company', 'brand_name', 'supplier'],
   image:       ['imageUrl', 'thumbnail', 'img', 'url', 'photo', 'picture', 'media', 'image_url',
-                'main_image', 'product_image', 'thumb'],
+                'main_image', 'product_image', 'thumb', 'image src', 'variant image'],
   stock:       ['inventory', 'quantity', 'count', 'availability', 'stock_level', 'inStock',
-                'is_available', 'in stock', 'status'],
+                'is_available', 'in stock', 'status', 'variant inventory qty'],
   category:    ['product_category', 'product category', 'category_name', 'category name',
-                'department', 'collection'],
-  description: ['summary', 'content', 'body', 'text', 'info', 'details'],
+                'department', 'collection', 'type'],
+  description: ['summary', 'content', 'body', 'text', 'info', 'details', 'body (html)', 'seo description'],
   link:        ['url', 'href', 'product_url', 'page_url', 'link'],
 };