@retrivora-ai/rag-engine 1.9.0 → 1.9.2

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -5,6 +5,7 @@ import { Pool, PoolClient } from 'pg';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
 import { VectorDBConfig } from '../../config/RagConfig';
+import { FIELD_SYNONYMS } from '../../utils/synonyms';
 
 
 /**
@@ -129,8 +130,13 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         
         // Extract headers from the first doc's metadata
         const firstMeta = fileDocs[0].metadata || {};
-        const systemKeys = ['fileName', 'fileSize', 'fileType', 'uploadedAt', 'dimension', 'chunkIndex', 'id', 'namespace', 'content', 'metadata', 'embedding'];
-        const csvHeaders = Object.keys(firstMeta).filter(k => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        let csvHeaders: string[] = [];
+        if (Array.isArray(firstMeta.csvHeaders)) {
+          csvHeaders = firstMeta.csvHeaders;
+        } else {
+          const systemKeys = ['fileName', 'fileSize', 'fileType', 'uploadedAt', 'dimension', 'chunkIndex', 'id', 'namespace', 'content', 'metadata', 'embedding', 'docId', 'docid', 'chunkId', 'chunkid', 'csvHeaders', 'csvheaders'];
+          csvHeaders = Object.keys(firstMeta).filter(k => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
+        }
         
         // 1. Create Table Dynamically
         const columnDefs = csvHeaders.map(h => `"${h}" TEXT`).join(',\n          ');
@@ -235,12 +241,9 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
 
     const queryText = _filter?.queryText as string | undefined;
-    const entityHints = Array.isArray(_filter?.__entityHints)
-      ? (_filter.__entityHints as unknown[])
-        .filter((hint): hint is { value: string; field?: string } =>
-          typeof hint === 'object' && hint !== null && typeof (hint as Record<string, unknown>).value === 'string'
-        )
-        .map((hint) => hint.value.trim().toLowerCase())
+    const entityHints = Array.isArray(_filter?.keywords)
+      ? (_filter.keywords as string[])
+        .map((k) => k.trim().toLowerCase())
         .filter(Boolean)
       : [];
 
@@ -252,15 +255,17 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       if (entityHints.length > 0) {
         // Use extracted entities as high-signal keywords
         return entityHints
-          .map(h => h.includes(' ') ? `"${h}"` : h) // Wrap multi-word hints in quotes
+          .map(h => h.replace(/\s+/g, ' & ')) // Replace spaces with & for valid tsquery
           .join(' & ');
       }
 
       if (queryText) {
         // Fallback to cleaned raw text if no high-signal hints were extracted
+        // Added 'get', 'details', 'information', 'info' to stop words
         return queryText
           .toLowerCase()
-          .replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, '')
+          .replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, '')
+          .replace(/\b\d+(?:\.\d+)?\b/g, '')
           .trim()
           .replace(/\s+/g, ' & ');
       }
@@ -268,13 +273,38 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     };
 
     const dynamicKeywordQuery = getDynamicKeywordQuery();
+    const tableLimit = Math.max(topK, 50);
+
+    const metadataFilters = _filter?.metadata as Record<string, string> | undefined;
 
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = '';
         let params: unknown[] = [];
+        let whereClause = '';
+        const filterParams: unknown[] = [];
+
+        // Build case-insensitive SQL metadata filter conditions with synonym-awareness
+        if (metadataFilters && Object.keys(metadataFilters).length > 0) {
+          const conditions = Object.entries(metadataFilters)
+            .map(([key, val]) => {
+              filterParams.push(String(val));
+              // Offset based on whether it is vector + keyword or vector only
+              const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
+              const paramIdx = baseOffset + filterParams.length;
+
+              // Check if we have synonyms for this key (e.g. 'brand' -> 'vendor', 'manufacturer')
+              const synonyms = FIELD_SYNONYMS[key] ?? [];
+              const keysToCheck = [key, ...synonyms];
+
+              // Map each candidate key to LOWER(metadata->>'candidate')
+              const coalesceExprs = keysToCheck.map(k => `LOWER(metadata->>'${k}')`);
+              return `COALESCE(${coalesceExprs.join(', ')}) = LOWER($${paramIdx})`;
+            });
+          whereClause = `WHERE ${conditions.join(' AND ')}`;
+        }
 
-        if (queryText) {
+        if (queryText && dynamicKeywordQuery) {
           const hasEntityHints = entityHints.length > 0;
           const exactNameScoreExpr = hasEntityHints
             ? `+ (
@@ -294,20 +324,22 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
              COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
              ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
+            ${whereClause}
             ORDER BY hybrid_score DESC
-            LIMIT 50
+            LIMIT ${tableLimit}
           `;
-          params = [vectorLiteral, dynamicKeywordQuery];
+          params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
         } else {
           // Fallback to Vector-only Search
           sqlQuery = `
             SELECT *, 
-             (1 - (embedding <=> $1::vector)) AS hybrid_score
+            (1 - (embedding <=> $1::vector)) AS hybrid_score
             FROM "${table}" t
+            ${whereClause}
             ORDER BY hybrid_score DESC
-            LIMIT 50
+            LIMIT ${tableLimit}
           `;
-          params = [vectorLiteral];
+          params = [vectorLiteral, ...filterParams];
         }
 
         const result = await this.pool.query(sqlQuery, params);

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -170,7 +170,7 @@ export class PostgreSQLProvider extends BaseVectorProvider {
       const filterConditions = Object.entries(publicFilter)
         .map(([key, val]) => {
           const paramIdx = params.length + 1;
-          params.push(JSON.stringify(val));
+          params.push(String(val));
           return `metadata->>'${key}' = $${paramIdx}`;
         })
         .join(' AND ');

package/src/providers/vectordb/QdrantProvider.ts

@@ -211,7 +211,7 @@ export class QdrantProvider extends BaseVectorProvider {
       limit: topK,
       with_payload: true,
       params: {
-        hnsw_ef: (this.config.options.efSearch as number) || Math.max(topK * 20, 128),
+        hnsw_ef: ((this.config.options as Record<string, unknown>)?.efSearch as number | undefined) || Math.max(topK * 20, 128),
         exact: false
       },
       filter: must.length > 0 ? { must } : undefined,

package/src/providers/vectordb/RedisProvider.ts

@@ -90,16 +90,15 @@ export class RedisProvider extends BaseVectorProvider {
   }
 
   /**
-   * Redis is TCP-based and has no HTTP health endpoint.
-   * Returns true; actual connectivity is validated on the first operation.
+   * Check reachability via a PING command.
+   * Returns false on connection failure so health checks surface real problems.
    */
   async ping(): Promise<boolean> {
     try {
       await this.http.post('/', ['PING']);
       return true;
     } catch {
-      // If REST ping fails, assume connectivity will be validated on first use
-      return true;
+      return false;
     }
   }
 

package/src/providers/vectordb/WeaviateProvider.ts

@@ -40,6 +40,7 @@ export class WeaviateProvider extends BaseVectorProvider {
   }
 
   async upsert(doc: UpsertDocument, namespace?: string): Promise<void> {
+    const primitiveMetadata = this.extractPrimitiveMetadata(doc.metadata);
     const payload = {
       class: this.indexName,
       id: doc.id,
@@ -47,6 +48,7 @@ export class WeaviateProvider extends BaseVectorProvider {
       properties: {
         content: doc.content,
         metadata: JSON.stringify(doc.metadata || {}),
+        ...primitiveMetadata,
         namespace: namespace || '',
       },
     };
@@ -62,6 +64,7 @@ export class WeaviateProvider extends BaseVectorProvider {
         properties: {
           content: doc.content,
           metadata: JSON.stringify(doc.metadata || {}),
+          ...this.extractPrimitiveMetadata(doc.metadata),
           namespace: namespace || '',
         },
       })),
@@ -70,12 +73,14 @@ export class WeaviateProvider extends BaseVectorProvider {
   }
 
   async query(vector: number[], topK: number, namespace?: string, _filter?: Record<string, unknown>): Promise<VectorMatch[]> {
+    // IMPORTANT: read queryText BEFORE calling sanitizeFilter() because sanitizeFilter strips it.
+    const queryText = _filter?.queryText as string | undefined;
     const sanitizedFilter = this.sanitizeFilter(_filter);
-    const queryText = sanitizedFilter.queryText as string;
 
-    const searchParams = queryText 
+    const searchParams = queryText
       ? `hybrid: { query: ${JSON.stringify(queryText)}, alpha: 0.5 }`
       : `nearVector: { vector: ${JSON.stringify(vector)} }`;
+    const where = this.buildWhereFilter(namespace, sanitizedFilter);
 
     const graphqlQuery = {
       query: `
@@ -84,7 +89,7 @@ export class WeaviateProvider extends BaseVectorProvider {
             ${this.indexName}(
               ${searchParams}
               limit: ${topK}
-              ${namespace ? `where: { path: ["namespace"], operator: Equal, valueString: "${namespace}" }` : ''}
+              ${where ? `where: ${where}` : ''}
             ) {
               content
               metadata
@@ -132,4 +137,37 @@ export class WeaviateProvider extends BaseVectorProvider {
   }
 
   async disconnect(): Promise<void> {}
+
+  private extractPrimitiveMetadata(metadata?: Record<string, unknown>): Record<string, string | number | boolean> {
+    const result: Record<string, string | number | boolean> = {};
+    Object.entries(metadata || {}).forEach(([key, value]) => {
+      if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
+        result[key] = value;
+      }
+    });
+    return result;
+  }
+
+  private buildWhereFilter(namespace?: string, filter?: Record<string, unknown>): string | undefined {
+    const operands: string[] = [];
+    if (namespace) operands.push(this.weaviateOperand('namespace', namespace));
+
+    Object.entries(filter || {}).forEach(([key, value]) => {
+      if (key === 'namespace') return;
+      if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') {
+        operands.push(this.weaviateOperand(key, value));
+      }
+    });
+
+    if (operands.length === 0) return undefined;
+    if (operands.length === 1) return operands[0];
+    return `{ operator: And, operands: [${operands.join(', ')}] }`;
+  }
+
+  private weaviateOperand(key: string, value: string | number | boolean): string {
+    const path = `path: [${JSON.stringify(key)}], operator: Equal`;
+    if (typeof value === 'number') return `{ ${path}, valueNumber: ${value} }`;
+    if (typeof value === 'boolean') return `{ ${path}, valueBoolean: ${value} }`;
+    return `{ ${path}, valueString: ${JSON.stringify(value)} }`;
+  }
 }

package/src/types/chat.ts

@@ -59,4 +59,6 @@ export interface UseRagChatReturn {
   retry: () => Promise<void>;
   /** Programmatically set the conversation (e.g. to restore from a DB) */
   setMessages: React.Dispatch<React.SetStateAction<RagMessage[]>>;
+  /** Abort the current active stream request */
+  stop: () => void;
 }

package/src/types/index.ts

@@ -125,7 +125,12 @@ export type VisualizationType =
   | 'pie_chart'
   | 'bar_chart'
   | 'line_chart'
+  | 'histogram'
+  | 'horizontal_bar'
+  | 'scatter_plot'
   | 'radar_chart'
+  | 'metric_card'
+  | 'geo_map'
   | 'table'
   | 'product_carousel'
   | 'carousel'
@@ -173,6 +178,27 @@ export interface LineChartDataPoint {
   [key: string]: unknown;
 }
 
+/**
+ * Scatter plot data point
+ */
+export interface ScatterPlotDataPoint {
+  x: number;
+  y: number;
+  label?: string;
+  [key: string]: unknown;
+}
+
+/**
+ * KPI / metric card representation
+ */
+export interface MetricCardData {
+  label: string;
+  value: number;
+  operation?: 'sum' | 'average' | 'count' | 'min' | 'max' | 'median';
+  unit?: string;
+  details?: Record<string, string | number | boolean>;
+}
+
 /**
  * Table representation
  */

package/src/utils/ProductExtractor.ts

@@ -199,11 +199,13 @@ export function extractProductsFromSources(sources: VectorMatch[] | undefined, i
     .filter(s => {
       const m = s.metadata ?? {};
       const keys = Object.keys(m).map(k => k.toLowerCase());
-      const hasProductKey = keys.some(k =>
-        ['price', 'image', 'img', 'thumbnail', 'images', 'brand', 'product', 'sku', 'category', 'model', 'cost'].includes(k)
+      const hasStrongProductKey = keys.some(k =>
+        ['price', 'image', 'img', 'thumbnail', 'images', 'product', 'sku', 'cost'].includes(k)
       );
+      const hasProductIdentity = keys.some(k => ['brand', 'model'].includes(k)) &&
+        keys.some(k => ['name', 'title', 'product_name', 'product'].includes(k));
       const hasPricePattern = /\$\s*\d+/.test(s.content);
-      return hasProductKey || hasPricePattern || m.type === 'product';
+      return hasStrongProductKey || hasProductIdentity || hasPricePattern || m.type === 'product';
     })
     .map(s => {
       const m = (s.metadata ?? {}) as Record<string, unknown>;

package/src/utils/SchemaMapper.ts

@@ -57,14 +57,16 @@ Given these metadata keys from a database: [${keys.join(', ')}]
 Identify which keys best correspond to these standard UI properties:
 ${propertyList}
 
-Return ONLY a JSON object where the keys are the UI properties and the values are the matching database keys. 
+Return ONLY a valid JSON object where the keys are the UI properties and the values are the EXACT matching database keys from the list above. 
 If no good match is found for a property, omit it.
 
 Example:
 {
-  "name": "Product_Title",
-  "price": "MSRP_USD",
-  "brand": "VendorName"
+  "name": "Title",
+  "price": "Variant Price",
+  "brand": "Vendor",
+  "image": "Image Src",
+  "stock": "Variant Inventory Qty"
 }
 `;