@retrivora-ai/rag-engine 1.0.6 → 1.0.8

package/src/core/QueryProcessor.ts

@@ -34,11 +34,6 @@ export class QueryProcessor {
     return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
   }
 
-  private static readonly COMMON_METADATA_FIELDS = [
-    'category', 'type', 'brand', 'status', 'priority', 'id', 'name', 
-    'email', 'phone', 'price', 'rating', 'color', 'size', 'material',
-    'sku', 'role', 'department', 'location', 'tag', 'label'
-  ];
 
   /**
    * Scans a natural language question for potential metadata hints and keywords.
@@ -49,7 +44,6 @@ export class QueryProcessor {
     if (!question.trim()) return [];
 
     const hints = new Map<string, QueryFieldHint>();
-    const fieldsToSearch = [...new Set([...this.COMMON_METADATA_FIELDS, ...validFields])];
 
     const addHint = (value: string, field?: string) => {
       const normalizedValue = this.normalizeHintValue(value);
@@ -98,21 +92,22 @@ export class QueryProcessor {
     }
 
     // 4. Dynamic Schema-driven patterns
-    // We look for common metadata fields (and any user-defined ones) in natural language.
-    if (fieldsToSearch.length > 0) {
-      for (const field of fieldsToSearch) {
-        // Pattern 1: "[field] is [value]", "[field]: [value]", "[field] of [value]"
-        const forwardPattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{2,60})["']?(?=[?.!,]|$)`, 'gi');
-        for (const match of question.matchAll(forwardPattern)) {
-          const value = match[1];
-          if (value && !this.isLikelyPromptPhrase(value)) {
-            addHint(value, field);
-          }
-        }
+    // We look for any word-like key followed by a colon or equals.
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
 
-        // Pattern 2: "[value] [field]" (e.g. "blue color", "electronics category")
-        const reversePattern = new RegExp(`\\b([^"\\n?.!,\\s]{2,60})\\s+${field}\\b`, 'gi');
-        for (const match of question.matchAll(reversePattern)) {
+    // 5. Explicitly requested fields (if any)
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        // More aggressive pattern for explicitly allowed fields: "[field] [value]"
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, 'gi');
+        for (const match of question.matchAll(pattern)) {
           const value = match[1];
           if (value && !this.isLikelyPromptPhrase(value)) {
             addHint(value, field);

package/src/providers/vectordb/MongoDBProvider.ts

@@ -139,6 +139,24 @@ export class MongoDBProvider extends BaseVectorProvider {
 
   async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
     const publicFilter = this.sanitizeFilter(filter);
+    
+    // Split filters: only 'namespace' is safe for the $vectorSearch 'filter' argument 
+    // unless explicitly indexed. Moving others to a subsequent $match stage.
+    const vectorSearchFilter: Record<string, unknown> = {};
+    const matchFilter: Record<string, unknown> = {};
+
+    if (namespace) {
+      vectorSearchFilter.namespace = namespace;
+    }
+
+    for (const [key, value] of Object.entries(publicFilter)) {
+      if (key === 'namespace') {
+        vectorSearchFilter.namespace = value;
+      } else {
+        matchFilter[key] = value;
+      }
+    }
+
     const pipeline: Record<string, unknown>[] = [
       {
         $vectorSearch: {
@@ -146,12 +164,20 @@ export class MongoDBProvider extends BaseVectorProvider {
           path: this.embeddingKey,
           queryVector: vector,
           numCandidates: (this.config.options.numCandidates as number) || Math.max(topK * 20, 200),
-          limit: topK,
-          ...(Object.keys(publicFilter).length > 0 || namespace
-            ? { filter: { ...publicFilter, ...(namespace ? { namespace } : {}) } }
-            : {}),
+          limit: topK * 2, // Increase limit slightly to account for post-filtering
+          ...(Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {}),
         },
       },
+    ];
+
+    // Add post-filtering stage if there are additional filters
+    if (Object.keys(matchFilter).length > 0) {
+      pipeline.push({ $match: matchFilter });
+    }
+
+    // Final limit and projection
+    pipeline.push(
+      { $limit: topK },
       {
         $project: {
           _id: 1,
@@ -160,8 +186,8 @@ export class MongoDBProvider extends BaseVectorProvider {
           namespace: 1,
           score: { $meta: 'vectorSearchScore' },
         },
-      },
-    ];
+      }
+    );
 
     const results = await this.collection!.aggregate(pipeline).toArray();