@retrivora-ai/rag-engine 1.0.6 → 1.0.7

package/dist/{MongoDBProvider-BO2Y5DRR.mjs → MongoDBProvider-YNKC7EJ6.mjs}

@@ -1,6 +1,6 @@
 import {
   MongoDBProvider
-} from "./chunk-73I6VWU3.mjs";
+} from "./chunk-5AJ4XHLW.mjs";
 import "./chunk-IMP6FUCY.mjs";
 import "./chunk-X4TOT24V.mjs";
 export {

package/dist/{chunk-ZCDJSGUW.mjs → chunk-3GKA3PJF.mjs}

@@ -1098,7 +1098,7 @@ var ProviderRegistry = class {
         return PostgreSQLProvider;
       }
       case "mongodb": {
-        const { MongoDBProvider } = await import("./MongoDBProvider-BO2Y5DRR.mjs");
+        const { MongoDBProvider } = await import("./MongoDBProvider-YNKC7EJ6.mjs");
         return MongoDBProvider;
       }
       case "milvus": {
@@ -1915,7 +1915,6 @@ var QueryProcessor = class {
     var _a, _b, _c, _d;
     if (!question.trim()) return [];
     const hints = /* @__PURE__ */ new Map();
-    const fieldsToSearch = [.../* @__PURE__ */ new Set([...this.COMMON_METADATA_FIELDS, ...validFields])];
     const addHint = (value, field) => {
       const normalizedValue = this.normalizeHintValue(value);
       if (!normalizedValue) return;
@@ -1946,17 +1945,18 @@ var QueryProcessor = class {
         if (name) addHint(name, "name");
       }
     }
-    if (fieldsToSearch.length > 0) {
-      for (const field of fieldsToSearch) {
-        const forwardPattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{2,60})["']?(?=[?.!,]|$)`, "gi");
-        for (const match of question.matchAll(forwardPattern)) {
-          const value = match[1];
-          if (value && !this.isLikelyPromptPhrase(value)) {
-            addHint(value, field);
-          }
-        }
-        const reversePattern = new RegExp(`\\b([^"\\n?.!,\\s]{2,60})\\s+${field}\\b`, "gi");
-        for (const match of question.matchAll(reversePattern)) {
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, "gi");
+        for (const match of question.matchAll(pattern)) {
           const value = match[1];
           if (value && !this.isLikelyPromptPhrase(value)) {
             addHint(value, field);
@@ -2032,28 +2032,6 @@ var QueryProcessor = class {
     return filter;
   }
 };
-QueryProcessor.COMMON_METADATA_FIELDS = [
-  "category",
-  "type",
-  "brand",
-  "status",
-  "priority",
-  "id",
-  "name",
-  "email",
-  "phone",
-  "price",
-  "rating",
-  "color",
-  "size",
-  "material",
-  "sku",
-  "role",
-  "department",
-  "location",
-  "tag",
-  "label"
-];
 
 // src/core/Pipeline.ts
 var LRUEmbeddingCache = class {

package/dist/{chunk-73I6VWU3.mjs → chunk-5AJ4XHLW.mjs}

@@ -118,6 +118,18 @@ var MongoDBProvider = class extends BaseVectorProvider {
   }
   async query(vector, topK, namespace, filter) {
     const publicFilter = this.sanitizeFilter(filter);
+    const vectorSearchFilter = {};
+    const matchFilter = {};
+    if (namespace) {
+      vectorSearchFilter.namespace = namespace;
+    }
+    for (const [key, value] of Object.entries(publicFilter)) {
+      if (key === "namespace") {
+        vectorSearchFilter.namespace = value;
+      } else {
+        matchFilter[key] = value;
+      }
+    }
     const pipeline = [
       {
         $vectorSearch: __spreadValues({
@@ -125,9 +137,15 @@ var MongoDBProvider = class extends BaseVectorProvider {
           path: this.embeddingKey,
           queryVector: vector,
           numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
-          limit: topK
-        }, Object.keys(publicFilter).length > 0 || namespace ? { filter: __spreadValues(__spreadValues({}, publicFilter), namespace ? { namespace } : {}) } : {})
-      },
+          limit: topK * 2
+        }, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
+      }
+    ];
+    if (Object.keys(matchFilter).length > 0) {
+      pipeline.push({ $match: matchFilter });
+    }
+    pipeline.push(
+      { $limit: topK },
       {
         $project: {
           _id: 1,
@@ -137,7 +155,7 @@ var MongoDBProvider = class extends BaseVectorProvider {
           score: { $meta: "vectorSearchScore" }
         }
       }
-    ];
+    );
     const results = await this.collection.aggregate(pipeline).toArray();
     return results.map((res) => ({
       id: res._id,

package/dist/handlers/index.js

@@ -650,6 +650,18 @@ var init_MongoDBProvider = __esm({
       }
       async query(vector, topK, namespace, filter) {
         const publicFilter = this.sanitizeFilter(filter);
+        const vectorSearchFilter = {};
+        const matchFilter = {};
+        if (namespace) {
+          vectorSearchFilter.namespace = namespace;
+        }
+        for (const [key, value] of Object.entries(publicFilter)) {
+          if (key === "namespace") {
+            vectorSearchFilter.namespace = value;
+          } else {
+            matchFilter[key] = value;
+          }
+        }
         const pipeline = [
           {
             $vectorSearch: __spreadValues({
@@ -657,9 +669,15 @@ var init_MongoDBProvider = __esm({
               path: this.embeddingKey,
               queryVector: vector,
               numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
-              limit: topK
-            }, Object.keys(publicFilter).length > 0 || namespace ? { filter: __spreadValues(__spreadValues({}, publicFilter), namespace ? { namespace } : {}) } : {})
-          },
+              limit: topK * 2
+            }, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
+          }
+        ];
+        if (Object.keys(matchFilter).length > 0) {
+          pipeline.push({ $match: matchFilter });
+        }
+        pipeline.push(
+          { $limit: topK },
           {
             $project: {
               _id: 1,
@@ -669,7 +687,7 @@ var init_MongoDBProvider = __esm({
               score: { $meta: "vectorSearchScore" }
             }
           }
-        ];
+        );
         const results = await this.collection.aggregate(pipeline).toArray();
         return results.map((res) => ({
           id: res._id,
@@ -3373,7 +3391,6 @@ var QueryProcessor = class {
     var _a, _b, _c, _d;
     if (!question.trim()) return [];
     const hints = /* @__PURE__ */ new Map();
-    const fieldsToSearch = [.../* @__PURE__ */ new Set([...this.COMMON_METADATA_FIELDS, ...validFields])];
     const addHint = (value, field) => {
       const normalizedValue = this.normalizeHintValue(value);
       if (!normalizedValue) return;
@@ -3404,17 +3421,18 @@ var QueryProcessor = class {
         if (name) addHint(name, "name");
       }
     }
-    if (fieldsToSearch.length > 0) {
-      for (const field of fieldsToSearch) {
-        const forwardPattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{2,60})["']?(?=[?.!,]|$)`, "gi");
-        for (const match of question.matchAll(forwardPattern)) {
-          const value = match[1];
-          if (value && !this.isLikelyPromptPhrase(value)) {
-            addHint(value, field);
-          }
-        }
-        const reversePattern = new RegExp(`\\b([^"\\n?.!,\\s]{2,60})\\s+${field}\\b`, "gi");
-        for (const match of question.matchAll(reversePattern)) {
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, "gi");
+        for (const match of question.matchAll(pattern)) {
           const value = match[1];
           if (value && !this.isLikelyPromptPhrase(value)) {
             addHint(value, field);
@@ -3490,28 +3508,6 @@ var QueryProcessor = class {
     return filter;
   }
 };
-QueryProcessor.COMMON_METADATA_FIELDS = [
-  "category",
-  "type",
-  "brand",
-  "status",
-  "priority",
-  "id",
-  "name",
-  "email",
-  "phone",
-  "price",
-  "rating",
-  "color",
-  "size",
-  "material",
-  "sku",
-  "role",
-  "department",
-  "location",
-  "tag",
-  "label"
-];
 
 // src/core/Pipeline.ts
 var LRUEmbeddingCache = class {

package/dist/handlers/index.mjs

@@ -8,7 +8,7 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "../chunk-ZCDJSGUW.mjs";
+} from "../chunk-3GKA3PJF.mjs";
 import "../chunk-YLTMFW4M.mjs";
 import "../chunk-X4TOT24V.mjs";
 export {

package/dist/server.js

@@ -662,6 +662,18 @@ var init_MongoDBProvider = __esm({
       }
       async query(vector, topK, namespace, filter) {
         const publicFilter = this.sanitizeFilter(filter);
+        const vectorSearchFilter = {};
+        const matchFilter = {};
+        if (namespace) {
+          vectorSearchFilter.namespace = namespace;
+        }
+        for (const [key, value] of Object.entries(publicFilter)) {
+          if (key === "namespace") {
+            vectorSearchFilter.namespace = value;
+          } else {
+            matchFilter[key] = value;
+          }
+        }
         const pipeline = [
           {
             $vectorSearch: __spreadValues({
@@ -669,9 +681,15 @@ var init_MongoDBProvider = __esm({
               path: this.embeddingKey,
               queryVector: vector,
               numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
-              limit: topK
-            }, Object.keys(publicFilter).length > 0 || namespace ? { filter: __spreadValues(__spreadValues({}, publicFilter), namespace ? { namespace } : {}) } : {})
-          },
+              limit: topK * 2
+            }, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
+          }
+        ];
+        if (Object.keys(matchFilter).length > 0) {
+          pipeline.push({ $match: matchFilter });
+        }
+        pipeline.push(
+          { $limit: topK },
           {
             $project: {
               _id: 1,
@@ -681,7 +699,7 @@ var init_MongoDBProvider = __esm({
               score: { $meta: "vectorSearchScore" }
             }
           }
-        ];
+        );
         const results = await this.collection.aggregate(pipeline).toArray();
         return results.map((res) => ({
           id: res._id,
@@ -3466,7 +3484,6 @@ var QueryProcessor = class {
     var _a, _b, _c, _d;
     if (!question.trim()) return [];
     const hints = /* @__PURE__ */ new Map();
-    const fieldsToSearch = [.../* @__PURE__ */ new Set([...this.COMMON_METADATA_FIELDS, ...validFields])];
     const addHint = (value, field) => {
       const normalizedValue = this.normalizeHintValue(value);
       if (!normalizedValue) return;
@@ -3497,17 +3514,18 @@ var QueryProcessor = class {
         if (name) addHint(name, "name");
       }
     }
-    if (fieldsToSearch.length > 0) {
-      for (const field of fieldsToSearch) {
-        const forwardPattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{2,60})["']?(?=[?.!,]|$)`, "gi");
-        for (const match of question.matchAll(forwardPattern)) {
-          const value = match[1];
-          if (value && !this.isLikelyPromptPhrase(value)) {
-            addHint(value, field);
-          }
-        }
-        const reversePattern = new RegExp(`\\b([^"\\n?.!,\\s]{2,60})\\s+${field}\\b`, "gi");
-        for (const match of question.matchAll(reversePattern)) {
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, "gi");
+        for (const match of question.matchAll(pattern)) {
           const value = match[1];
           if (value && !this.isLikelyPromptPhrase(value)) {
             addHint(value, field);
@@ -3583,28 +3601,6 @@ var QueryProcessor = class {
     return filter;
   }
 };
-QueryProcessor.COMMON_METADATA_FIELDS = [
-  "category",
-  "type",
-  "brand",
-  "status",
-  "priority",
-  "id",
-  "name",
-  "email",
-  "phone",
-  "price",
-  "rating",
-  "color",
-  "size",
-  "material",
-  "sku",
-  "role",
-  "department",
-  "location",
-  "tag",
-  "label"
-];
 
 // src/core/Pipeline.ts
 var LRUEmbeddingCache = class {

package/dist/server.mjs

@@ -40,7 +40,7 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "./chunk-ZCDJSGUW.mjs";
+} from "./chunk-3GKA3PJF.mjs";
 import "./chunk-YLTMFW4M.mjs";
 import {
   PineconeProvider
@@ -50,7 +50,7 @@ import {
 } from "./chunk-FLOSGE6A.mjs";
 import {
   MongoDBProvider
-} from "./chunk-73I6VWU3.mjs";
+} from "./chunk-5AJ4XHLW.mjs";
 import {
   MilvusProvider
 } from "./chunk-U55XRW3U.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "1.0.6",
+  "version": "1.0.7",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",

package/src/core/QueryProcessor.ts

@@ -34,11 +34,6 @@ export class QueryProcessor {
     return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
   }
 
-  private static readonly COMMON_METADATA_FIELDS = [
-    'category', 'type', 'brand', 'status', 'priority', 'id', 'name', 
-    'email', 'phone', 'price', 'rating', 'color', 'size', 'material',
-    'sku', 'role', 'department', 'location', 'tag', 'label'
-  ];
 
   /**
    * Scans a natural language question for potential metadata hints and keywords.
@@ -49,7 +44,6 @@ export class QueryProcessor {
     if (!question.trim()) return [];
 
     const hints = new Map<string, QueryFieldHint>();
-    const fieldsToSearch = [...new Set([...this.COMMON_METADATA_FIELDS, ...validFields])];
 
     const addHint = (value: string, field?: string) => {
       const normalizedValue = this.normalizeHintValue(value);
@@ -98,21 +92,22 @@ export class QueryProcessor {
     }
 
     // 4. Dynamic Schema-driven patterns
-    // We look for common metadata fields (and any user-defined ones) in natural language.
-    if (fieldsToSearch.length > 0) {
-      for (const field of fieldsToSearch) {
-        // Pattern 1: "[field] is [value]", "[field]: [value]", "[field] of [value]"
-        const forwardPattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{2,60})["']?(?=[?.!,]|$)`, 'gi');
-        for (const match of question.matchAll(forwardPattern)) {
-          const value = match[1];
-          if (value && !this.isLikelyPromptPhrase(value)) {
-            addHint(value, field);
-          }
-        }
+    // We look for any word-like key followed by a colon or equals.
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
 
-        // Pattern 2: "[value] [field]" (e.g. "blue color", "electronics category")
-        const reversePattern = new RegExp(`\\b([^"\\n?.!,\\s]{2,60})\\s+${field}\\b`, 'gi');
-        for (const match of question.matchAll(reversePattern)) {
+    // 5. Explicitly requested fields (if any)
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        // More aggressive pattern for explicitly allowed fields: "[field] [value]"
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, 'gi');
+        for (const match of question.matchAll(pattern)) {
           const value = match[1];
           if (value && !this.isLikelyPromptPhrase(value)) {
             addHint(value, field);

package/src/providers/vectordb/MongoDBProvider.ts

@@ -139,6 +139,24 @@ export class MongoDBProvider extends BaseVectorProvider {
 
   async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
     const publicFilter = this.sanitizeFilter(filter);
+    
+    // Split filters: only 'namespace' is safe for the $vectorSearch 'filter' argument 
+    // unless explicitly indexed. Moving others to a subsequent $match stage.
+    const vectorSearchFilter: Record<string, unknown> = {};
+    const matchFilter: Record<string, unknown> = {};
+
+    if (namespace) {
+      vectorSearchFilter.namespace = namespace;
+    }
+
+    for (const [key, value] of Object.entries(publicFilter)) {
+      if (key === 'namespace') {
+        vectorSearchFilter.namespace = value;
+      } else {
+        matchFilter[key] = value;
+      }
+    }
+
     const pipeline: Record<string, unknown>[] = [
       {
         $vectorSearch: {
@@ -146,12 +164,20 @@ export class MongoDBProvider extends BaseVectorProvider {
           path: this.embeddingKey,
           queryVector: vector,
           numCandidates: (this.config.options.numCandidates as number) || Math.max(topK * 20, 200),
-          limit: topK,
-          ...(Object.keys(publicFilter).length > 0 || namespace
-            ? { filter: { ...publicFilter, ...(namespace ? { namespace } : {}) } }
-            : {}),
+          limit: topK * 2, // Increase limit slightly to account for post-filtering
+          ...(Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {}),
         },
       },
+    ];
+
+    // Add post-filtering stage if there are additional filters
+    if (Object.keys(matchFilter).length > 0) {
+      pipeline.push({ $match: matchFilter });
+    }
+
+    // Final limit and projection
+    pipeline.push(
+      { $limit: topK },
       {
         $project: {
           _id: 1,
@@ -160,8 +186,8 @@ export class MongoDBProvider extends BaseVectorProvider {
           namespace: 1,
           score: { $meta: 'vectorSearchScore' },
         },
-      },
-    ];
+      }
+    );
 
     const results = await this.collection!.aggregate(pipeline).toArray();