@retrivora-ai/rag-engine 1.0.5 → 1.0.7

package/dist/{MongoDBProvider-BO2Y5DRR.mjs → MongoDBProvider-YNKC7EJ6.mjs}

@@ -1,6 +1,6 @@
 import {
   MongoDBProvider
-} from "./chunk-73I6VWU3.mjs";
+} from "./chunk-5AJ4XHLW.mjs";
 import "./chunk-IMP6FUCY.mjs";
 import "./chunk-X4TOT24V.mjs";
 export {

package/dist/{RagConfig-BEmz4hVP.d.mts → RagConfig-BjC6zSTV.d.mts}

@@ -244,6 +244,10 @@ interface RAGConfig {
     useGraphRetrieval?: boolean;
     /** Whether to perform reranking on retrieved results */
     useReranking?: boolean;
+    /** List of metadata fields that are valid for filtering.
+     * Used to dynamically extract hints from natural language queries.
+     */
+    filterableFields?: string[];
 }
 interface RagConfig {
     /**

package/dist/{RagConfig-BEmz4hVP.d.ts → RagConfig-BjC6zSTV.d.ts}

@@ -244,6 +244,10 @@ interface RAGConfig {
     useGraphRetrieval?: boolean;
     /** Whether to perform reranking on retrieved results */
     useReranking?: boolean;
+    /** List of metadata fields that are valid for filtering.
+     * Used to dynamically extract hints from natural language queries.
+     */
+    filterableFields?: string[];
 }
 interface RagConfig {
     /**

package/dist/{chunk-MWL4AGQI.mjs → chunk-3GKA3PJF.mjs}

@@ -1098,7 +1098,7 @@ var ProviderRegistry = class {
         return PostgreSQLProvider;
       }
       case "mongodb": {
-        const { MongoDBProvider } = await import("./MongoDBProvider-BO2Y5DRR.mjs");
+        const { MongoDBProvider } = await import("./MongoDBProvider-YNKC7EJ6.mjs");
         return MongoDBProvider;
       }
       case "milvus": {
@@ -1904,12 +1904,14 @@ var QueryProcessor = class {
    * Checks if a string is likely a question word or common prompt phrase.
    */
   static isLikelyPromptPhrase(value) {
-    return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
+    return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
   }
   /**
    * Scans a natural language question for potential metadata hints and keywords.
+   * @param question The user's query
+   * @param validFields Optional list of known filterable fields to look for
    */
-  static extractQueryFieldHints(question) {
+  static extractQueryFieldHints(question, validFields = []) {
     var _a, _b, _c, _d;
     if (!question.trim()) return [];
     const hints = /* @__PURE__ */ new Map();
@@ -1943,11 +1945,29 @@ var QueryProcessor = class {
         if (name) addHint(name, "name");
       }
     }
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, "gi");
+        for (const match of question.matchAll(pattern)) {
+          const value = match[1];
+          if (value && !this.isLikelyPromptPhrase(value)) {
+            addHint(value, field);
+          }
+        }
+      }
+    }
     const universalPatterns = [
       { regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: "email", group: 1 },
       { regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: "phone", group: 1 },
       { regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: "date", group: 1 },
-      { regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: "amount", group: 1 },
       { regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: "id", group: 2 },
       { regex: /"([^"]{2,120})"/g, group: 1 },
       { regex: /'([^']{2,120})'/g, group: 1 }
@@ -2241,7 +2261,7 @@ var Pipeline = class {
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f;
+      var _a, _b, _c, _d, _e, _f, _g;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
@@ -2251,7 +2271,7 @@ var Pipeline = class {
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
         }
-        const hints = QueryProcessor.extractQueryFieldHints(question);
+        const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
@@ -2259,7 +2279,7 @@ var Pipeline = class {
           filter
         }));
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
-        if ((_f = this.config.rag) == null ? void 0 : _f.useReranking) {
+        if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);

package/dist/{chunk-73I6VWU3.mjs → chunk-5AJ4XHLW.mjs}

@@ -118,6 +118,18 @@ var MongoDBProvider = class extends BaseVectorProvider {
   }
   async query(vector, topK, namespace, filter) {
     const publicFilter = this.sanitizeFilter(filter);
+    const vectorSearchFilter = {};
+    const matchFilter = {};
+    if (namespace) {
+      vectorSearchFilter.namespace = namespace;
+    }
+    for (const [key, value] of Object.entries(publicFilter)) {
+      if (key === "namespace") {
+        vectorSearchFilter.namespace = value;
+      } else {
+        matchFilter[key] = value;
+      }
+    }
     const pipeline = [
       {
         $vectorSearch: __spreadValues({
@@ -125,9 +137,15 @@ var MongoDBProvider = class extends BaseVectorProvider {
           path: this.embeddingKey,
           queryVector: vector,
           numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
-          limit: topK
-        }, Object.keys(publicFilter).length > 0 || namespace ? { filter: __spreadValues(__spreadValues({}, publicFilter), namespace ? { namespace } : {}) } : {})
-      },
+          limit: topK * 2
+        }, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
+      }
+    ];
+    if (Object.keys(matchFilter).length > 0) {
+      pipeline.push({ $match: matchFilter });
+    }
+    pipeline.push(
+      { $limit: topK },
       {
         $project: {
           _id: 1,
@@ -137,7 +155,7 @@ var MongoDBProvider = class extends BaseVectorProvider {
           score: { $meta: "vectorSearchScore" }
         }
       }
-    ];
+    );
     const results = await this.collection.aggregate(pipeline).toArray();
     return results.map((res) => ({
       id: res._id,

package/dist/handlers/index.d.mts

@@ -1,3 +1,3 @@
-import '../RagConfig-BEmz4hVP.mjs';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from '../index-DFSy0CG6.mjs';
+import '../RagConfig-BjC6zSTV.mjs';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from '../index-CU_fQq__.mjs';
 import 'next/server';

package/dist/handlers/index.d.ts

@@ -1,3 +1,3 @@
-import '../RagConfig-BEmz4hVP.js';
-export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from '../index-CSfaCeux.js';
+import '../RagConfig-BjC6zSTV.js';
+export { c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from '../index-C3bLmWcR.js';
 import 'next/server';

package/dist/handlers/index.js

@@ -650,6 +650,18 @@ var init_MongoDBProvider = __esm({
       }
       async query(vector, topK, namespace, filter) {
         const publicFilter = this.sanitizeFilter(filter);
+        const vectorSearchFilter = {};
+        const matchFilter = {};
+        if (namespace) {
+          vectorSearchFilter.namespace = namespace;
+        }
+        for (const [key, value] of Object.entries(publicFilter)) {
+          if (key === "namespace") {
+            vectorSearchFilter.namespace = value;
+          } else {
+            matchFilter[key] = value;
+          }
+        }
         const pipeline = [
           {
             $vectorSearch: __spreadValues({
@@ -657,9 +669,15 @@ var init_MongoDBProvider = __esm({
               path: this.embeddingKey,
               queryVector: vector,
               numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
-              limit: topK
-            }, Object.keys(publicFilter).length > 0 || namespace ? { filter: __spreadValues(__spreadValues({}, publicFilter), namespace ? { namespace } : {}) } : {})
-          },
+              limit: topK * 2
+            }, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
+          }
+        ];
+        if (Object.keys(matchFilter).length > 0) {
+          pipeline.push({ $match: matchFilter });
+        }
+        pipeline.push(
+          { $limit: topK },
           {
             $project: {
               _id: 1,
@@ -669,7 +687,7 @@ var init_MongoDBProvider = __esm({
               score: { $meta: "vectorSearchScore" }
             }
           }
-        ];
+        );
         const results = await this.collection.aggregate(pipeline).toArray();
         return results.map((res) => ({
           id: res._id,
@@ -3362,12 +3380,14 @@ var QueryProcessor = class {
    * Checks if a string is likely a question word or common prompt phrase.
    */
   static isLikelyPromptPhrase(value) {
-    return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
+    return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
   }
   /**
    * Scans a natural language question for potential metadata hints and keywords.
+   * @param question The user's query
+   * @param validFields Optional list of known filterable fields to look for
    */
-  static extractQueryFieldHints(question) {
+  static extractQueryFieldHints(question, validFields = []) {
     var _a, _b, _c, _d;
     if (!question.trim()) return [];
     const hints = /* @__PURE__ */ new Map();
@@ -3401,11 +3421,29 @@ var QueryProcessor = class {
         if (name) addHint(name, "name");
       }
     }
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, "gi");
+        for (const match of question.matchAll(pattern)) {
+          const value = match[1];
+          if (value && !this.isLikelyPromptPhrase(value)) {
+            addHint(value, field);
+          }
+        }
+      }
+    }
     const universalPatterns = [
       { regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: "email", group: 1 },
       { regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: "phone", group: 1 },
       { regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: "date", group: 1 },
-      { regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: "amount", group: 1 },
       { regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: "id", group: 2 },
       { regex: /"([^"]{2,120})"/g, group: 1 },
       { regex: /'([^']{2,120})'/g, group: 1 }
@@ -3699,7 +3737,7 @@ var Pipeline = class {
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f;
+      var _a, _b, _c, _d, _e, _f, _g;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
@@ -3709,7 +3747,7 @@ var Pipeline = class {
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
         }
-        const hints = QueryProcessor.extractQueryFieldHints(question);
+        const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
@@ -3717,7 +3755,7 @@ var Pipeline = class {
           filter
         }));
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
-        if ((_f = this.config.rag) == null ? void 0 : _f.useReranking) {
+        if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);

package/dist/handlers/index.mjs

@@ -8,7 +8,7 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "../chunk-MWL4AGQI.mjs";
+} from "../chunk-3GKA3PJF.mjs";
 import "../chunk-YLTMFW4M.mjs";
 import "../chunk-X4TOT24V.mjs";
 export {

package/dist/{index-CSfaCeux.d.ts → index-C3bLmWcR.d.ts}

@@ -1,4 +1,4 @@
-import { g as RagConfig, C as ChatMessage, b as ChatResponse, e as IngestDocument } from './RagConfig-BEmz4hVP.js';
+import { g as RagConfig, C as ChatMessage, b as ChatResponse, e as IngestDocument } from './RagConfig-BjC6zSTV.js';
 import { NextRequest, NextResponse } from 'next/server';
 
 interface ValidationError {

package/dist/{index-DFSy0CG6.d.mts → index-CU_fQq__.d.mts}

@@ -1,4 +1,4 @@
-import { g as RagConfig, C as ChatMessage, b as ChatResponse, e as IngestDocument } from './RagConfig-BEmz4hVP.mjs';
+import { g as RagConfig, C as ChatMessage, b as ChatResponse, e as IngestDocument } from './RagConfig-BjC6zSTV.mjs';
 import { NextRequest, NextResponse } from 'next/server';
 
 interface ValidationError {

package/dist/index.d.mts

@@ -1,6 +1,6 @@
 import React$1, { ReactNode } from 'react';
-import { C as ChatMessage, V as VectorMatch, U as UIConfig } from './RagConfig-BEmz4hVP.mjs';
-export { a as ChatOptions, b as ChatResponse, E as EmbedOptions, c as EmbeddingConfig, d as EmbeddingProvider, I as ILLMProvider, e as IngestDocument, L as LLMConfig, f as LLMProvider, R as RAGConfig, g as RagConfig, h as UpsertDocument, i as VectorDBConfig, j as VectorDBProvider } from './RagConfig-BEmz4hVP.mjs';
+import { C as ChatMessage, V as VectorMatch, U as UIConfig } from './RagConfig-BjC6zSTV.mjs';
+export { a as ChatOptions, b as ChatResponse, E as EmbedOptions, c as EmbeddingConfig, d as EmbeddingProvider, I as ILLMProvider, e as IngestDocument, L as LLMConfig, f as LLMProvider, R as RAGConfig, g as RagConfig, h as UpsertDocument, i as VectorDBConfig, j as VectorDBProvider } from './RagConfig-BjC6zSTV.mjs';
 export { C as Chunk, a as ChunkOptions } from './DocumentChunker-Dh9TvmGG.mjs';
 
 interface ChatWidgetProps {

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import React$1, { ReactNode } from 'react';
-import { C as ChatMessage, V as VectorMatch, U as UIConfig } from './RagConfig-BEmz4hVP.js';
-export { a as ChatOptions, b as ChatResponse, E as EmbedOptions, c as EmbeddingConfig, d as EmbeddingProvider, I as ILLMProvider, e as IngestDocument, L as LLMConfig, f as LLMProvider, R as RAGConfig, g as RagConfig, h as UpsertDocument, i as VectorDBConfig, j as VectorDBProvider } from './RagConfig-BEmz4hVP.js';
+import { C as ChatMessage, V as VectorMatch, U as UIConfig } from './RagConfig-BjC6zSTV.js';
+export { a as ChatOptions, b as ChatResponse, E as EmbedOptions, c as EmbeddingConfig, d as EmbeddingProvider, I as ILLMProvider, e as IngestDocument, L as LLMConfig, f as LLMProvider, R as RAGConfig, g as RagConfig, h as UpsertDocument, i as VectorDBConfig, j as VectorDBProvider } from './RagConfig-BjC6zSTV.js';
 export { C as Chunk, a as ChunkOptions } from './DocumentChunker-Dh9TvmGG.js';
 
 interface ChatWidgetProps {

package/dist/server.d.mts

@@ -1,7 +1,7 @@
-import { i as VectorDBConfig, L as LLMConfig, c as EmbeddingConfig, g as RagConfig, e as IngestDocument, C as ChatMessage, b as ChatResponse, k as RetrievalResult, h as UpsertDocument, V as VectorMatch, G as GraphDBConfig, l as GraphNode, m as Edge, n as GraphSearchResult, I as ILLMProvider, j as VectorDBProvider, f as LLMProvider, d as EmbeddingProvider, R as RAGConfig, U as UIConfig, a as ChatOptions, E as EmbedOptions } from './RagConfig-BEmz4hVP.mjs';
+import { i as VectorDBConfig, L as LLMConfig, c as EmbeddingConfig, g as RagConfig, e as IngestDocument, C as ChatMessage, b as ChatResponse, k as RetrievalResult, h as UpsertDocument, V as VectorMatch, G as GraphDBConfig, l as GraphNode, m as Edge, n as GraphSearchResult, I as ILLMProvider, j as VectorDBProvider, f as LLMProvider, d as EmbeddingProvider, R as RAGConfig, U as UIConfig, a as ChatOptions, E as EmbedOptions } from './RagConfig-BjC6zSTV.mjs';
 export { C as Chunk, a as ChunkOptions, D as DocumentChunker } from './DocumentChunker-Dh9TvmGG.mjs';
-import { H as HealthCheckResult, I as IProviderValidator, a as IProviderHealthChecker } from './index-DFSy0CG6.mjs';
-export { C as ConfigValidator, V as ValidationError, b as VectorPlugin, c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from './index-DFSy0CG6.mjs';
+import { H as HealthCheckResult, I as IProviderValidator, a as IProviderHealthChecker } from './index-CU_fQq__.mjs';
+export { C as ConfigValidator, V as ValidationError, b as VectorPlugin, c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from './index-CU_fQq__.mjs';
 import 'next/server';
 
 /**

package/dist/server.d.ts

@@ -1,7 +1,7 @@
-import { i as VectorDBConfig, L as LLMConfig, c as EmbeddingConfig, g as RagConfig, e as IngestDocument, C as ChatMessage, b as ChatResponse, k as RetrievalResult, h as UpsertDocument, V as VectorMatch, G as GraphDBConfig, l as GraphNode, m as Edge, n as GraphSearchResult, I as ILLMProvider, j as VectorDBProvider, f as LLMProvider, d as EmbeddingProvider, R as RAGConfig, U as UIConfig, a as ChatOptions, E as EmbedOptions } from './RagConfig-BEmz4hVP.js';
+import { i as VectorDBConfig, L as LLMConfig, c as EmbeddingConfig, g as RagConfig, e as IngestDocument, C as ChatMessage, b as ChatResponse, k as RetrievalResult, h as UpsertDocument, V as VectorMatch, G as GraphDBConfig, l as GraphNode, m as Edge, n as GraphSearchResult, I as ILLMProvider, j as VectorDBProvider, f as LLMProvider, d as EmbeddingProvider, R as RAGConfig, U as UIConfig, a as ChatOptions, E as EmbedOptions } from './RagConfig-BjC6zSTV.js';
 export { C as Chunk, a as ChunkOptions, D as DocumentChunker } from './DocumentChunker-Dh9TvmGG.js';
-import { H as HealthCheckResult, I as IProviderValidator, a as IProviderHealthChecker } from './index-CSfaCeux.js';
-export { C as ConfigValidator, V as ValidationError, b as VectorPlugin, c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from './index-CSfaCeux.js';
+import { H as HealthCheckResult, I as IProviderValidator, a as IProviderHealthChecker } from './index-C3bLmWcR.js';
+export { C as ConfigValidator, V as ValidationError, b as VectorPlugin, c as createChatHandler, d as createHealthHandler, e as createIngestHandler, f as createStreamHandler, g as createUploadHandler, s as sseErrorFrame, h as sseFrame, i as sseMetaFrame, j as sseTextFrame } from './index-C3bLmWcR.js';
 import 'next/server';
 
 /**

package/dist/server.js

@@ -662,6 +662,18 @@ var init_MongoDBProvider = __esm({
       }
       async query(vector, topK, namespace, filter) {
         const publicFilter = this.sanitizeFilter(filter);
+        const vectorSearchFilter = {};
+        const matchFilter = {};
+        if (namespace) {
+          vectorSearchFilter.namespace = namespace;
+        }
+        for (const [key, value] of Object.entries(publicFilter)) {
+          if (key === "namespace") {
+            vectorSearchFilter.namespace = value;
+          } else {
+            matchFilter[key] = value;
+          }
+        }
         const pipeline = [
           {
             $vectorSearch: __spreadValues({
@@ -669,9 +681,15 @@ var init_MongoDBProvider = __esm({
               path: this.embeddingKey,
               queryVector: vector,
               numCandidates: this.config.options.numCandidates || Math.max(topK * 20, 200),
-              limit: topK
-            }, Object.keys(publicFilter).length > 0 || namespace ? { filter: __spreadValues(__spreadValues({}, publicFilter), namespace ? { namespace } : {}) } : {})
-          },
+              limit: topK * 2
+            }, Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {})
+          }
+        ];
+        if (Object.keys(matchFilter).length > 0) {
+          pipeline.push({ $match: matchFilter });
+        }
+        pipeline.push(
+          { $limit: topK },
           {
             $project: {
               _id: 1,
@@ -681,7 +699,7 @@ var init_MongoDBProvider = __esm({
               score: { $meta: "vectorSearchScore" }
             }
           }
-        ];
+        );
         const results = await this.collection.aggregate(pipeline).toArray();
         return results.map((res) => ({
           id: res._id,
@@ -3455,12 +3473,14 @@ var QueryProcessor = class {
    * Checks if a string is likely a question word or common prompt phrase.
    */
   static isLikelyPromptPhrase(value) {
-    return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
+    return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
   }
   /**
    * Scans a natural language question for potential metadata hints and keywords.
+   * @param question The user's query
+   * @param validFields Optional list of known filterable fields to look for
    */
-  static extractQueryFieldHints(question) {
+  static extractQueryFieldHints(question, validFields = []) {
     var _a, _b, _c, _d;
     if (!question.trim()) return [];
     const hints = /* @__PURE__ */ new Map();
@@ -3494,11 +3514,29 @@ var QueryProcessor = class {
         if (name) addHint(name, "name");
       }
     }
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, "gi");
+        for (const match of question.matchAll(pattern)) {
+          const value = match[1];
+          if (value && !this.isLikelyPromptPhrase(value)) {
+            addHint(value, field);
+          }
+        }
+      }
+    }
     const universalPatterns = [
       { regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: "email", group: 1 },
       { regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: "phone", group: 1 },
       { regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: "date", group: 1 },
-      { regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: "amount", group: 1 },
       { regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: "id", group: 2 },
       { regex: /"([^"]{2,120})"/g, group: 1 },
       { regex: /'([^']{2,120})'/g, group: 1 }
@@ -3792,7 +3830,7 @@ var Pipeline = class {
    */
   askStream(_0) {
     return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
-      var _a, _b, _c, _d, _e, _f;
+      var _a, _b, _c, _d, _e, _f, _g;
       yield new __await(this.initialize());
       const ns = namespace != null ? namespace : this.config.projectId;
       const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
@@ -3802,7 +3840,7 @@ var Pipeline = class {
         if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
           searchQuery = yield new __await(this.rewriteQuery(question, history));
         }
-        const hints = QueryProcessor.extractQueryFieldHints(question);
+        const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
         const filter = QueryProcessor.buildQueryFilter(question, hints);
         const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
           namespace: ns,
@@ -3810,7 +3848,7 @@ var Pipeline = class {
           filter
         }));
         let sources = rawSources.filter((m) => m.score >= scoreThreshold);
-        if ((_f = this.config.rag) == null ? void 0 : _f.useReranking) {
+        if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
           sources = yield new __await(this.reranker.rerank(sources, question, topK));
         } else {
           sources = sources.slice(0, topK);

package/dist/server.mjs

@@ -40,7 +40,7 @@ import {
   sseFrame,
   sseMetaFrame,
   sseTextFrame
-} from "./chunk-MWL4AGQI.mjs";
+} from "./chunk-3GKA3PJF.mjs";
 import "./chunk-YLTMFW4M.mjs";
 import {
   PineconeProvider
@@ -50,7 +50,7 @@ import {
 } from "./chunk-FLOSGE6A.mjs";
 import {
   MongoDBProvider
-} from "./chunk-73I6VWU3.mjs";
+} from "./chunk-5AJ4XHLW.mjs";
 import {
   MilvusProvider
 } from "./chunk-U55XRW3U.mjs";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@retrivora-ai/rag-engine",
-  "version": "1.0.5",
+  "version": "1.0.7",
   "description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
   "author": "Abhinav Alkuchi",
   "license": "MIT",

package/src/config/RagConfig.ts

@@ -174,6 +174,10 @@ export interface RAGConfig {
   useGraphRetrieval?: boolean;
   /** Whether to perform reranking on retrieved results */
   useReranking?: boolean;
+  /** List of metadata fields that are valid for filtering. 
+   * Used to dynamically extract hints from natural language queries. 
+   */
+  filterableFields?: string[];
 }
 
 // ---------------------------------------------------------------------------

package/src/core/Pipeline.ts

@@ -315,7 +315,7 @@ export class Pipeline {
       }
 
       // 2. Parallel Retrieval
-      const hints = QueryProcessor.extractQueryFieldHints(question);
+      const hints = QueryProcessor.extractQueryFieldHints(question, this.config.rag?.filterableFields);
       const filter = QueryProcessor.buildQueryFilter(question, hints);
 
       const { sources: rawSources, graphData } = await this.retrieve(searchQuery, { 

package/src/core/QueryProcessor.ts

@@ -31,13 +31,16 @@ export class QueryProcessor {
    * Checks if a string is likely a question word or common prompt phrase.
    */
   static isLikelyPromptPhrase(value: string): boolean {
-    return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
+    return /^(what|which|who|where|when|why|how|the|is|are|was|were|in|on|at|by|for|with|about|a|an|to|of)\b/i.test(value.trim());
   }
 
+
   /**
    * Scans a natural language question for potential metadata hints and keywords.
+   * @param question The user's query
+   * @param validFields Optional list of known filterable fields to look for
    */
-  static extractQueryFieldHints(question: string): QueryFieldHint[] {
+  static extractQueryFieldHints(question: string, validFields: string[] = []): QueryFieldHint[] {
     if (!question.trim()) return [];
 
     const hints = new Map<string, QueryFieldHint>();
@@ -88,12 +91,36 @@ export class QueryProcessor {
       }
     }
 
-    // 4. Universal patterns (email, phone, etc.)
+    // 4. Dynamic Schema-driven patterns
+    // We look for any word-like key followed by a colon or equals.
+    const genericPattern = /\b([a-z][a-z0-9_]{1,30})\s*[:=]\s*["']?([^"'\n?.!,]{1,60})["']?/gi;
+    for (const match of question.matchAll(genericPattern)) {
+      const field = match[1];
+      const value = match[2];
+      if (field && !this.isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      }
+    }
+
+    // 5. Explicitly requested fields (if any)
+    if (validFields.length > 0) {
+      for (const field of validFields) {
+        // More aggressive pattern for explicitly allowed fields: "[field] [value]"
+        const pattern = new RegExp(`\\b${field}\\s*(?:is|are|:|of)?\\s+["']?([^"\\n?.!,]{1,60})["']?(?=[?.!,]|$)`, 'gi');
+        for (const match of question.matchAll(pattern)) {
+          const value = match[1];
+          if (value && !this.isLikelyPromptPhrase(value)) {
+            addHint(value, field);
+          }
+        }
+      }
+    }
+
+    // 5. Universal patterns (email, phone, etc.)
     const universalPatterns: Array<{ regex: RegExp; field?: string; group?: number }> = [
       { regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: 'email', group: 1 },
       { regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: 'phone', group: 1 },
       { regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: 'date', group: 1 },
-      { regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: 'amount', group: 1 },
       { regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: 'id', group: 2 },
       { regex: /"([^"]{2,120})"/g, group: 1 },
       { regex: /'([^']{2,120})'/g, group: 1 },

package/src/providers/vectordb/MongoDBProvider.ts

@@ -139,6 +139,24 @@ export class MongoDBProvider extends BaseVectorProvider {
 
   async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
     const publicFilter = this.sanitizeFilter(filter);
+    
+    // Split filters: only 'namespace' is safe for the $vectorSearch 'filter' argument 
+    // unless explicitly indexed. Moving others to a subsequent $match stage.
+    const vectorSearchFilter: Record<string, unknown> = {};
+    const matchFilter: Record<string, unknown> = {};
+
+    if (namespace) {
+      vectorSearchFilter.namespace = namespace;
+    }
+
+    for (const [key, value] of Object.entries(publicFilter)) {
+      if (key === 'namespace') {
+        vectorSearchFilter.namespace = value;
+      } else {
+        matchFilter[key] = value;
+      }
+    }
+
     const pipeline: Record<string, unknown>[] = [
       {
         $vectorSearch: {
@@ -146,12 +164,20 @@ export class MongoDBProvider extends BaseVectorProvider {
           path: this.embeddingKey,
           queryVector: vector,
           numCandidates: (this.config.options.numCandidates as number) || Math.max(topK * 20, 200),
-          limit: topK,
-          ...(Object.keys(publicFilter).length > 0 || namespace
-            ? { filter: { ...publicFilter, ...(namespace ? { namespace } : {}) } }
-            : {}),
+          limit: topK * 2, // Increase limit slightly to account for post-filtering
+          ...(Object.keys(vectorSearchFilter).length > 0 ? { filter: vectorSearchFilter } : {}),
         },
       },
+    ];
+
+    // Add post-filtering stage if there are additional filters
+    if (Object.keys(matchFilter).length > 0) {
+      pipeline.push({ $match: matchFilter });
+    }
+
+    // Final limit and projection
+    pipeline.push(
+      { $limit: topK },
       {
         $project: {
           _id: 1,
@@ -160,8 +186,8 @@ export class MongoDBProvider extends BaseVectorProvider {
           namespace: 1,
           score: { $meta: 'vectorSearchScore' },
         },
-      },
-    ];
+      }
+    );
 
     const results = await this.collection!.aggregate(pipeline).toArray();