@retrivora-ai/rag-engine 0.2.7 → 0.2.9

package/src/core/Pipeline.ts

@@ -7,30 +7,119 @@ import { BatchProcessor, BatchOptions } from './BatchProcessor';
 import { EmbeddingStrategyResolver } from '../config/EmbeddingStrategy';
 import { IngestDocument, ChatResponse } from '../types';
 
-function extractEntityHints(question: string): string[] {
-  const hints = new Set<string>();
+interface QueryFieldHint {
+  field?: string;
+  value: string;
+}
+
+interface QueryFilter {
+  metadata?: Record<string, string>;
+  keywords?: string[];
+  queryText?: string;
+}
+
+function normalizeHintValue(value: string): string {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function isLikelyPromptPhrase(value: string): boolean {
+  return /^(what|which|who|where|when|why|how)\b/i.test(value.trim());
+}
+
+function extractQueryFieldHints(question: string): QueryFieldHint[] {
+  if (!question.trim()) return [];
+
+  const hints = new Map<string, QueryFieldHint>();
+
+  const addHint = (value: string, field?: string) => {
+    const normalizedValue = normalizeHintValue(value);
+    if (!normalizedValue) return;
+
+    const normalizedField = field
+      ? field
+          .toLowerCase()
+          .replace(/[^a-z0-9]+/g, ' ')
+          .trim()
+      : undefined;
+
+    const key = `${normalizedField ?? '*'}::${normalizedValue.toLowerCase()}`;
+    if (!hints.has(key)) {
+      hints.set(key, {
+        value: normalizedValue,
+        ...(normalizedField ? { field: normalizedField } : {}),
+      });
+    }
+  };
 
   for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
-    const value = match[1].trim();
-    if (value) hints.add(value);
+    addHint(match[1]);
   }
 
-  const namedEntityPatterns = [
-    /\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})$/i,
-    /\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})(?=[?.!,]|$)/i,
+  const naturalQuestionPatterns = [
+    /\b(?:what|which)\s+(?:is|are|was|were)\s+(?:the\s+)?([^?.!,]{1,60}?)\s+of\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
+    /\b(?:who|what)\s+(?:is|are|was|were)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
+    /\b(?:about|for|regarding)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
   ];
 
-  for (const pattern of namedEntityPatterns) {
-    const match = question.match(pattern);
-    const value = match?.[1]?.trim();
-    if (value) {
-      hints.add(value);
+  for (const pattern of naturalQuestionPatterns) {
+    for (const match of question.matchAll(pattern)) {
+      const value = match[2] ?? match[1];
+      if (value) addHint(value);
     }
   }
 
-  return [...hints]
-    .map((hint) => hint.replace(/\s+/g, ' ').trim())
-    .filter(Boolean);
+  const fieldPattern = `([^\\n:=?.!,]{1,60}?)`;
+  const valuePattern = `([^\\n?.!,]{1,120}?)`;
+  const fieldValuePatterns = [
+    new RegExp(`\\b${fieldPattern}\\s*(?:=|:)\\s*["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
+    new RegExp(`\\b${fieldPattern}\\s+(?:is|are|was|were|equals?|equal to|named|called)\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
+    new RegExp(`\\bwith\\s+${fieldPattern}\\s+["']?${valuePattern}["']?(?=[?.!,]|$)`, 'gi'),
+  ];
+
+  for (const pattern of fieldValuePatterns) {
+    for (const match of question.matchAll(pattern)) {
+      const field = normalizeHintValue(match[1] ?? '');
+      const value = match[2] ?? '';
+
+      if (field && !isLikelyPromptPhrase(field)) {
+        addHint(value, field);
+      } else {
+        addHint(value);
+      }
+    }
+  }
+
+  for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3}\b/g)) {
+    addHint(match[0]);
+  }
+
+  return [...hints.values()];
+}
+
+function buildQueryFilter(question: string, hints: QueryFieldHint[]): QueryFilter {
+  const filter: QueryFilter = { metadata: {}, keywords: [], queryText: question };
+
+  for (const hint of hints) {
+    if (hint.field) {
+      // prefer last-seen value for a field; providers may interpret filters differently
+      filter.metadata![hint.field] = hint.value;
+    } else {
+      // treat as keyword / named entity
+      filter.keywords!.push(hint.value);
+    }
+  }
+
+  // Also extract quoted phrases (if not already captured) as keywords
+  for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
+    const term = normalizeHintValue(match[0]);
+    if (term && !filter.keywords!.includes(term)) filter.keywords!.push(term);
+  }
+
+  // Remove empty metadata object when no fields present to keep filter minimal
+  if (Object.keys(filter.metadata || {}).length === 0) delete filter.metadata;
+  if (filter.keywords && filter.keywords.length === 0) delete filter.keywords;
+
+  return filter;
 }
 
 /**
@@ -161,12 +250,16 @@ export class Pipeline {
 
     try {
       const queryVector = await this.embeddingProvider.embed(question, { taskType: 'query' });
-      const entityHints = extractEntityHints(question);
-      // Pass the original text to the vector DB so it can perform hybrid (keyword) search if supported
-      const rawMatches = await this.vectorDB.query(queryVector, topK, ns, {
-        __queryText: question,
-        __entityHints: entityHints,
-      });
+      const fieldHints = extractQueryFieldHints(question);
+      // Build a provider-agnostic filter from hints + original query text. Providers
+      // may choose to use `queryText` for hybrid keyword matching and `metadata`
+      // for exact/filtered metadata queries.
+      const filter = buildQueryFilter(question, fieldHints) as Record<string, unknown>;
+
+      // Keep the raw hints available as well for providers that want richer parsing
+      filter.__fieldHints = fieldHints;
+
+      const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
 
       const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
       const context = sources.length

package/src/providers/vectordb/MongoDBProvider.ts

@@ -3,6 +3,14 @@ import { VectorDBConfig } from '../../config/RagConfig';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
 
+function stripInternalFilterKeys(filter?: Record<string, unknown>): Record<string, unknown> {
+  if (!filter) return {};
+
+  return Object.fromEntries(
+    Object.entries(filter).filter(([key]) => !key.startsWith('__'))
+  );
+}
+
 /**
  * MongoDBProvider — MongoDB Atlas Vector Search implementation.
  */
@@ -67,6 +75,7 @@ export class MongoDBProvider extends BaseVectorProvider {
   }
 
   async query(vector: number[], topK: number, namespace?: string, filter?: Record<string, unknown>): Promise<VectorMatch[]> {
+    const publicFilter = stripInternalFilterKeys(filter);
     const pipeline: Record<string, unknown>[] = [
       {
         $vectorSearch: {
@@ -75,7 +84,9 @@ export class MongoDBProvider extends BaseVectorProvider {
           queryVector: vector,
           numCandidates: Math.max(topK * 10, 100),
           limit: topK,
-          ...(filter || namespace ? { filter: { ...(filter || {}), ...(namespace ? { namespace } : {}) } } : {}),
+          ...(Object.keys(publicFilter).length > 0 || namespace
+            ? { filter: { ...publicFilter, ...(namespace ? { namespace } : {}) } }
+            : {}),
         },
       },
       {

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -3,7 +3,42 @@ import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
 import { VectorDBConfig } from '../../config/RagConfig';
 
-const EXACT_MATCH_FIELDS = ['name', 'product_name', 'productname', 'title'] as const;
+interface QueryFieldHint {
+  field?: string;
+  value: string;
+}
+
+interface TableSearchConfig {
+  availableFields: string[];
+  searchableFields: string[];
+  normalizedFieldMap: Map<string, string>;
+}
+
+function normalizeFieldName(value: string): string {
+  return value.toLowerCase().replace(/[^a-z0-9]+/g, '');
+}
+
+function sanitizeFilterHints(rawFilter?: Record<string, unknown>): QueryFieldHint[] {
+  if (!Array.isArray(rawFilter?.__fieldHints)) return [];
+
+  return (rawFilter.__fieldHints as unknown[])
+    .filter((hint): hint is QueryFieldHint => typeof hint === 'object' && hint !== null)
+    .map((hint) => ({
+      value: typeof hint.value === 'string' ? hint.value.trim().toLowerCase() : '',
+      ...(typeof hint.field === 'string' && hint.field.trim()
+        ? { field: normalizeFieldName(hint.field) }
+        : {}),
+    }))
+    .filter((hint) => Boolean(hint.value));
+}
+
+function stripInternalFilterKeys(filter?: Record<string, unknown>): Record<string, unknown> {
+  if (!filter) return {};
+
+  return Object.fromEntries(
+    Object.entries(filter).filter(([key]) => !key.startsWith('__'))
+  );
+}
 
 /**
  * MultiTablePostgresProvider — PostgreSQL implementation that searches across
@@ -18,6 +53,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
   private readonly dimensions: number;
   private readonly connectionString: string;
   private tables: string[];
+  private tableSearchConfig = new Map<string, TableSearchConfig>();
 
   constructor(config: VectorDBConfig) {
     super(config);
@@ -78,6 +114,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       if (this.tables.length === 0) {
         console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
       } else {
+        this.tableSearchConfig = await this.loadTableSearchConfig(client, this.tables);
         console.log(
           `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(', ')}`
         );
@@ -126,44 +163,73 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
     console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
 
     const queryText = _filter?.__queryText as string | undefined;
-    const entityHints = Array.isArray(_filter?.__entityHints)
-      ? (_filter.__entityHints as unknown[])
-        .filter((hint): hint is string => typeof hint === 'string')
-        .map((hint) => hint.trim().toLowerCase())
-        .filter(Boolean)
-      : [];
+    const fieldHints = sanitizeFilterHints(_filter);
+    const explicitFilter = stripInternalFilterKeys(_filter);
 
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = '';
         let params: unknown[] = [];
+        const tableConfig = this.tableSearchConfig.get(table);
+        const availableFields = tableConfig?.availableFields ?? [];
+        const searchableFields = tableConfig?.searchableFields ?? [];
+        const normalizedFieldMap = tableConfig?.normalizedFieldMap ?? new Map<string, string>();
+        const whereConditions: string[] = [];
 
         if (queryText) {
-          const hasEntityHints = entityHints.length > 0;
-          const exactNameScoreExpr = hasEntityHints
-            ? `GREATEST(${EXACT_MATCH_FIELDS.map((field) =>
-              `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${field}', '')) = ANY($3::text[]) THEN 1 ELSE 0 END`
+          const genericValueHints = fieldHints
+            .filter((hint) => !hint.field)
+            .map((hint) => hint.value);
+
+          params = genericValueHints.length > 0
+            ? [vectorLiteral, queryText, genericValueHints]
+            : [vectorLiteral, queryText];
+
+          const genericValueParamIndex = genericValueHints.length > 0 ? 3 : -1;
+
+          const exactValueScoreExpr = genericValueHints.length > 0 && searchableFields.length > 0
+            ? `GREATEST(${searchableFields.map((field) =>
+              `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${field}', '')) = ANY($${genericValueParamIndex}::text[]) THEN 1 ELSE 0 END`
             ).join(', ')})`
             : '0';
 
+          const fieldSpecificChecks = fieldHints
+            .filter((hint) => hint.field)
+            .map((hint) => {
+              const column = hint.field ? normalizedFieldMap.get(hint.field) : undefined;
+              if (!column) return null;
+              params.push(hint.value);
+              const paramIndex = params.length;
+              return `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${column}', '')) = $${paramIndex}::text THEN 1 ELSE 0 END`;
+            })
+            .filter((expr): expr is string => Boolean(expr));
+
+          const exactFieldScoreExpr = fieldSpecificChecks.length > 0
+            ? `GREATEST(${fieldSpecificChecks.join(', ')})`
+            : '0';
+
           // Actual Hybrid Search: Semantic Vector Score + Full-Text Keyword Rank
           // We convert plainto_tsquery's AND (&) operator to OR (|) so that natural language queries
-          // (which contain words like "price", "product" not found in the row text) still match entities like "Router".
+          // (which contain words that are not present verbatim in the row text) still match the intended records.
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS vector_score,
              COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
-             ${exactNameScoreExpr} AS exact_name_score,
-             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) + (${exactNameScoreExpr} * 5.0)) AS hybrid_score
+             ${exactValueScoreExpr} AS exact_value_score,
+             ${exactFieldScoreExpr} AS exact_field_score,
+             (
+               (1 - (embedding <=> $1::vector)) +
+               (COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) +
+               (${exactValueScoreExpr} * 3.0) +
+               (${exactFieldScoreExpr} * 5.0)
+             ) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
-          params = hasEntityHints
-            ? [vectorLiteral, queryText, entityHints]
-            : [vectorLiteral, queryText];
         } else {
           // Fallback to Vector-only Search
+          params = [vectorLiteral];
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS hybrid_score
@@ -171,7 +237,19 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
-          params = [vectorLiteral];
+        }
+
+        for (const [key, value] of Object.entries(explicitFilter)) {
+          const column = availableFields.includes(key)
+            ? key
+            : normalizedFieldMap.get(normalizeFieldName(key));
+          if (!column) continue;
+          params.push(String(value));
+          whereConditions.push(`COALESCE(to_jsonb(t)->>'${column}', '') = $${params.length}::text`);
+        }
+
+        if (whereConditions.length > 0) {
+          sqlQuery = sqlQuery.replace(`FROM "${table}" t`, `FROM "${table}" t\n            WHERE ${whereConditions.join(' AND ')}`);
         }
 
         const result = await this.pool.query(sqlQuery, params);
@@ -188,7 +266,8 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           delete rest.embedding;
           delete rest.vector_score;
           delete rest.keyword_score;
-          delete rest.exact_name_score;
+          delete rest.exact_value_score;
+          delete rest.exact_field_score;
 
           const content = `[TYPE: ${table.replace(/s$/, '').toUpperCase()}]\n` +
             Object.entries(rest)
@@ -250,4 +329,67 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       await this.pool.end();
     }
   }
+
+  private async loadTableSearchConfig(
+    client: PoolClient,
+    tables: string[]
+  ): Promise<Map<string, TableSearchConfig>> {
+    if (tables.length === 0) return new Map();
+
+    const configuredSearchFields = this.parseConfiguredSearchFields();
+    const result = await client.query(`
+      SELECT table_name, column_name, data_type
+      FROM information_schema.columns
+      WHERE table_schema = 'public'
+        AND table_name = ANY($1::text[])
+      ORDER BY ordinal_position
+    `, [tables]);
+
+    const rowsByTable = new Map<string, Array<{ column_name: string; data_type: string }>>();
+    for (const row of result.rows as Array<{ table_name: string; column_name: string; data_type: string }>) {
+      const rows = rowsByTable.get(row.table_name) ?? [];
+      rows.push({ column_name: row.column_name, data_type: row.data_type });
+      rowsByTable.set(row.table_name, rows);
+    }
+
+    const configByTable = new Map<string, TableSearchConfig>();
+    for (const table of tables) {
+      const columns = rowsByTable.get(table) ?? [];
+      const availableFields = columns
+        .filter(({ column_name }) => column_name !== 'embedding')
+        .map(({ column_name }) => column_name);
+
+      const inferredFields = columns
+        .filter(({ column_name, data_type }) =>
+          column_name !== 'embedding' &&
+          !['ARRAY', 'json', 'jsonb', 'bytea', 'tsvector', 'USER-DEFINED'].includes(data_type)
+        )
+        .map(({ column_name }) => column_name);
+
+      const searchableFields = configuredSearchFields.length > 0
+        ? configuredSearchFields.filter((field) => columns.some((column) => column.column_name === field))
+        : inferredFields;
+
+      const normalizedFieldMap = new Map<string, string>();
+      for (const field of availableFields) {
+        normalizedFieldMap.set(normalizeFieldName(field), field);
+      }
+
+      configByTable.set(table, {
+        availableFields,
+        searchableFields,
+        normalizedFieldMap,
+      });
+    }
+
+    return configByTable;
+  }
+
+  private parseConfiguredSearchFields(): string[] {
+    const raw = this.config.options?.searchFields;
+    if (Array.isArray(raw)) {
+      return raw.filter((field): field is string => typeof field === 'string' && field.trim().length > 0);
+    }
+    return [];
+  }
 }

package/src/providers/vectordb/PostgreSQLProvider.ts

@@ -3,6 +3,14 @@ import { VectorDBConfig } from '../../config/RagConfig';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
 
+function stripInternalFilterKeys(filter?: Record<string, unknown>): Record<string, unknown> {
+  if (!filter) return {};
+
+  return Object.fromEntries(
+    Object.entries(filter).filter(([key]) => !key.startsWith('__'))
+  );
+}
+
 /**
  * PostgreSQLProvider — PostgreSQL implementation using the pgvector extension.
  */
@@ -103,8 +111,9 @@ export class PostgreSQLProvider extends BaseVectorProvider {
     const params: unknown[] = [vectorLiteral, topK];
     if (namespace) params.push(namespace);
 
-    if (filter && Object.keys(filter).length > 0) {
-      const filterConditions = Object.entries(filter)
+    const publicFilter = stripInternalFilterKeys(filter);
+    if (Object.keys(publicFilter).length > 0) {
+      const filterConditions = Object.entries(publicFilter)
         .map(([key, val]) => {
           const paramIdx = params.length + 1;
           params.push(JSON.stringify(val));