@retrivora-ai/rag-engine 0.2.9 → 0.3.1

package/src/core/Pipeline.ts

@@ -37,9 +37,9 @@ function extractQueryFieldHints(question: string): QueryFieldHint[] {
 
     const normalizedField = field
       ? field
-          .toLowerCase()
-          .replace(/[^a-z0-9]+/g, ' ')
-          .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, ' ')
+        .trim()
       : undefined;
 
     const key = `${normalizedField ?? '*'}::${normalizedValue.toLowerCase()}`;
@@ -61,6 +61,43 @@ function extractQueryFieldHints(question: string): QueryFieldHint[] {
     /\b(?:about|for|regarding)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
   ];
 
+  // Targeted patterns: map common person/company question forms to a `name` metadata hint
+  const personCompanyPatterns = [
+    /\bcompany(?:\s+name)?\s+(?:of|for)\s+["']?([^"'\n?.!,]{2,120})["']?(?=[?.!,]|$)/gi,
+    /\b(?:which|what)\s+company\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
+    /\bwhere\s+does\s+["']?([^"'\n?.!,]{2,120})["']?\s+work(?:\s+for|\s+at)?(?=[?.!,]|$)/gi,
+  ];
+
+  for (const pattern of personCompanyPatterns) {
+    for (const match of question.matchAll(pattern)) {
+      const name = match[1];
+      if (name) addHint(name, 'name');
+    }
+  }
+
+  // Universal patterns: capture common data types across domains so the system
+  // can construct metadata filters for arbitrary datasets. These are intentionally
+  // generic and provider-agnostic (e.g., `email`, `phone`, `date`, `amount`, `id`).
+  const universalPatterns: Array<{ regex: RegExp; field?: string; group?: number }> = [
+    { regex: /([a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/gi, field: 'email', group: 1 },
+    { regex: /(\+?\d[\d\-\.\s\(\)]{6,}\d)/g, field: 'phone', group: 1 },
+    { regex: /(\b\d{4}-\d{2}-\d{2}\b|\b\d{1,2}\/\d{1,2}\/\d{2,4}\b)/g, field: 'date', group: 1 },
+    { regex: /(\$\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?)/g, field: 'amount', group: 1 },
+    { regex: /\b(ID|id|identifier)[: ]\s*([A-Za-z0-9\-]{3,})\b/gi, field: 'id', group: 2 },
+    // Generic quoted phrase / proper-noun sequences as keywords (already partially handled above)
+    { regex: /"([^"]{2,120})"/g, group: 1 },
+    { regex: /'([^']{2,120})'/g, group: 1 },
+  ];
+
+  for (const p of universalPatterns) {
+    for (const match of question.matchAll(p.regex)) {
+      const val = p.group ? match[p.group] ?? match[0] : match[0];
+      if (!val) continue;
+      if (p.field) addHint(val, p.field);
+      else addHint(val);
+    }
+  }
+
   for (const pattern of naturalQuestionPatterns) {
     for (const match of question.matchAll(pattern)) {
       const value = match[2] ?? match[1];
@@ -89,7 +126,7 @@ function extractQueryFieldHints(question: string): QueryFieldHint[] {
     }
   }
 
-  for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){1,3}\b/g)) {
+  for (const match of question.matchAll(/\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,3}\b/g)) {
     addHint(match[0]);
   }
 
@@ -257,7 +294,7 @@ export class Pipeline {
       const filter = buildQueryFilter(question, fieldHints) as Record<string, unknown>;
 
       // Keep the raw hints available as well for providers that want richer parsing
-      filter.__fieldHints = fieldHints;
+      filter.__entityHints = fieldHints;
 
       const rawMatches = await this.vectorDB.query(queryVector, topK, ns, filter);
 

package/src/providers/vectordb/MultiTablePostgresProvider.ts

@@ -1,44 +1,11 @@
+
+
+
 import { Pool, PoolClient } from 'pg';
 import { BaseVectorProvider } from './BaseVectorProvider';
 import { VectorMatch, UpsertDocument } from '../../types';
 import { VectorDBConfig } from '../../config/RagConfig';
 
-interface QueryFieldHint {
-  field?: string;
-  value: string;
-}
-
-interface TableSearchConfig {
-  availableFields: string[];
-  searchableFields: string[];
-  normalizedFieldMap: Map<string, string>;
-}
-
-function normalizeFieldName(value: string): string {
-  return value.toLowerCase().replace(/[^a-z0-9]+/g, '');
-}
-
-function sanitizeFilterHints(rawFilter?: Record<string, unknown>): QueryFieldHint[] {
-  if (!Array.isArray(rawFilter?.__fieldHints)) return [];
-
-  return (rawFilter.__fieldHints as unknown[])
-    .filter((hint): hint is QueryFieldHint => typeof hint === 'object' && hint !== null)
-    .map((hint) => ({
-      value: typeof hint.value === 'string' ? hint.value.trim().toLowerCase() : '',
-      ...(typeof hint.field === 'string' && hint.field.trim()
-        ? { field: normalizeFieldName(hint.field) }
-        : {}),
-    }))
-    .filter((hint) => Boolean(hint.value));
-}
-
-function stripInternalFilterKeys(filter?: Record<string, unknown>): Record<string, unknown> {
-  if (!filter) return {};
-
-  return Object.fromEntries(
-    Object.entries(filter).filter(([key]) => !key.startsWith('__'))
-  );
-}
 
 /**
  * MultiTablePostgresProvider — PostgreSQL implementation that searches across
@@ -53,7 +20,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
   private readonly dimensions: number;
   private readonly connectionString: string;
   private tables: string[];
-  private tableSearchConfig = new Map<string, TableSearchConfig>();
+  private searchFields: string[];
 
   constructor(config: VectorDBConfig) {
     super(config);
@@ -80,6 +47,16 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
         'Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables.'
       );
     }
+
+    // Exact match fields can come from options.searchFields or be derived from VECTOR_DB_SEARCH_FIELDS env var
+    const rawSearchFields: string | string[] =
+      (opts.searchFields as string | string[]) ??
+      process.env.VECTOR_DB_SEARCH_FIELDS ??
+      ['name', 'product_name', 'productname', 'title'];
+
+    this.searchFields = typeof rawSearchFields === 'string'
+      ? rawSearchFields.split(',').map((f) => f.trim()).filter(Boolean)
+      : rawSearchFields;
   }
 
   async initialize(): Promise<void> {
@@ -114,7 +91,6 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       if (this.tables.length === 0) {
         console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
       } else {
-        this.tableSearchConfig = await this.loadTableSearchConfig(client, this.tables);
         console.log(
           `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(', ')}`
         );
@@ -162,74 +138,50 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
 
     console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
 
-    const queryText = _filter?.__queryText as string | undefined;
-    const fieldHints = sanitizeFilterHints(_filter);
-    const explicitFilter = stripInternalFilterKeys(_filter);
+    const queryText = _filter?.queryText as string | undefined;
+    const entityHints = Array.isArray(_filter?.__entityHints)
+      ? (_filter.__entityHints as unknown[])
+        .filter((hint): hint is { value: string; field?: string } =>
+          typeof hint === 'object' && hint !== null && typeof (hint as Record<string, unknown>).value === 'string'
+        )
+        .map((hint) => hint.value.trim().toLowerCase())
+        .filter(Boolean)
+      : [];
+
+    console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
+    console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(', ')}]`);
 
     const queryPromises = this.tables.map(async (table) => {
       try {
         let sqlQuery = '';
         let params: unknown[] = [];
-        const tableConfig = this.tableSearchConfig.get(table);
-        const availableFields = tableConfig?.availableFields ?? [];
-        const searchableFields = tableConfig?.searchableFields ?? [];
-        const normalizedFieldMap = tableConfig?.normalizedFieldMap ?? new Map<string, string>();
-        const whereConditions: string[] = [];
 
         if (queryText) {
-          const genericValueHints = fieldHints
-            .filter((hint) => !hint.field)
-            .map((hint) => hint.value);
-
-          params = genericValueHints.length > 0
-            ? [vectorLiteral, queryText, genericValueHints]
-            : [vectorLiteral, queryText];
-
-          const genericValueParamIndex = genericValueHints.length > 0 ? 3 : -1;
-
-          const exactValueScoreExpr = genericValueHints.length > 0 && searchableFields.length > 0
-            ? `GREATEST(${searchableFields.map((field) =>
-              `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${field}', '')) = ANY($${genericValueParamIndex}::text[]) THEN 1 ELSE 0 END`
-            ).join(', ')})`
-            : '0';
-
-          const fieldSpecificChecks = fieldHints
-            .filter((hint) => hint.field)
-            .map((hint) => {
-              const column = hint.field ? normalizedFieldMap.get(hint.field) : undefined;
-              if (!column) return null;
-              params.push(hint.value);
-              const paramIndex = params.length;
-              return `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${column}', '')) = $${paramIndex}::text THEN 1 ELSE 0 END`;
-            })
-            .filter((expr): expr is string => Boolean(expr));
-
-          const exactFieldScoreExpr = fieldSpecificChecks.length > 0
-            ? `GREATEST(${fieldSpecificChecks.join(', ')})`
-            : '0';
+          const hasEntityHints = entityHints.length > 0;
+          const exactNameScoreExpr = hasEntityHints
+            ? `+ (
+                 SELECT COALESCE(MAX(
+                   CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(', ')})
+                   THEN 1.0 ELSE 0.0 END
+                 ), 0)
+                 FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
+                 WHERE key IN (${this.searchFields.map(f => `'${f}'`).join(', ')})
+               ) * 3.0`
+            : '';
 
           // Actual Hybrid Search: Semantic Vector Score + Full-Text Keyword Rank
-          // We convert plainto_tsquery's AND (&) operator to OR (|) so that natural language queries
-          // (which contain words that are not present verbatim in the row text) still match the intended records.
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS vector_score,
-             COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
-             ${exactValueScoreExpr} AS exact_value_score,
-             ${exactFieldScoreExpr} AS exact_field_score,
-             (
-               (1 - (embedding <=> $1::vector)) +
-               (COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) +
-               (${exactValueScoreExpr} * 3.0) +
-               (${exactFieldScoreExpr} * 5.0)
-             ) AS hybrid_score
+             COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
+             ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
             FROM "${table}" t
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
+          params = [vectorLiteral, queryText];
         } else {
           // Fallback to Vector-only Search
-          params = [vectorLiteral];
           sqlQuery = `
             SELECT *, 
              (1 - (embedding <=> $1::vector)) AS hybrid_score
@@ -237,19 +189,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
             ORDER BY hybrid_score DESC
             LIMIT 50
           `;
-        }
-
-        for (const [key, value] of Object.entries(explicitFilter)) {
-          const column = availableFields.includes(key)
-            ? key
-            : normalizedFieldMap.get(normalizeFieldName(key));
-          if (!column) continue;
-          params.push(String(value));
-          whereConditions.push(`COALESCE(to_jsonb(t)->>'${column}', '') = $${params.length}::text`);
-        }
-
-        if (whereConditions.length > 0) {
-          sqlQuery = sqlQuery.replace(`FROM "${table}" t`, `FROM "${table}" t\n            WHERE ${whereConditions.join(' AND ')}`);
+          params = [vectorLiteral];
         }
 
         const result = await this.pool.query(sqlQuery, params);
@@ -266,8 +206,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
           delete rest.embedding;
           delete rest.vector_score;
           delete rest.keyword_score;
-          delete rest.exact_value_score;
-          delete rest.exact_field_score;
+          delete rest.exact_name_score;
 
           const content = `[TYPE: ${table.replace(/s$/, '').toUpperCase()}]\n` +
             Object.entries(rest)
@@ -297,14 +236,32 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       allResults.push(...tableResults);
     }
 
-    const finalSorted = allResults.sort((a, b) => b.score - a.score);
+    // 1. Group results by table to ensure we take the top ones from each
+    const resultsByTable: Record<string, VectorMatch[]> = {};
+    for (const res of allResults) {
+      const table = res.metadata?.source_table as string;
+      if (!resultsByTable[table]) resultsByTable[table] = [];
+      resultsByTable[table].push(res);
+    }
+
+    // 2. Take top 3 from EACH table guaranteed, then fill the rest with overall best
+    const balancedResults: VectorMatch[] = [];
+    const tables = Object.keys(resultsByTable);
+
+    // First, ensure every table's best match is included
+    for (const table of tables) {
+      balancedResults.push(...resultsByTable[table].slice(0, 3));
+    }
+
+    // 3. Sort the balanced list by score
+    const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
 
     if (finalSorted.length > 0) {
       console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
       console.log(`[MultiTablePostgresProvider] Final top match from "${finalSorted[0].metadata?.source_table ?? 'unknown'}" with score ${finalSorted[0].score.toFixed(4)}`);
     }
 
-    return finalSorted.slice(0, topK);
+    return finalSorted.slice(0, Math.max(topK, 15));
   }
 
   async delete(_id: string | number, _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
@@ -329,67 +286,4 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
       await this.pool.end();
     }
   }
-
-  private async loadTableSearchConfig(
-    client: PoolClient,
-    tables: string[]
-  ): Promise<Map<string, TableSearchConfig>> {
-    if (tables.length === 0) return new Map();
-
-    const configuredSearchFields = this.parseConfiguredSearchFields();
-    const result = await client.query(`
-      SELECT table_name, column_name, data_type
-      FROM information_schema.columns
-      WHERE table_schema = 'public'
-        AND table_name = ANY($1::text[])
-      ORDER BY ordinal_position
-    `, [tables]);
-
-    const rowsByTable = new Map<string, Array<{ column_name: string; data_type: string }>>();
-    for (const row of result.rows as Array<{ table_name: string; column_name: string; data_type: string }>) {
-      const rows = rowsByTable.get(row.table_name) ?? [];
-      rows.push({ column_name: row.column_name, data_type: row.data_type });
-      rowsByTable.set(row.table_name, rows);
-    }
-
-    const configByTable = new Map<string, TableSearchConfig>();
-    for (const table of tables) {
-      const columns = rowsByTable.get(table) ?? [];
-      const availableFields = columns
-        .filter(({ column_name }) => column_name !== 'embedding')
-        .map(({ column_name }) => column_name);
-
-      const inferredFields = columns
-        .filter(({ column_name, data_type }) =>
-          column_name !== 'embedding' &&
-          !['ARRAY', 'json', 'jsonb', 'bytea', 'tsvector', 'USER-DEFINED'].includes(data_type)
-        )
-        .map(({ column_name }) => column_name);
-
-      const searchableFields = configuredSearchFields.length > 0
-        ? configuredSearchFields.filter((field) => columns.some((column) => column.column_name === field))
-        : inferredFields;
-
-      const normalizedFieldMap = new Map<string, string>();
-      for (const field of availableFields) {
-        normalizedFieldMap.set(normalizeFieldName(field), field);
-      }
-
-      configByTable.set(table, {
-        availableFields,
-        searchableFields,
-        normalizedFieldMap,
-      });
-    }
-
-    return configByTable;
-  }
-
-  private parseConfiguredSearchFields(): string[] {
-    const raw = this.config.options?.searchFields;
-    if (Array.isArray(raw)) {
-      return raw.filter((field): field is string => typeof field === 'string' && field.trim().length > 0);
-    }
-    return [];
-  }
 }