@retrivora-ai/rag-engine 2.4.3 → 2.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{BatchProcessor-zbQMaZHm.d.mts → BatchProcessor-Csy_K1qQ.d.mts} +2 -0
- package/dist/{BatchProcessor-D42E706a.d.ts → BatchProcessor-Dg18zZGk.d.ts} +2 -0
- package/dist/handlers/index.js +27 -4
- package/dist/handlers/index.mjs +27 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +12 -10
- package/dist/index.mjs +12 -10
- package/dist/server.d.mts +2 -2
- package/dist/server.d.ts +2 -2
- package/dist/server.js +28 -4
- package/dist/server.mjs +28 -4
- package/package.json +1 -3
- package/src/LicensedRetrivora.ts +0 -278
- package/src/app/api/chat/route.ts +0 -4
- package/src/app/api/health/route.ts +0 -5
- package/src/app/api/ingest/route.ts +0 -27
- package/src/app/api/suggestions/route.ts +0 -5
- package/src/app/api/upload/route.ts +0 -5
- package/src/app/constants.tsx +0 -270
- package/src/app/favicon.ico +0 -0
- package/src/app/globals.css +0 -449
- package/src/app/layout.tsx +0 -22
- package/src/app/page.tsx +0 -64
- package/src/app/types.ts +0 -30
- package/src/browser.tsx +0 -127
- package/src/components/ChatWidget.tsx +0 -393
- package/src/components/ChatWindow.tsx +0 -952
- package/src/components/CodeViewer.tsx +0 -56
- package/src/components/ConfigProvider.tsx +0 -141
- package/src/components/DocumentUpload.tsx +0 -354
- package/src/components/DynamicChart.tsx +0 -297
- package/src/components/HourglassLoader.tsx +0 -87
- package/src/components/MarkdownComponents.tsx +0 -186
- package/src/components/MessageBubble.tsx +0 -498
- package/src/components/ObservabilityPanel.tsx +0 -374
- package/src/components/ProductCard.tsx +0 -107
- package/src/components/ProductCarousel.tsx +0 -97
- package/src/components/SourceCard.tsx +0 -61
- package/src/components/ThemeProvider.tsx +0 -8
- package/src/components/ThemeToggle.tsx +0 -29
- package/src/components/ThinkingBlock.tsx +0 -75
- package/src/components/UIDispatcher.tsx +0 -342
- package/src/components/VisualizationRenderer.tsx +0 -621
- package/src/components/constants.tsx +0 -503
- package/src/config/ConfigBuilder.ts +0 -285
- package/src/config/EmbeddingStrategy.ts +0 -148
- package/src/config/LicenseConfigResolver.ts +0 -298
- package/src/config/RagConfig.ts +0 -328
- package/src/config/UniversalProfiles.ts +0 -82
- package/src/config/constants.ts +0 -64
- package/src/config/serverConfig.ts +0 -378
- package/src/config/uiConstants.ts +0 -47
- package/src/core/BatchProcessor.ts +0 -385
- package/src/core/CircuitBreaker.ts +0 -118
- package/src/core/ConfigFetcher.ts +0 -220
- package/src/core/ConfigResolver.ts +0 -165
- package/src/core/ConfigValidator.ts +0 -197
- package/src/core/DatabaseStorage.ts +0 -526
- package/src/core/FreeTierLimitsGuard.ts +0 -281
- package/src/core/LLMRouter.ts +0 -79
- package/src/core/LangChainAgent.ts +0 -132
- package/src/core/LicenseValidator.ts +0 -284
- package/src/core/LicenseVerifier.ts +0 -261
- package/src/core/MultiAgentCoordinator.ts +0 -239
- package/src/core/Pipeline.ts +0 -1279
- package/src/core/PluginManager.ts +0 -277
- package/src/core/ProjectSettings.ts +0 -247
- package/src/core/ProviderHealthCheck.ts +0 -133
- package/src/core/ProviderInterfaces.ts +0 -37
- package/src/core/ProviderRegistry.ts +0 -156
- package/src/core/QueryProcessor.ts +0 -399
- package/src/core/Retrivora.ts +0 -98
- package/src/core/VectorPlugin.ts +0 -193
- package/src/core/mcp.ts +0 -261
- package/src/exceptions/index.ts +0 -131
- package/src/handlers/index.ts +0 -1625
- package/src/hooks/useHydrated.ts +0 -13
- package/src/hooks/useProjectSettings.ts +0 -105
- package/src/hooks/useRagChat.ts +0 -436
- package/src/hooks/useStoredMessages.ts +0 -64
- package/src/index.css +0 -4269
- package/src/index.ts +0 -137
- package/src/lib/plugin.ts +0 -24
- package/src/llm/ILLMProvider.ts +0 -60
- package/src/llm/LLMFactory.ts +0 -141
- package/src/llm/providers/AnthropicProvider.ts +0 -215
- package/src/llm/providers/GeminiProvider.ts +0 -311
- package/src/llm/providers/GroqProvider.ts +0 -184
- package/src/llm/providers/OllamaProvider.ts +0 -232
- package/src/llm/providers/OpenAIProvider.ts +0 -188
- package/src/llm/providers/QwenProvider.ts +0 -199
- package/src/llm/providers/UniversalLLMAdapter.ts +0 -422
- package/src/llm/utils.ts +0 -46
- package/src/providers/graphdb/BaseGraphProvider.ts +0 -43
- package/src/providers/graphdb/SimpleGraphProvider.ts +0 -66
- package/src/providers/vectordb/BaseVectorProvider.ts +0 -98
- package/src/providers/vectordb/ChromaDBProvider.ts +0 -130
- package/src/providers/vectordb/MilvusProvider.ts +0 -134
- package/src/providers/vectordb/MongoDBProvider.ts +0 -250
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +0 -461
- package/src/providers/vectordb/PineconeProvider.ts +0 -295
- package/src/providers/vectordb/PostgreSQLProvider.ts +0 -227
- package/src/providers/vectordb/QdrantProvider.ts +0 -294
- package/src/providers/vectordb/RedisProvider.ts +0 -105
- package/src/providers/vectordb/UniversalVectorProvider.ts +0 -221
- package/src/providers/vectordb/WeaviateProvider.ts +0 -172
- package/src/rag/DocumentChunker.ts +0 -144
- package/src/rag/EntityExtractor.ts +0 -90
- package/src/rag/LlamaIndexIngestor.ts +0 -60
- package/src/rag/Reranker.ts +0 -106
- package/src/rendering/IntentClassifier.ts +0 -167
- package/src/rendering/RendererRegistry.ts +0 -164
- package/src/rendering/RuleEngine.ts +0 -67
- package/src/rendering/VisualizationDecisionEngine.ts +0 -164
- package/src/rendering/__tests__/VisualizationDecisionEngine.test.ts +0 -237
- package/src/rendering/index.ts +0 -12
- package/src/rendering/rules/Rule1SpecificInfoRule.ts +0 -28
- package/src/rendering/rules/Rule2ComparisonRule.ts +0 -24
- package/src/rendering/rules/Rule3ProductDiscoveryRule.ts +0 -28
- package/src/rendering/rules/Rule4AnalyticalRule.ts +0 -38
- package/src/rendering/rules/Rule5MixedResponseRule.ts +0 -29
- package/src/rendering/rules/Rule6SmallResultSetRule.ts +0 -24
- package/src/rendering/rules/Rule7LargeTableRule.ts +0 -35
- package/src/rendering/types.ts +0 -96
- package/src/server.ts +0 -156
- package/src/tailwind.css +0 -194
- package/src/telemetry/TelemetryService.ts +0 -330
- package/src/types/chat.ts +0 -89
- package/src/types/index.ts +0 -278
- package/src/types/pdf-parse.d.ts +0 -13
- package/src/types/props.ts +0 -169
- package/src/utils/DocumentParser.ts +0 -111
- package/src/utils/ProductExtractor.ts +0 -350
- package/src/utils/SchemaMapper.ts +0 -176
- package/src/utils/UITransformer.ts +0 -1812
- package/src/utils/synonyms.ts +0 -80
- package/src/utils/templateUtils.ts +0 -78
- package/src/version.ts +0 -6
|
@@ -1,461 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import { Pool, PoolClient } from 'pg';
|
|
5
|
-
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
6
|
-
import { VectorMatch, UpsertDocument } from '../../types';
|
|
7
|
-
import { VectorDBConfig } from '../../config/RagConfig';
|
|
8
|
-
import { FIELD_SYNONYMS } from '../../utils/synonyms';
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
/**
|
|
12
|
-
* MultiTablePostgresProvider — PostgreSQL implementation that searches across
|
|
13
|
-
* multiple existing tables with pre-existing embeddings.
|
|
14
|
-
*
|
|
15
|
-
* Extends BaseVectorProvider so it can be registered with ProviderRegistry.
|
|
16
|
-
* Upsert operations are not supported — data is assumed to be managed externally
|
|
17
|
-
* or via custom ingestion scripts that write directly to each table.
|
|
18
|
-
*/
|
|
19
|
-
export class MultiTablePostgresProvider extends BaseVectorProvider {
|
|
20
|
-
private pool!: Pool;
|
|
21
|
-
private readonly dimensions: number;
|
|
22
|
-
private readonly connectionString: string;
|
|
23
|
-
private tables: string[];
|
|
24
|
-
private searchFields: string[];
|
|
25
|
-
private readonly uploadTable: string;
|
|
26
|
-
|
|
27
|
-
constructor(config: VectorDBConfig) {
|
|
28
|
-
super(config);
|
|
29
|
-
const opts = config.options || {};
|
|
30
|
-
if (!opts.connectionString) {
|
|
31
|
-
throw new Error('[MultiTablePostgresProvider] options.connectionString is required');
|
|
32
|
-
}
|
|
33
|
-
this.connectionString = opts.connectionString as string;
|
|
34
|
-
this.dimensions = (opts.dimensions as number) ?? 768;
|
|
35
|
-
|
|
36
|
-
// The provider will dynamically discover tables during initialize().
|
|
37
|
-
this.tables = [];
|
|
38
|
-
|
|
39
|
-
// Exact match fields can come from options.searchFields or be derived from VECTOR_DB_SEARCH_FIELDS env var
|
|
40
|
-
const rawSearchFields: string | string[] =
|
|
41
|
-
(opts.searchFields as string | string[]) ??
|
|
42
|
-
process.env.VECTOR_DB_SEARCH_FIELDS ??
|
|
43
|
-
['name', 'product_name', 'productname', 'title'];
|
|
44
|
-
|
|
45
|
-
this.searchFields = typeof rawSearchFields === 'string'
|
|
46
|
-
? rawSearchFields.split(',').map((f) => f.trim()).filter(Boolean)
|
|
47
|
-
: rawSearchFields;
|
|
48
|
-
|
|
49
|
-
this.uploadTable = (opts.uploadTable as string) || 'document_chunks';
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
async initialize(): Promise<void> {
|
|
53
|
-
this.pool = new Pool({ connectionString: this.connectionString });
|
|
54
|
-
|
|
55
|
-
// Dynamically discover all tables that have an 'embedding' column
|
|
56
|
-
const client: PoolClient = await this.pool.connect();
|
|
57
|
-
try {
|
|
58
|
-
await client.query('CREATE EXTENSION IF NOT EXISTS vector');
|
|
59
|
-
|
|
60
|
-
// Ensure the upload table exists
|
|
61
|
-
await client.query(`
|
|
62
|
-
CREATE TABLE IF NOT EXISTS ${this.uploadTable} (
|
|
63
|
-
id TEXT PRIMARY KEY,
|
|
64
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
65
|
-
content TEXT NOT NULL,
|
|
66
|
-
metadata JSONB,
|
|
67
|
-
embedding VECTOR(${this.dimensions})
|
|
68
|
-
)
|
|
69
|
-
`);
|
|
70
|
-
await client.query(`
|
|
71
|
-
CREATE INDEX IF NOT EXISTS ${this.uploadTable}_embedding_idx
|
|
72
|
-
ON ${this.uploadTable}
|
|
73
|
-
USING hnsw (embedding vector_cosine_ops)
|
|
74
|
-
`);
|
|
75
|
-
await client.query(`
|
|
76
|
-
CREATE INDEX IF NOT EXISTS "${this.uploadTable}_fts_idx"
|
|
77
|
-
ON "${this.uploadTable}"
|
|
78
|
-
USING gin (to_tsvector('english', content))
|
|
79
|
-
`);
|
|
80
|
-
|
|
81
|
-
const res = await client.query(`
|
|
82
|
-
SELECT table_name
|
|
83
|
-
FROM information_schema.columns
|
|
84
|
-
WHERE column_name = 'embedding'
|
|
85
|
-
AND table_schema = 'public'
|
|
86
|
-
`);
|
|
87
|
-
|
|
88
|
-
// Dynamically use all tables that have an embedding column
|
|
89
|
-
this.tables = res.rows.map(r => r.table_name);
|
|
90
|
-
|
|
91
|
-
if (this.tables.length === 0) {
|
|
92
|
-
console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
|
|
93
|
-
} else {
|
|
94
|
-
// console.log(
|
|
95
|
-
// `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(', ')}`
|
|
96
|
-
// );
|
|
97
|
-
}
|
|
98
|
-
} finally {
|
|
99
|
-
client.release();
|
|
100
|
-
}
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* Upsert a document by dynamically provisioning a table and columns.
|
|
105
|
-
*/
|
|
106
|
-
async upsert(doc: UpsertDocument, namespace = ''): Promise<void> {
|
|
107
|
-
await this.batchUpsert([doc], namespace);
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
/**
|
|
111
|
-
* Batch upsert documents by dynamically provisioning tables and columns based on CSV headers.
|
|
112
|
-
*/
|
|
113
|
-
async batchUpsert(docs: UpsertDocument[], namespace = ''): Promise<void> {
|
|
114
|
-
if (docs.length === 0) return;
|
|
115
|
-
|
|
116
|
-
// Group docs by fileName to support dynamic table creation per CSV
|
|
117
|
-
const docsByFile: Record<string, UpsertDocument[]> = {};
|
|
118
|
-
for (const doc of docs) {
|
|
119
|
-
const fileName = (doc.metadata?.fileName as string) || this.uploadTable;
|
|
120
|
-
if (!docsByFile[fileName]) docsByFile[fileName] = [];
|
|
121
|
-
docsByFile[fileName].push(doc);
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
const client = await this.pool.connect();
|
|
125
|
-
try {
|
|
126
|
-
await client.query('BEGIN');
|
|
127
|
-
|
|
128
|
-
for (const [fileName, fileDocs] of Object.entries(docsByFile)) {
|
|
129
|
-
// Sanitize table name (e.g., "jewelery.csv" -> "jewelery")
|
|
130
|
-
let tableName = fileName;
|
|
131
|
-
if (tableName.toLowerCase().endsWith('.csv')) {
|
|
132
|
-
tableName = tableName.slice(0, -4);
|
|
133
|
-
}
|
|
134
|
-
tableName = tableName.replace(/[^a-z0-9_]/gi, '_').toLowerCase() || this.uploadTable;
|
|
135
|
-
|
|
136
|
-
// Extract headers from the first doc's metadata
|
|
137
|
-
const firstMeta = fileDocs[0].metadata || {};
|
|
138
|
-
let csvHeaders: string[] = [];
|
|
139
|
-
if (Array.isArray(firstMeta.csvHeaders)) {
|
|
140
|
-
csvHeaders = firstMeta.csvHeaders;
|
|
141
|
-
} else {
|
|
142
|
-
const systemKeys = ['fileName', 'fileSize', 'fileType', 'uploadedAt', 'dimension', 'chunkIndex', 'id', 'namespace', 'content', 'metadata', 'embedding', 'docId', 'docid', 'chunkId', 'chunkid', 'csvHeaders', 'csvheaders'];
|
|
143
|
-
csvHeaders = Object.keys(firstMeta).filter(k => !systemKeys.includes(k) && !systemKeys.includes(k.toLowerCase()));
|
|
144
|
-
}
|
|
145
|
-
|
|
146
|
-
// 1. Create Table Dynamically
|
|
147
|
-
const columnDefs = csvHeaders.map(h => `"${h}" TEXT`).join(',\n ');
|
|
148
|
-
const createTableSql = `
|
|
149
|
-
CREATE TABLE IF NOT EXISTS "${tableName}" (
|
|
150
|
-
id TEXT PRIMARY KEY,
|
|
151
|
-
${columnDefs ? columnDefs + ',' : ''}
|
|
152
|
-
namespace TEXT NOT NULL DEFAULT '',
|
|
153
|
-
content TEXT NOT NULL,
|
|
154
|
-
metadata JSONB,
|
|
155
|
-
embedding VECTOR(${this.dimensions})
|
|
156
|
-
)
|
|
157
|
-
`;
|
|
158
|
-
await client.query(createTableSql);
|
|
159
|
-
|
|
160
|
-
// Create Index
|
|
161
|
-
await client.query(`
|
|
162
|
-
CREATE INDEX IF NOT EXISTS "${tableName}_embedding_idx"
|
|
163
|
-
ON "${tableName}"
|
|
164
|
-
USING hnsw (embedding vector_cosine_ops)
|
|
165
|
-
`);
|
|
166
|
-
await client.query(`
|
|
167
|
-
CREATE INDEX IF NOT EXISTS "${tableName}_fts_idx"
|
|
168
|
-
ON "${tableName}"
|
|
169
|
-
USING gin (to_tsvector('english', content))
|
|
170
|
-
`);
|
|
171
|
-
|
|
172
|
-
// Ensure this new table is added to our search scope
|
|
173
|
-
if (!this.tables.includes(tableName)) {
|
|
174
|
-
this.tables.push(tableName);
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
// 2. Insert Data
|
|
178
|
-
const BATCH_SIZE = 50;
|
|
179
|
-
for (let i = 0; i < fileDocs.length; i += BATCH_SIZE) {
|
|
180
|
-
const batch = fileDocs.slice(i, i + BATCH_SIZE);
|
|
181
|
-
const values: unknown[] = [];
|
|
182
|
-
|
|
183
|
-
const headerColumns = csvHeaders.map(h => `"${h}"`).join(', ');
|
|
184
|
-
const allColumns = `id, ${headerColumns ? headerColumns + ', ' : ''}namespace, content, metadata, embedding`;
|
|
185
|
-
|
|
186
|
-
const valuePlaceholders = batch.map((doc, idx) => {
|
|
187
|
-
const offset = idx * (5 + csvHeaders.length);
|
|
188
|
-
|
|
189
|
-
values.push(doc.id);
|
|
190
|
-
// Push header values
|
|
191
|
-
for (const h of csvHeaders) {
|
|
192
|
-
values.push(String(doc.metadata?.[h] || ''));
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
values.push(namespace, doc.content, JSON.stringify(doc.metadata ?? {}), `[${doc.vector.join(',')}]`);
|
|
196
|
-
|
|
197
|
-
const docPlaceholders = [];
|
|
198
|
-
for (let j = 1; j <= 5 + csvHeaders.length; j++) {
|
|
199
|
-
if (j === 5 + csvHeaders.length) {
|
|
200
|
-
docPlaceholders.push(`$${offset + j}::vector`);
|
|
201
|
-
} else {
|
|
202
|
-
docPlaceholders.push(`$${offset + j}`);
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
return `(${docPlaceholders.join(', ')})`;
|
|
207
|
-
}).join(', ');
|
|
208
|
-
|
|
209
|
-
const conflictUpdates = csvHeaders.map(h => `"${h}" = EXCLUDED."${h}"`).join(',\n ');
|
|
210
|
-
|
|
211
|
-
const query = `
|
|
212
|
-
INSERT INTO "${tableName}" (${allColumns})
|
|
213
|
-
VALUES ${valuePlaceholders}
|
|
214
|
-
ON CONFLICT (id) DO UPDATE
|
|
215
|
-
SET ${conflictUpdates ? conflictUpdates + ',' : ''}
|
|
216
|
-
namespace = EXCLUDED.namespace,
|
|
217
|
-
content = EXCLUDED.content,
|
|
218
|
-
metadata = EXCLUDED.metadata,
|
|
219
|
-
embedding = EXCLUDED.embedding
|
|
220
|
-
`;
|
|
221
|
-
|
|
222
|
-
await client.query(query, values);
|
|
223
|
-
}
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
await client.query('COMMIT');
|
|
227
|
-
} catch (error) {
|
|
228
|
-
await client.query('ROLLBACK');
|
|
229
|
-
throw error;
|
|
230
|
-
} finally {
|
|
231
|
-
client.release();
|
|
232
|
-
}
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
/**
|
|
236
|
-
* Query all configured tables and merge results, sorted by cosine similarity score.
|
|
237
|
-
*/
|
|
238
|
-
async query(
|
|
239
|
-
vector: number[],
|
|
240
|
-
topK: number,
|
|
241
|
-
_namespace?: string,
|
|
242
|
-
_filter?: Record<string, unknown>
|
|
243
|
-
): Promise<VectorMatch[]> {
|
|
244
|
-
if (!this.pool) {
|
|
245
|
-
throw new Error('[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.');
|
|
246
|
-
}
|
|
247
|
-
|
|
248
|
-
const vectorLiteral = `[${vector.join(',')}]`;
|
|
249
|
-
const allResults: VectorMatch[] = [];
|
|
250
|
-
|
|
251
|
-
//console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
252
|
-
|
|
253
|
-
const queryText = _filter?.queryText as string | undefined;
|
|
254
|
-
const entityHints = Array.isArray(_filter?.keywords)
|
|
255
|
-
? (_filter.keywords as string[])
|
|
256
|
-
.map((k) => k.trim().toLowerCase())
|
|
257
|
-
.filter(Boolean)
|
|
258
|
-
: [];
|
|
259
|
-
|
|
260
|
-
//console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
261
|
-
//console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(', ')}]`);
|
|
262
|
-
|
|
263
|
-
// Dynamic Keyword Search: Prioritize intelligently extracted hints over raw text
|
|
264
|
-
const getDynamicKeywordQuery = () => {
|
|
265
|
-
if (entityHints.length > 0) {
|
|
266
|
-
// Use extracted entities as high-signal keywords
|
|
267
|
-
return entityHints
|
|
268
|
-
.map(h => h.replace(/\s+/g, ' & ')) // Replace spaces with & for valid tsquery
|
|
269
|
-
.join(' & ');
|
|
270
|
-
}
|
|
271
|
-
|
|
272
|
-
if (queryText) {
|
|
273
|
-
// Fallback to cleaned raw text if no high-signal hints were extracted
|
|
274
|
-
// Added 'get', 'details', 'information', 'info' to stop words
|
|
275
|
-
return queryText
|
|
276
|
-
.toLowerCase()
|
|
277
|
-
.replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|organization[s]?|comp(?:any|anies)|of|category|find|list|browse|what|are|display|all|whose|that|which|have|has|having|greater|than|more|less|equal|equals|above|below|over|under|at|least|most|for|in|with|about|get|details|information|info)\b/g, '')
|
|
278
|
-
.replace(/\b\d+(?:\.\d+)?\b/g, '')
|
|
279
|
-
.trim()
|
|
280
|
-
.replace(/\s+/g, ' & ');
|
|
281
|
-
}
|
|
282
|
-
return '';
|
|
283
|
-
};
|
|
284
|
-
|
|
285
|
-
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
286
|
-
const tableLimit = Math.max(topK, 50);
|
|
287
|
-
|
|
288
|
-
const metadataFilters = _filter?.metadata as Record<string, string> | undefined;
|
|
289
|
-
|
|
290
|
-
const queryPromises = this.tables.map(async (table) => {
|
|
291
|
-
try {
|
|
292
|
-
let sqlQuery = '';
|
|
293
|
-
let params: unknown[] = [];
|
|
294
|
-
let whereClause = '';
|
|
295
|
-
const filterParams: unknown[] = [];
|
|
296
|
-
|
|
297
|
-
// Build case-insensitive SQL metadata filter conditions with synonym-awareness
|
|
298
|
-
if (metadataFilters && Object.keys(metadataFilters).length > 0) {
|
|
299
|
-
const conditions = Object.entries(metadataFilters)
|
|
300
|
-
.map(([key, val]) => {
|
|
301
|
-
filterParams.push(String(val));
|
|
302
|
-
// Offset based on whether it is vector + keyword or vector only
|
|
303
|
-
const baseOffset = queryText && dynamicKeywordQuery ? 2 : 1;
|
|
304
|
-
const paramIdx = baseOffset + filterParams.length;
|
|
305
|
-
|
|
306
|
-
// Check if we have synonyms for this key (e.g. 'brand' -> 'vendor', 'manufacturer')
|
|
307
|
-
const synonyms = FIELD_SYNONYMS[key] ?? [];
|
|
308
|
-
const keysToCheck = [key, ...synonyms];
|
|
309
|
-
|
|
310
|
-
// Map each candidate key to LOWER(coalesce(metadata->>'key', to_jsonb(t)->>'key'))
|
|
311
|
-
const coalesceExprs = keysToCheck.flatMap(k => [
|
|
312
|
-
`metadata->>'${k}'`,
|
|
313
|
-
`to_jsonb(t)->>'${k}'`
|
|
314
|
-
]);
|
|
315
|
-
return `COALESCE(${coalesceExprs.map(expr => `LOWER(${expr})`).join(', ')}) = LOWER($${paramIdx})`;
|
|
316
|
-
});
|
|
317
|
-
whereClause = `WHERE ${conditions.join(' AND ')}`;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
if (queryText && dynamicKeywordQuery) {
|
|
321
|
-
const hasEntityHints = entityHints.length > 0;
|
|
322
|
-
const exactNameScoreExpr = hasEntityHints
|
|
323
|
-
? `+ (
|
|
324
|
-
SELECT COALESCE(MAX(
|
|
325
|
-
CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(', ')})
|
|
326
|
-
THEN 1.0 ELSE 0.0 END
|
|
327
|
-
), 0)
|
|
328
|
-
FROM jsonb_each_text(COALESCE(v.metadata, k.metadata)) AS kv(key, val)
|
|
329
|
-
WHERE key IN (${this.searchFields.map(f => `'${f}'`).join(', ')})
|
|
330
|
-
) * 3.0`
|
|
331
|
-
: '';
|
|
332
|
-
|
|
333
|
-
// Optimized Subquery CTE Hybrid Search (Uses HNSW vector index & GIN fulltext index)
|
|
334
|
-
sqlQuery = `
|
|
335
|
-
WITH vector_search AS (
|
|
336
|
-
SELECT *, (1 - (embedding <=> $1::vector)) AS vector_score
|
|
337
|
-
FROM "${table}" t
|
|
338
|
-
${whereClause}
|
|
339
|
-
ORDER BY embedding <=> $1::vector ASC
|
|
340
|
-
LIMIT ${tableLimit}
|
|
341
|
-
),
|
|
342
|
-
keyword_search AS (
|
|
343
|
-
SELECT *, COALESCE(ts_rank(to_tsvector('english', content), to_tsquery('english', $2)), 0) AS keyword_score
|
|
344
|
-
FROM "${table}" t
|
|
345
|
-
WHERE ${whereClause ? whereClause.replace('WHERE', '') : 'TRUE'}
|
|
346
|
-
AND to_tsvector('english', content) @@ to_tsquery('english', $2)
|
|
347
|
-
ORDER BY keyword_score DESC
|
|
348
|
-
LIMIT ${tableLimit}
|
|
349
|
-
)
|
|
350
|
-
SELECT
|
|
351
|
-
COALESCE(v.id, k.id) AS id,
|
|
352
|
-
COALESCE(v.namespace, k.namespace) AS namespace,
|
|
353
|
-
COALESCE(v.content, k.content) AS content,
|
|
354
|
-
COALESCE(v.metadata, k.metadata) AS metadata,
|
|
355
|
-
COALESCE(v.vector_score, 0) AS vector_score,
|
|
356
|
-
COALESCE(k.keyword_score, 0) AS keyword_score,
|
|
357
|
-
(COALESCE(v.vector_score, 0) + COALESCE(k.keyword_score, 0) * 2.0 ${exactNameScoreExpr}) AS hybrid_score
|
|
358
|
-
FROM vector_search v
|
|
359
|
-
FULL OUTER JOIN keyword_search k ON v.id = k.id
|
|
360
|
-
ORDER BY hybrid_score DESC
|
|
361
|
-
LIMIT ${tableLimit}
|
|
362
|
-
`;
|
|
363
|
-
params = [vectorLiteral, dynamicKeywordQuery, ...filterParams];
|
|
364
|
-
} else {
|
|
365
|
-
// Fallback to Vector-only Search (ordered ASC to use HNSW index)
|
|
366
|
-
sqlQuery = `
|
|
367
|
-
SELECT *,
|
|
368
|
-
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
369
|
-
FROM "${table}" t
|
|
370
|
-
${whereClause}
|
|
371
|
-
ORDER BY embedding <=> $1::vector ASC
|
|
372
|
-
LIMIT ${tableLimit}
|
|
373
|
-
`;
|
|
374
|
-
params = [vectorLiteral, ...filterParams];
|
|
375
|
-
}
|
|
376
|
-
|
|
377
|
-
const result = await this.pool.query(sqlQuery, params);
|
|
378
|
-
|
|
379
|
-
if (result.rowCount && result.rowCount > 0) {
|
|
380
|
-
//console.log(` ✅ Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
|
|
381
|
-
} else {
|
|
382
|
-
//console.log(` ⚪ Table "${table}": No relevant matches found.`);
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
const tableResults: VectorMatch[] = [];
|
|
386
|
-
for (const row of result.rows) {
|
|
387
|
-
const { hybrid_score, id, ...rest } = row as Record<string, unknown>;
|
|
388
|
-
delete rest.embedding;
|
|
389
|
-
delete rest.vector_score;
|
|
390
|
-
delete rest.keyword_score;
|
|
391
|
-
delete rest.exact_name_score;
|
|
392
|
-
|
|
393
|
-
const content = `[TYPE: ${table.replace(/s$/, '').toUpperCase()}]\n` +
|
|
394
|
-
Object.entries(rest)
|
|
395
|
-
.filter(([k, v]) => v !== null && typeof v !== 'object' && k !== 'id')
|
|
396
|
-
.map(([k, v]) => `${k}: ${v}`)
|
|
397
|
-
.join('\n');
|
|
398
|
-
|
|
399
|
-
tableResults.push({
|
|
400
|
-
id: `${table}-${id}`,
|
|
401
|
-
score: parseFloat(String(hybrid_score)),
|
|
402
|
-
content,
|
|
403
|
-
metadata: { ...rest, source_table: table },
|
|
404
|
-
});
|
|
405
|
-
}
|
|
406
|
-
return tableResults;
|
|
407
|
-
} catch (err) {
|
|
408
|
-
console.error(
|
|
409
|
-
`[MultiTablePostgresProvider] Error querying table "${table}":`,
|
|
410
|
-
(err as Error).message
|
|
411
|
-
);
|
|
412
|
-
return [];
|
|
413
|
-
}
|
|
414
|
-
});
|
|
415
|
-
|
|
416
|
-
const resultsArray = await Promise.all(queryPromises);
|
|
417
|
-
for (const tableResults of resultsArray) {
|
|
418
|
-
allResults.push(...tableResults);
|
|
419
|
-
}
|
|
420
|
-
|
|
421
|
-
// 1. Sort all results globally by semantic similarity score
|
|
422
|
-
allResults.sort((a, b) => b.score - a.score);
|
|
423
|
-
|
|
424
|
-
if (allResults.length === 0) return [];
|
|
425
|
-
|
|
426
|
-
// 2. Identify the "Active Table Context" from the absolute highest scoring match
|
|
427
|
-
const bestMatchTable = allResults[0].metadata?.source_table as string;
|
|
428
|
-
|
|
429
|
-
// 3. Filter out all results that don't belong to the active table to prevent cross-contamination
|
|
430
|
-
const finalSorted = allResults.filter(res => res.metadata?.source_table === bestMatchTable);
|
|
431
|
-
|
|
432
|
-
//console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
|
|
433
|
-
//console.log(`[MultiTablePostgresProvider] Final top match from "${bestMatchTable}" with score ${finalSorted[0].score.toFixed(4)}`);
|
|
434
|
-
//console.log(`[MultiTablePostgresProvider] Restricting recommendations strictly to table: "${bestMatchTable}"`);
|
|
435
|
-
|
|
436
|
-
return finalSorted.slice(0, Math.max(topK, 15));
|
|
437
|
-
}
|
|
438
|
-
|
|
439
|
-
async delete(_id: string | number, _namespace?: string): Promise<void> {
|
|
440
|
-
console.warn('[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.');
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
async deleteNamespace(_namespace: string): Promise<void> {
|
|
444
|
-
console.warn('[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.');
|
|
445
|
-
}
|
|
446
|
-
|
|
447
|
-
async ping(): Promise<boolean> {
|
|
448
|
-
try {
|
|
449
|
-
await this.pool.query('SELECT 1');
|
|
450
|
-
return true;
|
|
451
|
-
} catch {
|
|
452
|
-
return false;
|
|
453
|
-
}
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
async disconnect(): Promise<void> {
|
|
457
|
-
if (this.pool) {
|
|
458
|
-
await this.pool.end();
|
|
459
|
-
}
|
|
460
|
-
}
|
|
461
|
-
}
|