@retrivora-ai/rag-engine 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-7K4KXB6G.mjs → chunk-6Q7DNWTG.mjs} +25 -1
- package/dist/handlers/index.js +25 -1
- package/dist/handlers/index.mjs +1 -1
- package/dist/server.js +40 -19
- package/dist/server.mjs +16 -19
- package/package.json +1 -1
- package/src/core/Pipeline.ts +31 -1
- package/src/providers/vectordb/MultiTablePostgresProvider.ts +26 -26
|
@@ -1553,6 +1553,26 @@ var EmbeddingStrategyResolver = class {
|
|
|
1553
1553
|
};
|
|
1554
1554
|
|
|
1555
1555
|
// src/core/Pipeline.ts
|
|
1556
|
+
function extractEntityHints(question) {
|
|
1557
|
+
var _a;
|
|
1558
|
+
const hints = /* @__PURE__ */ new Set();
|
|
1559
|
+
for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
|
|
1560
|
+
const value = match[1].trim();
|
|
1561
|
+
if (value) hints.add(value);
|
|
1562
|
+
}
|
|
1563
|
+
const namedEntityPatterns = [
|
|
1564
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})$/i,
|
|
1565
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})(?=[?.!,]|$)/i
|
|
1566
|
+
];
|
|
1567
|
+
for (const pattern of namedEntityPatterns) {
|
|
1568
|
+
const match = question.match(pattern);
|
|
1569
|
+
const value = (_a = match == null ? void 0 : match[1]) == null ? void 0 : _a.trim();
|
|
1570
|
+
if (value) {
|
|
1571
|
+
hints.add(value);
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
return [...hints].map((hint) => hint.replace(/\s+/g, " ").trim()).filter(Boolean);
|
|
1575
|
+
}
|
|
1556
1576
|
var Pipeline = class {
|
|
1557
1577
|
constructor(config) {
|
|
1558
1578
|
this.initialised = false;
|
|
@@ -1640,7 +1660,11 @@ var Pipeline = class {
|
|
|
1640
1660
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
1641
1661
|
try {
|
|
1642
1662
|
const queryVector = await this.embeddingProvider.embed(question, { taskType: "query" });
|
|
1643
|
-
const
|
|
1663
|
+
const entityHints = extractEntityHints(question);
|
|
1664
|
+
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, {
|
|
1665
|
+
__queryText: question,
|
|
1666
|
+
__entityHints: entityHints
|
|
1667
|
+
});
|
|
1644
1668
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
1645
1669
|
const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
1646
1670
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
package/dist/handlers/index.js
CHANGED
|
@@ -2663,6 +2663,26 @@ var EmbeddingStrategyResolver = class {
|
|
|
2663
2663
|
};
|
|
2664
2664
|
|
|
2665
2665
|
// src/core/Pipeline.ts
|
|
2666
|
+
function extractEntityHints(question) {
|
|
2667
|
+
var _a;
|
|
2668
|
+
const hints = /* @__PURE__ */ new Set();
|
|
2669
|
+
for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
|
|
2670
|
+
const value = match[1].trim();
|
|
2671
|
+
if (value) hints.add(value);
|
|
2672
|
+
}
|
|
2673
|
+
const namedEntityPatterns = [
|
|
2674
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})$/i,
|
|
2675
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})(?=[?.!,]|$)/i
|
|
2676
|
+
];
|
|
2677
|
+
for (const pattern of namedEntityPatterns) {
|
|
2678
|
+
const match = question.match(pattern);
|
|
2679
|
+
const value = (_a = match == null ? void 0 : match[1]) == null ? void 0 : _a.trim();
|
|
2680
|
+
if (value) {
|
|
2681
|
+
hints.add(value);
|
|
2682
|
+
}
|
|
2683
|
+
}
|
|
2684
|
+
return [...hints].map((hint) => hint.replace(/\s+/g, " ").trim()).filter(Boolean);
|
|
2685
|
+
}
|
|
2666
2686
|
var Pipeline = class {
|
|
2667
2687
|
constructor(config) {
|
|
2668
2688
|
this.initialised = false;
|
|
@@ -2750,7 +2770,11 @@ var Pipeline = class {
|
|
|
2750
2770
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
2751
2771
|
try {
|
|
2752
2772
|
const queryVector = await this.embeddingProvider.embed(question, { taskType: "query" });
|
|
2753
|
-
const
|
|
2773
|
+
const entityHints = extractEntityHints(question);
|
|
2774
|
+
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, {
|
|
2775
|
+
__queryText: question,
|
|
2776
|
+
__entityHints: entityHints
|
|
2777
|
+
});
|
|
2754
2778
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
2755
2779
|
const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
2756
2780
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
package/dist/handlers/index.mjs
CHANGED
package/dist/server.js
CHANGED
|
@@ -2752,6 +2752,26 @@ var EmbeddingStrategyResolver = class {
|
|
|
2752
2752
|
};
|
|
2753
2753
|
|
|
2754
2754
|
// src/core/Pipeline.ts
|
|
2755
|
+
function extractEntityHints(question) {
|
|
2756
|
+
var _a;
|
|
2757
|
+
const hints = /* @__PURE__ */ new Set();
|
|
2758
|
+
for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
|
|
2759
|
+
const value = match[1].trim();
|
|
2760
|
+
if (value) hints.add(value);
|
|
2761
|
+
}
|
|
2762
|
+
const namedEntityPatterns = [
|
|
2763
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})$/i,
|
|
2764
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})(?=[?.!,]|$)/i
|
|
2765
|
+
];
|
|
2766
|
+
for (const pattern of namedEntityPatterns) {
|
|
2767
|
+
const match = question.match(pattern);
|
|
2768
|
+
const value = (_a = match == null ? void 0 : match[1]) == null ? void 0 : _a.trim();
|
|
2769
|
+
if (value) {
|
|
2770
|
+
hints.add(value);
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
return [...hints].map((hint) => hint.replace(/\s+/g, " ").trim()).filter(Boolean);
|
|
2774
|
+
}
|
|
2755
2775
|
var Pipeline = class {
|
|
2756
2776
|
constructor(config) {
|
|
2757
2777
|
this.initialised = false;
|
|
@@ -2839,7 +2859,11 @@ var Pipeline = class {
|
|
|
2839
2859
|
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
2840
2860
|
try {
|
|
2841
2861
|
const queryVector = await this.embeddingProvider.embed(question, { taskType: "query" });
|
|
2842
|
-
const
|
|
2862
|
+
const entityHints = extractEntityHints(question);
|
|
2863
|
+
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, {
|
|
2864
|
+
__queryText: question,
|
|
2865
|
+
__entityHints: entityHints
|
|
2866
|
+
});
|
|
2843
2867
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
2844
2868
|
const context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
2845
2869
|
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
@@ -3692,6 +3716,7 @@ init_PostgreSQLProvider();
|
|
|
3692
3716
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
3693
3717
|
var import_pg2 = require("pg");
|
|
3694
3718
|
init_BaseVectorProvider();
|
|
3719
|
+
var EXACT_MATCH_FIELDS = ["name", "product_name", "productname", "title"];
|
|
3695
3720
|
var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
3696
3721
|
constructor(config) {
|
|
3697
3722
|
var _a, _b, _c;
|
|
@@ -3763,7 +3788,7 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
3763
3788
|
* Query all configured tables and merge results, sorted by cosine similarity score.
|
|
3764
3789
|
*/
|
|
3765
3790
|
async query(vector, topK, _namespace, _filter) {
|
|
3766
|
-
var _a, _b
|
|
3791
|
+
var _a, _b;
|
|
3767
3792
|
if (!this.pool) {
|
|
3768
3793
|
throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
|
|
3769
3794
|
}
|
|
@@ -3771,21 +3796,27 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
3771
3796
|
const allResults = [];
|
|
3772
3797
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
3773
3798
|
const queryText = _filter == null ? void 0 : _filter.__queryText;
|
|
3799
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter((hint) => typeof hint === "string").map((hint) => hint.trim().toLowerCase()).filter(Boolean) : [];
|
|
3774
3800
|
const queryPromises = this.tables.map(async (table) => {
|
|
3775
3801
|
try {
|
|
3776
3802
|
let sqlQuery = "";
|
|
3777
3803
|
let params = [];
|
|
3778
3804
|
if (queryText) {
|
|
3805
|
+
const hasEntityHints = entityHints.length > 0;
|
|
3806
|
+
const exactNameScoreExpr = hasEntityHints ? `GREATEST(${EXACT_MATCH_FIELDS.map(
|
|
3807
|
+
(field) => `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${field}', '')) = ANY($3::text[]) THEN 1 ELSE 0 END`
|
|
3808
|
+
).join(", ")})` : "0";
|
|
3779
3809
|
sqlQuery = `
|
|
3780
3810
|
SELECT *,
|
|
3781
3811
|
(1 - (embedding <=> $1::vector)) AS vector_score,
|
|
3782
|
-
ts_rank(to_tsvector('english', t.*::text), plainto_tsquery('english', $2)) AS keyword_score,
|
|
3783
|
-
|
|
3812
|
+
COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
|
|
3813
|
+
${exactNameScoreExpr} AS exact_name_score,
|
|
3814
|
+
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) + (${exactNameScoreExpr} * 5.0)) AS hybrid_score
|
|
3784
3815
|
FROM "${table}" t
|
|
3785
3816
|
ORDER BY hybrid_score DESC
|
|
3786
3817
|
LIMIT 50
|
|
3787
3818
|
`;
|
|
3788
|
-
params = [vectorLiteral, queryText];
|
|
3819
|
+
params = hasEntityHints ? [vectorLiteral, queryText, entityHints] : [vectorLiteral, queryText];
|
|
3789
3820
|
} else {
|
|
3790
3821
|
sqlQuery = `
|
|
3791
3822
|
SELECT *,
|
|
@@ -3808,6 +3839,7 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
3808
3839
|
delete rest.embedding;
|
|
3809
3840
|
delete rest.vector_score;
|
|
3810
3841
|
delete rest.keyword_score;
|
|
3842
|
+
delete rest.exact_name_score;
|
|
3811
3843
|
const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
|
|
3812
3844
|
` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
|
|
3813
3845
|
tableResults.push({
|
|
@@ -3830,23 +3862,12 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
3830
3862
|
for (const tableResults of resultsArray) {
|
|
3831
3863
|
allResults.push(...tableResults);
|
|
3832
3864
|
}
|
|
3833
|
-
const
|
|
3834
|
-
for (const res of allResults) {
|
|
3835
|
-
const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
|
|
3836
|
-
if (!resultsByTable[table]) resultsByTable[table] = [];
|
|
3837
|
-
resultsByTable[table].push(res);
|
|
3838
|
-
}
|
|
3839
|
-
const balancedResults = [];
|
|
3840
|
-
const tables = Object.keys(resultsByTable);
|
|
3841
|
-
for (const table of tables) {
|
|
3842
|
-
balancedResults.push(...resultsByTable[table].slice(0, 3));
|
|
3843
|
-
}
|
|
3844
|
-
const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
|
|
3865
|
+
const finalSorted = allResults.sort((a, b) => b.score - a.score);
|
|
3845
3866
|
if (finalSorted.length > 0) {
|
|
3846
3867
|
console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
|
|
3847
|
-
console.log(`[MultiTablePostgresProvider] Final top match from "${(
|
|
3868
|
+
console.log(`[MultiTablePostgresProvider] Final top match from "${(_b = (_a = finalSorted[0].metadata) == null ? void 0 : _a.source_table) != null ? _b : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
|
|
3848
3869
|
}
|
|
3849
|
-
return finalSorted.slice(0,
|
|
3870
|
+
return finalSorted.slice(0, topK);
|
|
3850
3871
|
}
|
|
3851
3872
|
async delete(_id, _namespace) {
|
|
3852
3873
|
console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
|
package/dist/server.mjs
CHANGED
|
@@ -34,7 +34,7 @@ import {
|
|
|
34
34
|
createIngestHandler,
|
|
35
35
|
createUploadHandler,
|
|
36
36
|
getRagConfig
|
|
37
|
-
} from "./chunk-
|
|
37
|
+
} from "./chunk-6Q7DNWTG.mjs";
|
|
38
38
|
import "./chunk-EDLTMSNY.mjs";
|
|
39
39
|
import {
|
|
40
40
|
PineconeProvider
|
|
@@ -335,6 +335,7 @@ function createFromPreset(presetName) {
|
|
|
335
335
|
|
|
336
336
|
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
337
337
|
import { Pool } from "pg";
|
|
338
|
+
var EXACT_MATCH_FIELDS = ["name", "product_name", "productname", "title"];
|
|
338
339
|
var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
339
340
|
constructor(config) {
|
|
340
341
|
var _a, _b, _c;
|
|
@@ -406,7 +407,7 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
406
407
|
* Query all configured tables and merge results, sorted by cosine similarity score.
|
|
407
408
|
*/
|
|
408
409
|
async query(vector, topK, _namespace, _filter) {
|
|
409
|
-
var _a, _b
|
|
410
|
+
var _a, _b;
|
|
410
411
|
if (!this.pool) {
|
|
411
412
|
throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
|
|
412
413
|
}
|
|
@@ -414,21 +415,27 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
414
415
|
const allResults = [];
|
|
415
416
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
416
417
|
const queryText = _filter == null ? void 0 : _filter.__queryText;
|
|
418
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter((hint) => typeof hint === "string").map((hint) => hint.trim().toLowerCase()).filter(Boolean) : [];
|
|
417
419
|
const queryPromises = this.tables.map(async (table) => {
|
|
418
420
|
try {
|
|
419
421
|
let sqlQuery = "";
|
|
420
422
|
let params = [];
|
|
421
423
|
if (queryText) {
|
|
424
|
+
const hasEntityHints = entityHints.length > 0;
|
|
425
|
+
const exactNameScoreExpr = hasEntityHints ? `GREATEST(${EXACT_MATCH_FIELDS.map(
|
|
426
|
+
(field) => `CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${field}', '')) = ANY($3::text[]) THEN 1 ELSE 0 END`
|
|
427
|
+
).join(", ")})` : "0";
|
|
422
428
|
sqlQuery = `
|
|
423
429
|
SELECT *,
|
|
424
430
|
(1 - (embedding <=> $1::vector)) AS vector_score,
|
|
425
|
-
ts_rank(to_tsvector('english', t.*::text), plainto_tsquery('english', $2)) AS keyword_score,
|
|
426
|
-
|
|
431
|
+
COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
|
|
432
|
+
${exactNameScoreExpr} AS exact_name_score,
|
|
433
|
+
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) + (${exactNameScoreExpr} * 5.0)) AS hybrid_score
|
|
427
434
|
FROM "${table}" t
|
|
428
435
|
ORDER BY hybrid_score DESC
|
|
429
436
|
LIMIT 50
|
|
430
437
|
`;
|
|
431
|
-
params = [vectorLiteral, queryText];
|
|
438
|
+
params = hasEntityHints ? [vectorLiteral, queryText, entityHints] : [vectorLiteral, queryText];
|
|
432
439
|
} else {
|
|
433
440
|
sqlQuery = `
|
|
434
441
|
SELECT *,
|
|
@@ -451,6 +458,7 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
451
458
|
delete rest.embedding;
|
|
452
459
|
delete rest.vector_score;
|
|
453
460
|
delete rest.keyword_score;
|
|
461
|
+
delete rest.exact_name_score;
|
|
454
462
|
const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
|
|
455
463
|
` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
|
|
456
464
|
tableResults.push({
|
|
@@ -473,23 +481,12 @@ var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
|
473
481
|
for (const tableResults of resultsArray) {
|
|
474
482
|
allResults.push(...tableResults);
|
|
475
483
|
}
|
|
476
|
-
const
|
|
477
|
-
for (const res of allResults) {
|
|
478
|
-
const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
|
|
479
|
-
if (!resultsByTable[table]) resultsByTable[table] = [];
|
|
480
|
-
resultsByTable[table].push(res);
|
|
481
|
-
}
|
|
482
|
-
const balancedResults = [];
|
|
483
|
-
const tables = Object.keys(resultsByTable);
|
|
484
|
-
for (const table of tables) {
|
|
485
|
-
balancedResults.push(...resultsByTable[table].slice(0, 3));
|
|
486
|
-
}
|
|
487
|
-
const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
|
|
484
|
+
const finalSorted = allResults.sort((a, b) => b.score - a.score);
|
|
488
485
|
if (finalSorted.length > 0) {
|
|
489
486
|
console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
|
|
490
|
-
console.log(`[MultiTablePostgresProvider] Final top match from "${(
|
|
487
|
+
console.log(`[MultiTablePostgresProvider] Final top match from "${(_b = (_a = finalSorted[0].metadata) == null ? void 0 : _a.source_table) != null ? _b : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
|
|
491
488
|
}
|
|
492
|
-
return finalSorted.slice(0,
|
|
489
|
+
return finalSorted.slice(0, topK);
|
|
493
490
|
}
|
|
494
491
|
async delete(_id, _namespace) {
|
|
495
492
|
console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@retrivora-ai/rag-engine",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.7",
|
|
4
4
|
"description": "Retrivora AI is a plug-and-play AI engine for RAG chat experiences — generic vector DB + LLM provider, embeddable or standalone.",
|
|
5
5
|
"author": "Abhinav Alkuchi",
|
|
6
6
|
"license": "MIT",
|
package/src/core/Pipeline.ts
CHANGED
|
@@ -7,6 +7,32 @@ import { BatchProcessor, BatchOptions } from './BatchProcessor';
|
|
|
7
7
|
import { EmbeddingStrategyResolver } from '../config/EmbeddingStrategy';
|
|
8
8
|
import { IngestDocument, ChatResponse } from '../types';
|
|
9
9
|
|
|
10
|
+
function extractEntityHints(question: string): string[] {
|
|
11
|
+
const hints = new Set<string>();
|
|
12
|
+
|
|
13
|
+
for (const match of question.matchAll(/["']([^"']{2,100})["']/g)) {
|
|
14
|
+
const value = match[1].trim();
|
|
15
|
+
if (value) hints.add(value);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const namedEntityPatterns = [
|
|
19
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})$/i,
|
|
20
|
+
/\b(?:product\s+name|named|called)\s+([a-z0-9][\w\s-]{0,80})(?=[?.!,]|$)/i,
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
for (const pattern of namedEntityPatterns) {
|
|
24
|
+
const match = question.match(pattern);
|
|
25
|
+
const value = match?.[1]?.trim();
|
|
26
|
+
if (value) {
|
|
27
|
+
hints.add(value);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
return [...hints]
|
|
32
|
+
.map((hint) => hint.replace(/\s+/g, ' ').trim())
|
|
33
|
+
.filter(Boolean);
|
|
34
|
+
}
|
|
35
|
+
|
|
10
36
|
/**
|
|
11
37
|
* Pipeline — orchestrates the RAG flow: Embed → Search → Augment → Generate.
|
|
12
38
|
*
|
|
@@ -135,8 +161,12 @@ export class Pipeline {
|
|
|
135
161
|
|
|
136
162
|
try {
|
|
137
163
|
const queryVector = await this.embeddingProvider.embed(question, { taskType: 'query' });
|
|
164
|
+
const entityHints = extractEntityHints(question);
|
|
138
165
|
// Pass the original text to the vector DB so it can perform hybrid (keyword) search if supported
|
|
139
|
-
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, {
|
|
166
|
+
const rawMatches = await this.vectorDB.query(queryVector, topK, ns, {
|
|
167
|
+
__queryText: question,
|
|
168
|
+
__entityHints: entityHints,
|
|
169
|
+
});
|
|
140
170
|
|
|
141
171
|
const sources = rawMatches.filter((m) => m.score >= scoreThreshold);
|
|
142
172
|
const context = sources.length
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
1
|
import { Pool, PoolClient } from 'pg';
|
|
5
2
|
import { BaseVectorProvider } from './BaseVectorProvider';
|
|
6
3
|
import { VectorMatch, UpsertDocument } from '../../types';
|
|
7
4
|
import { VectorDBConfig } from '../../config/RagConfig';
|
|
8
5
|
|
|
6
|
+
const EXACT_MATCH_FIELDS = ['name', 'product_name', 'productname', 'title'] as const;
|
|
7
|
+
|
|
9
8
|
/**
|
|
10
9
|
* MultiTablePostgresProvider — PostgreSQL implementation that searches across
|
|
11
10
|
* multiple existing tables with pre-existing embeddings.
|
|
@@ -127,6 +126,12 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
|
|
|
127
126
|
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
128
127
|
|
|
129
128
|
const queryText = _filter?.__queryText as string | undefined;
|
|
129
|
+
const entityHints = Array.isArray(_filter?.__entityHints)
|
|
130
|
+
? (_filter.__entityHints as unknown[])
|
|
131
|
+
.filter((hint): hint is string => typeof hint === 'string')
|
|
132
|
+
.map((hint) => hint.trim().toLowerCase())
|
|
133
|
+
.filter(Boolean)
|
|
134
|
+
: [];
|
|
130
135
|
|
|
131
136
|
const queryPromises = this.tables.map(async (table) => {
|
|
132
137
|
try {
|
|
@@ -134,17 +139,29 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
|
|
|
134
139
|
let params: unknown[] = [];
|
|
135
140
|
|
|
136
141
|
if (queryText) {
|
|
142
|
+
const hasEntityHints = entityHints.length > 0;
|
|
143
|
+
const exactNameScoreExpr = hasEntityHints
|
|
144
|
+
? `GREATEST(${EXACT_MATCH_FIELDS.map((field) =>
|
|
145
|
+
`CASE WHEN LOWER(COALESCE(to_jsonb(t)->>'${field}', '')) = ANY($3::text[]) THEN 1 ELSE 0 END`
|
|
146
|
+
).join(', ')})`
|
|
147
|
+
: '0';
|
|
148
|
+
|
|
137
149
|
// Actual Hybrid Search: Semantic Vector Score + Full-Text Keyword Rank
|
|
150
|
+
// We convert plainto_tsquery's AND (&) operator to OR (|) so that natural language queries
|
|
151
|
+
// (which contain words like "price", "product" not found in the row text) still match entities like "Router".
|
|
138
152
|
sqlQuery = `
|
|
139
153
|
SELECT *,
|
|
140
154
|
(1 - (embedding <=> $1::vector)) AS vector_score,
|
|
141
|
-
ts_rank(to_tsvector('english', t.*::text), plainto_tsquery('english', $2)) AS keyword_score,
|
|
142
|
-
|
|
155
|
+
COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) AS keyword_score,
|
|
156
|
+
${exactNameScoreExpr} AS exact_name_score,
|
|
157
|
+
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', t.*::text), NULLIF(REPLACE(plainto_tsquery('english', $2)::text, '&', '|'), '')::tsquery), 0) * 2.0) + (${exactNameScoreExpr} * 5.0)) AS hybrid_score
|
|
143
158
|
FROM "${table}" t
|
|
144
159
|
ORDER BY hybrid_score DESC
|
|
145
160
|
LIMIT 50
|
|
146
161
|
`;
|
|
147
|
-
params =
|
|
162
|
+
params = hasEntityHints
|
|
163
|
+
? [vectorLiteral, queryText, entityHints]
|
|
164
|
+
: [vectorLiteral, queryText];
|
|
148
165
|
} else {
|
|
149
166
|
// Fallback to Vector-only Search
|
|
150
167
|
sqlQuery = `
|
|
@@ -171,6 +188,7 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
|
|
|
171
188
|
delete rest.embedding;
|
|
172
189
|
delete rest.vector_score;
|
|
173
190
|
delete rest.keyword_score;
|
|
191
|
+
delete rest.exact_name_score;
|
|
174
192
|
|
|
175
193
|
const content = `[TYPE: ${table.replace(/s$/, '').toUpperCase()}]\n` +
|
|
176
194
|
Object.entries(rest)
|
|
@@ -200,32 +218,14 @@ export class MultiTablePostgresProvider extends BaseVectorProvider {
|
|
|
200
218
|
allResults.push(...tableResults);
|
|
201
219
|
}
|
|
202
220
|
|
|
203
|
-
|
|
204
|
-
const resultsByTable: Record<string, VectorMatch[]> = {};
|
|
205
|
-
for (const res of allResults) {
|
|
206
|
-
const table = res.metadata?.source_table as string;
|
|
207
|
-
if (!resultsByTable[table]) resultsByTable[table] = [];
|
|
208
|
-
resultsByTable[table].push(res);
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
// 2. Take top 3 from EACH table guaranteed, then fill the rest with overall best
|
|
212
|
-
const balancedResults: VectorMatch[] = [];
|
|
213
|
-
const tables = Object.keys(resultsByTable);
|
|
214
|
-
|
|
215
|
-
// First, ensure every table's best match is included
|
|
216
|
-
for (const table of tables) {
|
|
217
|
-
balancedResults.push(...resultsByTable[table].slice(0, 3));
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// 3. Sort the balanced list by score
|
|
221
|
-
const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
|
|
221
|
+
const finalSorted = allResults.sort((a, b) => b.score - a.score);
|
|
222
222
|
|
|
223
223
|
if (finalSorted.length > 0) {
|
|
224
224
|
console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
|
|
225
225
|
console.log(`[MultiTablePostgresProvider] Final top match from "${finalSorted[0].metadata?.source_table ?? 'unknown'}" with score ${finalSorted[0].score.toFixed(4)}`);
|
|
226
226
|
}
|
|
227
227
|
|
|
228
|
-
return finalSorted.slice(0,
|
|
228
|
+
return finalSorted.slice(0, topK);
|
|
229
229
|
}
|
|
230
230
|
|
|
231
231
|
async delete(_id: string | number, _namespace?: string): Promise<void> { // eslint-disable-line @typescript-eslint/no-unused-vars
|