@retrivora-ai/rag-engine 1.7.6 → 1.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{DocumentChunker-BXOUMKoP.d.ts → DocumentChunker-DMZVv6hi.d.ts} +1 -1
- package/dist/{DocumentChunker-D1dg5iCi.d.mts → DocumentChunker-wKE98F_G.d.mts} +1 -1
- package/dist/{chunk-XGVKIPNY.mjs → chunk-UHGYLTTY.mjs} +890 -882
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +890 -882
- package/dist/handlers/index.mjs +1 -1
- package/dist/{index-BejNscWZ.d.mts → index-B1u4loP6.d.mts} +1 -1
- package/dist/{index-CbkMJvu5.d.ts → index-D9w9fLjh.d.ts} +1 -1
- package/dist/{index-Cti1u0y1.d.mts → index-DfsVx0a4.d.mts} +1 -0
- package/dist/{index-Cti1u0y1.d.ts → index-DfsVx0a4.d.ts} +1 -0
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +185 -5
- package/dist/index.mjs +318 -138
- package/dist/server.d.mts +5 -5
- package/dist/server.d.ts +5 -5
- package/dist/server.js +1241 -1235
- package/dist/server.mjs +1 -1
- package/package.json +1 -1
- package/src/components/MessageBubble.tsx +8 -0
- package/src/core/Pipeline.ts +15 -2
- package/src/hooks/useRagChat.ts +12 -2
- package/src/types/chat.ts +1 -0
package/dist/server.js
CHANGED
|
@@ -3865,65 +3865,410 @@ var QueryProcessor = class {
|
|
|
3865
3865
|
}
|
|
3866
3866
|
};
|
|
3867
3867
|
|
|
3868
|
-
// src/
|
|
3869
|
-
var
|
|
3870
|
-
|
|
3871
|
-
|
|
3872
|
-
|
|
3873
|
-
|
|
3874
|
-
|
|
3875
|
-
|
|
3876
|
-
|
|
3877
|
-
|
|
3878
|
-
|
|
3868
|
+
// src/utils/UITransformer.ts
|
|
3869
|
+
var UITransformer = class {
|
|
3870
|
+
/**
|
|
3871
|
+
* Main transformation method
|
|
3872
|
+
* Analyzes user query and retrieved data to determine the best visualization format
|
|
3873
|
+
*
|
|
3874
|
+
* @param userQuery - The original user question/query
|
|
3875
|
+
* @param retrievedData - Vector database retrieval results
|
|
3876
|
+
* @returns Structured JSON response for UI rendering
|
|
3877
|
+
*/
|
|
3878
|
+
static transform(userQuery, retrievedData) {
|
|
3879
|
+
if (!retrievedData || retrievedData.length === 0) {
|
|
3880
|
+
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
3881
|
+
}
|
|
3882
|
+
const analysis = this.analyzeData(userQuery, retrievedData);
|
|
3883
|
+
switch (analysis.suggestedType) {
|
|
3884
|
+
case "product_carousel":
|
|
3885
|
+
return this.transformToProductCarousel(retrievedData);
|
|
3886
|
+
case "pie_chart":
|
|
3887
|
+
return this.transformToPieChart(retrievedData);
|
|
3888
|
+
case "bar_chart":
|
|
3889
|
+
return this.transformToBarChart(retrievedData);
|
|
3890
|
+
case "line_chart":
|
|
3891
|
+
return this.transformToLineChart(retrievedData);
|
|
3892
|
+
case "table":
|
|
3893
|
+
return this.transformToTable(retrievedData);
|
|
3894
|
+
default:
|
|
3895
|
+
return this.transformToText(retrievedData);
|
|
3879
3896
|
}
|
|
3880
|
-
return value;
|
|
3881
3897
|
}
|
|
3882
|
-
|
|
3883
|
-
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3898
|
+
/**
|
|
3899
|
+
* Analyzes the data and query to suggest the best visualization type
|
|
3900
|
+
*/
|
|
3901
|
+
static analyzeData(query, data) {
|
|
3902
|
+
const queryLower = query.toLowerCase();
|
|
3903
|
+
const hasProducts = data.some(
|
|
3904
|
+
(item) => this.isProductData(item)
|
|
3905
|
+
);
|
|
3906
|
+
const hasTimeSeries = data.some(
|
|
3907
|
+
(item) => this.isTimeSeriesData(item)
|
|
3908
|
+
);
|
|
3909
|
+
const hasCategories = this.detectCategories(data).length > 0;
|
|
3910
|
+
const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
|
|
3911
|
+
const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
|
|
3912
|
+
const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
|
|
3913
|
+
if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
|
|
3914
|
+
return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
|
|
3888
3915
|
}
|
|
3889
|
-
|
|
3916
|
+
if (isTrendQuery && hasTimeSeries) {
|
|
3917
|
+
return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
|
|
3918
|
+
}
|
|
3919
|
+
if (isDistributionQuery && hasCategories) {
|
|
3920
|
+
return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
|
|
3921
|
+
}
|
|
3922
|
+
if (isComparisonQuery && hasCategories && data.length > 2) {
|
|
3923
|
+
return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
|
|
3924
|
+
}
|
|
3925
|
+
if (data.length > 5 && this.hasMultipleFields(data)) {
|
|
3926
|
+
return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
|
|
3927
|
+
}
|
|
3928
|
+
return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
|
|
3890
3929
|
}
|
|
3891
|
-
|
|
3892
|
-
|
|
3930
|
+
/**
|
|
3931
|
+
* Transform data to product carousel format
|
|
3932
|
+
*/
|
|
3933
|
+
static transformToProductCarousel(data) {
|
|
3934
|
+
const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
|
|
3935
|
+
return {
|
|
3936
|
+
type: "product_carousel",
|
|
3937
|
+
title: "Recommended Products",
|
|
3938
|
+
description: `Found ${products.length} relevant products`,
|
|
3939
|
+
data: products
|
|
3940
|
+
};
|
|
3893
3941
|
}
|
|
3894
|
-
|
|
3895
|
-
|
|
3942
|
+
/**
|
|
3943
|
+
* Transform data to pie chart format
|
|
3944
|
+
*/
|
|
3945
|
+
static transformToPieChart(data) {
|
|
3946
|
+
const categories = this.detectCategories(data);
|
|
3947
|
+
const categoryData = this.aggregateByCategory(data, categories);
|
|
3948
|
+
const pieData = Object.entries(categoryData).map(([label, count]) => ({
|
|
3949
|
+
label,
|
|
3950
|
+
value: count,
|
|
3951
|
+
inStock: this.checkStockStatus(label, data)
|
|
3952
|
+
}));
|
|
3953
|
+
return {
|
|
3954
|
+
type: "pie_chart",
|
|
3955
|
+
title: "Distribution by Category",
|
|
3956
|
+
description: `Showing breakdown across ${categories.length} categories`,
|
|
3957
|
+
data: pieData
|
|
3958
|
+
};
|
|
3896
3959
|
}
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
|
|
3900
|
-
|
|
3901
|
-
|
|
3902
|
-
|
|
3903
|
-
|
|
3904
|
-
|
|
3905
|
-
|
|
3906
|
-
|
|
3907
|
-
|
|
3960
|
+
/**
|
|
3961
|
+
* Transform data to bar chart format
|
|
3962
|
+
*/
|
|
3963
|
+
static transformToBarChart(data) {
|
|
3964
|
+
const categories = this.detectCategories(data);
|
|
3965
|
+
const categoryData = this.aggregateByCategory(data, categories);
|
|
3966
|
+
const barData = Object.entries(categoryData).map(([category, value]) => ({
|
|
3967
|
+
category,
|
|
3968
|
+
value,
|
|
3969
|
+
inStock: this.checkStockStatus(category, data)
|
|
3970
|
+
}));
|
|
3971
|
+
return {
|
|
3972
|
+
type: "bar_chart",
|
|
3973
|
+
title: "Comparison by Category",
|
|
3974
|
+
description: `Comparing ${categories.length} categories`,
|
|
3975
|
+
data: barData
|
|
3976
|
+
};
|
|
3977
|
+
}
|
|
3978
|
+
/**
|
|
3979
|
+
* Transform data to line chart format
|
|
3980
|
+
*/
|
|
3981
|
+
static transformToLineChart(data) {
|
|
3982
|
+
const timePoints = this.extractTimeSeriesData(data);
|
|
3983
|
+
const lineData = timePoints.map((point) => ({
|
|
3984
|
+
timestamp: point.timestamp,
|
|
3985
|
+
value: point.value,
|
|
3986
|
+
label: point.label
|
|
3987
|
+
}));
|
|
3988
|
+
return {
|
|
3989
|
+
type: "line_chart",
|
|
3990
|
+
title: "Trend Over Time",
|
|
3991
|
+
description: `Showing ${lineData.length} data points`,
|
|
3992
|
+
data: lineData
|
|
3993
|
+
};
|
|
3994
|
+
}
|
|
3995
|
+
/**
|
|
3996
|
+
* Transform data to table format
|
|
3997
|
+
*/
|
|
3998
|
+
static transformToTable(data) {
|
|
3999
|
+
const columns = this.extractTableColumns(data);
|
|
4000
|
+
const rows = data.map((item) => this.extractTableRow(item, columns));
|
|
4001
|
+
const tableData = {
|
|
4002
|
+
columns,
|
|
4003
|
+
rows
|
|
4004
|
+
};
|
|
4005
|
+
return {
|
|
4006
|
+
type: "table",
|
|
4007
|
+
title: "Detailed Results",
|
|
4008
|
+
description: `Showing ${data.length} results`,
|
|
4009
|
+
data: tableData
|
|
4010
|
+
};
|
|
4011
|
+
}
|
|
4012
|
+
/**
|
|
4013
|
+
* Transform data to text format (fallback)
|
|
4014
|
+
*/
|
|
4015
|
+
static transformToText(data) {
|
|
4016
|
+
const textContent = data.map((item) => item.content).join("\n\n");
|
|
4017
|
+
return this.createTextResponse(
|
|
4018
|
+
"Search Results",
|
|
4019
|
+
textContent,
|
|
4020
|
+
`Found ${data.length} relevant results`
|
|
3908
4021
|
);
|
|
3909
|
-
|
|
3910
|
-
|
|
4022
|
+
}
|
|
4023
|
+
/**
|
|
4024
|
+
* Helper: Create text response
|
|
4025
|
+
*/
|
|
4026
|
+
static createTextResponse(title, content, description) {
|
|
4027
|
+
return {
|
|
4028
|
+
type: "text",
|
|
4029
|
+
title,
|
|
4030
|
+
description,
|
|
4031
|
+
data: { content }
|
|
4032
|
+
};
|
|
4033
|
+
}
|
|
4034
|
+
/**
|
|
4035
|
+
* Helper: Check if data item is product-related
|
|
4036
|
+
*/
|
|
4037
|
+
static isProductData(item) {
|
|
4038
|
+
const content = (item.content || "").toLowerCase();
|
|
4039
|
+
const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
|
|
4040
|
+
return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
|
|
4041
|
+
(k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
|
|
4042
|
+
);
|
|
4043
|
+
}
|
|
4044
|
+
/**
|
|
4045
|
+
* Helper: Check if data contains time series
|
|
4046
|
+
*/
|
|
4047
|
+
static isTimeSeriesData(item) {
|
|
4048
|
+
const content = (item.content || "").toLowerCase();
|
|
4049
|
+
const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
|
|
4050
|
+
return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
|
|
4051
|
+
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
4052
|
+
);
|
|
4053
|
+
}
|
|
4054
|
+
/**
|
|
4055
|
+
* Helper: Extract product information from vector match
|
|
4056
|
+
*/
|
|
4057
|
+
static extractProductInfo(item) {
|
|
4058
|
+
const meta = item.metadata || {};
|
|
4059
|
+
if (meta.name || meta.product) {
|
|
4060
|
+
return {
|
|
4061
|
+
id: item.id,
|
|
4062
|
+
name: meta.name || meta.product,
|
|
4063
|
+
price: meta.price,
|
|
4064
|
+
image: meta.image || meta.imageUrl,
|
|
4065
|
+
brand: meta.brand,
|
|
4066
|
+
description: item.content,
|
|
4067
|
+
inStock: this.determineStockStatus(item)
|
|
4068
|
+
};
|
|
3911
4069
|
}
|
|
3912
|
-
|
|
4070
|
+
const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
|
|
4071
|
+
const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
|
|
4072
|
+
if (nameMatch) {
|
|
4073
|
+
return {
|
|
4074
|
+
id: item.id,
|
|
4075
|
+
name: nameMatch[1],
|
|
4076
|
+
price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
|
|
4077
|
+
description: item.content,
|
|
4078
|
+
inStock: this.determineStockStatus(item)
|
|
4079
|
+
};
|
|
4080
|
+
}
|
|
4081
|
+
return null;
|
|
3913
4082
|
}
|
|
3914
|
-
|
|
3915
|
-
|
|
3916
|
-
|
|
3917
|
-
|
|
3918
|
-
const
|
|
3919
|
-
|
|
3920
|
-
|
|
3921
|
-
|
|
3922
|
-
|
|
3923
|
-
|
|
3924
|
-
|
|
3925
|
-
|
|
3926
|
-
|
|
4083
|
+
/**
|
|
4084
|
+
* Helper: Detect categories in data
|
|
4085
|
+
*/
|
|
4086
|
+
static detectCategories(data) {
|
|
4087
|
+
const categories = /* @__PURE__ */ new Set();
|
|
4088
|
+
data.forEach((item) => {
|
|
4089
|
+
const meta = item.metadata || {};
|
|
4090
|
+
if (meta.category) {
|
|
4091
|
+
categories.add(String(meta.category));
|
|
4092
|
+
}
|
|
4093
|
+
if (meta.type) {
|
|
4094
|
+
categories.add(String(meta.type));
|
|
4095
|
+
}
|
|
4096
|
+
if (meta.tag) {
|
|
4097
|
+
const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
|
|
4098
|
+
tags.forEach((t) => categories.add(String(t)));
|
|
4099
|
+
}
|
|
4100
|
+
const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
|
|
4101
|
+
if (categoryMatch) {
|
|
4102
|
+
categories.add(categoryMatch[1].trim());
|
|
4103
|
+
}
|
|
4104
|
+
});
|
|
4105
|
+
return Array.from(categories);
|
|
4106
|
+
}
|
|
4107
|
+
/**
|
|
4108
|
+
* Helper: Aggregate data by category
|
|
4109
|
+
*/
|
|
4110
|
+
static aggregateByCategory(data, categories) {
|
|
4111
|
+
const result = {};
|
|
4112
|
+
categories.forEach((cat) => {
|
|
4113
|
+
result[cat] = 0;
|
|
4114
|
+
});
|
|
4115
|
+
data.forEach((item) => {
|
|
4116
|
+
const meta = item.metadata || {};
|
|
4117
|
+
const itemCategory = meta.category || meta.type || "Other";
|
|
4118
|
+
if (result.hasOwnProperty(itemCategory)) {
|
|
4119
|
+
result[itemCategory]++;
|
|
4120
|
+
} else {
|
|
4121
|
+
result["Other"] = (result["Other"] || 0) + 1;
|
|
4122
|
+
}
|
|
4123
|
+
});
|
|
4124
|
+
return result;
|
|
4125
|
+
}
|
|
4126
|
+
/**
|
|
4127
|
+
* Helper: Extract time series data
|
|
4128
|
+
*/
|
|
4129
|
+
static extractTimeSeriesData(data) {
|
|
4130
|
+
return data.map((item) => {
|
|
4131
|
+
const meta = item.metadata || {};
|
|
4132
|
+
return {
|
|
4133
|
+
timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
|
|
4134
|
+
value: meta.value || item.score || 0,
|
|
4135
|
+
label: meta.label || item.content.substring(0, 50)
|
|
4136
|
+
};
|
|
4137
|
+
});
|
|
4138
|
+
}
|
|
4139
|
+
/**
|
|
4140
|
+
* Helper: Extract table columns
|
|
4141
|
+
*/
|
|
4142
|
+
static extractTableColumns(data) {
|
|
4143
|
+
const columnSet = /* @__PURE__ */ new Set();
|
|
4144
|
+
columnSet.add("Content");
|
|
4145
|
+
data.forEach((item) => {
|
|
4146
|
+
Object.keys(item.metadata || {}).forEach((key) => {
|
|
4147
|
+
columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
|
|
4148
|
+
});
|
|
4149
|
+
});
|
|
4150
|
+
return Array.from(columnSet);
|
|
4151
|
+
}
|
|
4152
|
+
/**
|
|
4153
|
+
* Helper: Extract table row
|
|
4154
|
+
*/
|
|
4155
|
+
static extractTableRow(item, columns) {
|
|
4156
|
+
const meta = item.metadata || {};
|
|
4157
|
+
return columns.map((col) => {
|
|
4158
|
+
if (col === "Content") {
|
|
4159
|
+
return item.content.substring(0, 100);
|
|
4160
|
+
}
|
|
4161
|
+
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
4162
|
+
const value = meta[metaKey];
|
|
4163
|
+
return value !== void 0 ? String(value) : "";
|
|
4164
|
+
});
|
|
4165
|
+
}
|
|
4166
|
+
/**
|
|
4167
|
+
* Helper: Check stock status
|
|
4168
|
+
*/
|
|
4169
|
+
static checkStockStatus(category, data) {
|
|
4170
|
+
const item = data.find((d) => {
|
|
4171
|
+
const meta = d.metadata || {};
|
|
4172
|
+
return meta.category === category || meta.type === category;
|
|
4173
|
+
});
|
|
4174
|
+
return this.determineStockStatus(item || {});
|
|
4175
|
+
}
|
|
4176
|
+
/**
|
|
4177
|
+
* Helper: Determine if item is in stock
|
|
4178
|
+
*/
|
|
4179
|
+
static determineStockStatus(item) {
|
|
4180
|
+
const meta = item.metadata || {};
|
|
4181
|
+
if (meta.inStock !== void 0) {
|
|
4182
|
+
return Boolean(meta.inStock);
|
|
4183
|
+
}
|
|
4184
|
+
if (meta.stock !== void 0) {
|
|
4185
|
+
return Boolean(meta.stock);
|
|
4186
|
+
}
|
|
4187
|
+
if (meta.available !== void 0) {
|
|
4188
|
+
return Boolean(meta.available);
|
|
4189
|
+
}
|
|
4190
|
+
const content = (item.content || "").toLowerCase();
|
|
4191
|
+
if (content.includes("out of stock") || content.includes("unavailable")) {
|
|
4192
|
+
return false;
|
|
4193
|
+
}
|
|
4194
|
+
if (content.includes("in stock") || content.includes("available")) {
|
|
4195
|
+
return true;
|
|
4196
|
+
}
|
|
4197
|
+
return true;
|
|
4198
|
+
}
|
|
4199
|
+
/**
|
|
4200
|
+
* Helper: Check if data has multiple fields
|
|
4201
|
+
*/
|
|
4202
|
+
static hasMultipleFields(data) {
|
|
4203
|
+
const fieldCount = /* @__PURE__ */ new Set();
|
|
4204
|
+
data.forEach((item) => {
|
|
4205
|
+
Object.keys(item.metadata || {}).forEach((key) => {
|
|
4206
|
+
fieldCount.add(key);
|
|
4207
|
+
});
|
|
4208
|
+
});
|
|
4209
|
+
return fieldCount.size > 2;
|
|
4210
|
+
}
|
|
4211
|
+
};
|
|
4212
|
+
|
|
4213
|
+
// src/core/Pipeline.ts
|
|
4214
|
+
var LRUEmbeddingCache = class {
|
|
4215
|
+
constructor(maxSize = 500) {
|
|
4216
|
+
this.cache = /* @__PURE__ */ new Map();
|
|
4217
|
+
this.maxSize = maxSize;
|
|
4218
|
+
}
|
|
4219
|
+
get(key) {
|
|
4220
|
+
const value = this.cache.get(key);
|
|
4221
|
+
if (value !== void 0) {
|
|
4222
|
+
this.cache.delete(key);
|
|
4223
|
+
this.cache.set(key, value);
|
|
4224
|
+
}
|
|
4225
|
+
return value;
|
|
4226
|
+
}
|
|
4227
|
+
set(key, value) {
|
|
4228
|
+
if (this.cache.has(key)) {
|
|
4229
|
+
this.cache.delete(key);
|
|
4230
|
+
} else if (this.cache.size >= this.maxSize) {
|
|
4231
|
+
const oldestKey = this.cache.keys().next().value;
|
|
4232
|
+
if (oldestKey !== void 0) this.cache.delete(oldestKey);
|
|
4233
|
+
}
|
|
4234
|
+
this.cache.set(key, value);
|
|
4235
|
+
}
|
|
4236
|
+
clear() {
|
|
4237
|
+
this.cache.clear();
|
|
4238
|
+
}
|
|
4239
|
+
get size() {
|
|
4240
|
+
return this.cache.size;
|
|
4241
|
+
}
|
|
4242
|
+
};
|
|
4243
|
+
var Pipeline = class {
|
|
4244
|
+
constructor(config) {
|
|
4245
|
+
this.config = config;
|
|
4246
|
+
/** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
|
|
4247
|
+
this.embeddingCache = new LRUEmbeddingCache(500);
|
|
4248
|
+
this.initialised = false;
|
|
4249
|
+
var _a, _b, _c, _d, _e;
|
|
4250
|
+
this.chunker = new DocumentChunker(
|
|
4251
|
+
(_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
|
|
4252
|
+
(_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
|
|
4253
|
+
);
|
|
4254
|
+
if (((_e = config.rag) == null ? void 0 : _e.chunkingStrategy) === "llamaindex") {
|
|
4255
|
+
this.llamaIngestor = new LlamaIndexIngestor();
|
|
4256
|
+
}
|
|
4257
|
+
this.reranker = new Reranker();
|
|
4258
|
+
}
|
|
4259
|
+
async initialize() {
|
|
4260
|
+
var _a, _b;
|
|
4261
|
+
if (this.initialised) return;
|
|
4262
|
+
const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
|
|
4263
|
+
const chartInstruction = `
|
|
4264
|
+
|
|
4265
|
+
${CHART_MARKER}
|
|
4266
|
+
### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
|
|
4267
|
+
|
|
4268
|
+
You are responsible for returning a SINGLE structured UI block when visualization is required.
|
|
4269
|
+
|
|
4270
|
+
---
|
|
4271
|
+
|
|
3927
4272
|
## 1. INTENT CLASSIFICATION (MANDATORY)
|
|
3928
4273
|
|
|
3929
4274
|
Classify the query into ONE of the following intents:
|
|
@@ -4043,6 +4388,8 @@ Classify the query into ONE of the following intents:
|
|
|
4043
4388
|
- ALWAYS return EXACTLY ONE \`\`\`ui\`\`\` block
|
|
4044
4389
|
- JSON must be VALID
|
|
4045
4390
|
- No explanation inside block
|
|
4391
|
+
- NEVER use ASCII charts, Markdown tables, or text-based drawings for data.
|
|
4392
|
+
- If data is tabular or statistical, ALWAYS use the \`\`\`ui\`\`\` block.
|
|
4046
4393
|
|
|
4047
4394
|
---
|
|
4048
4395
|
|
|
@@ -4101,1311 +4448,970 @@ User: "distribution of products across categories"
|
|
|
4101
4448
|
"insights": ["Beauty dominates inventory"]
|
|
4102
4449
|
}
|
|
4103
4450
|
\`\`\`
|
|
4104
|
-
`;
|
|
4105
|
-
if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
|
|
4106
|
-
let cleanPrompt = this.config.llm.systemPrompt || "";
|
|
4107
|
-
cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
|
|
4108
|
-
cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
|
|
4109
|
-
this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
|
|
4110
|
-
}
|
|
4111
|
-
console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
|
|
4112
|
-
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
4113
|
-
const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
4114
|
-
this.config.llm,
|
|
4115
|
-
this.config.embedding
|
|
4116
|
-
);
|
|
4117
|
-
this.llmProvider = llmProvider;
|
|
4118
|
-
this.embeddingProvider = embeddingProvider;
|
|
4119
|
-
if (this.config.graphDb) {
|
|
4120
|
-
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
4121
|
-
await this.graphDB.initialize();
|
|
4122
|
-
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
4123
|
-
}
|
|
4124
|
-
await this.vectorDB.initialize();
|
|
4125
|
-
if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
|
|
4126
|
-
this.agent = new LangChainAgent(this, this.config);
|
|
4127
|
-
await this.agent.initialize(this.llmProvider);
|
|
4128
|
-
}
|
|
4129
|
-
this.initialised = true;
|
|
4130
|
-
}
|
|
4131
|
-
/**
|
|
4132
|
-
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
4133
|
-
* Handles retries for transient failures.
|
|
4134
|
-
*/
|
|
4135
|
-
async ingest(documents, namespace) {
|
|
4136
|
-
await this.initialize();
|
|
4137
|
-
const ns = namespace != null ? namespace : this.config.projectId;
|
|
4138
|
-
const results = [];
|
|
4139
|
-
for (const doc of documents) {
|
|
4140
|
-
try {
|
|
4141
|
-
const chunks = await this.prepareChunks(doc);
|
|
4142
|
-
const vectors = await this.processEmbeddings(chunks);
|
|
4143
|
-
const upsertDocs = chunks.map((chunk, i) => ({
|
|
4144
|
-
id: chunk.id,
|
|
4145
|
-
vector: vectors[i],
|
|
4146
|
-
content: chunk.content,
|
|
4147
|
-
metadata: chunk.metadata
|
|
4148
|
-
}));
|
|
4149
|
-
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
4150
|
-
results.push({
|
|
4151
|
-
docId: doc.docId,
|
|
4152
|
-
chunksIngested: totalProcessed
|
|
4153
|
-
});
|
|
4154
|
-
if (this.graphDB && this.entityExtractor) {
|
|
4155
|
-
await this.processGraphIngestion(doc.docId, chunks);
|
|
4156
|
-
}
|
|
4157
|
-
} catch (error) {
|
|
4158
|
-
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
4159
|
-
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
4160
|
-
}
|
|
4161
|
-
}
|
|
4162
|
-
return results;
|
|
4163
|
-
}
|
|
4164
|
-
/**
|
|
4165
|
-
* Step 1: Chunk the document content.
|
|
4166
|
-
*/
|
|
4167
|
-
async prepareChunks(doc) {
|
|
4168
|
-
var _a, _b;
|
|
4169
|
-
if (this.llamaIngestor) {
|
|
4170
|
-
return await this.llamaIngestor.chunk(doc.content, {
|
|
4171
|
-
docId: doc.docId,
|
|
4172
|
-
metadata: doc.metadata,
|
|
4173
|
-
chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
|
|
4174
|
-
chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
|
|
4175
|
-
});
|
|
4176
|
-
}
|
|
4177
|
-
return this.chunker.chunk(doc.content, {
|
|
4178
|
-
docId: doc.docId,
|
|
4179
|
-
metadata: doc.metadata
|
|
4180
|
-
});
|
|
4181
|
-
}
|
|
4182
|
-
/**
|
|
4183
|
-
* Step 2: Generate embeddings for chunks with retry logic.
|
|
4184
|
-
* Uses batchEmbed when available for efficiency; falls back to sequential embedding.
|
|
4185
|
-
*/
|
|
4186
|
-
async processEmbeddings(chunks) {
|
|
4187
|
-
const embedBatchOptions = {
|
|
4188
|
-
batchSize: 50,
|
|
4189
|
-
maxRetries: 3,
|
|
4190
|
-
initialDelayMs: 100
|
|
4191
|
-
};
|
|
4192
|
-
const vectors = await BatchProcessor.mapWithRetry(
|
|
4193
|
-
chunks.map((c) => c.content),
|
|
4194
|
-
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
4195
|
-
embedBatchOptions
|
|
4196
|
-
);
|
|
4197
|
-
if (vectors.length !== chunks.length) {
|
|
4198
|
-
throw new Error(
|
|
4199
|
-
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
|
|
4200
|
-
);
|
|
4201
|
-
}
|
|
4202
|
-
return vectors;
|
|
4203
|
-
}
|
|
4204
|
-
/**
|
|
4205
|
-
* Step 3: Upsert chunks to vector database with retry logic.
|
|
4206
|
-
*/
|
|
4207
|
-
async processUpserts(upsertDocs, namespace) {
|
|
4208
|
-
const upsertBatchOptions = {
|
|
4209
|
-
batchSize: 100,
|
|
4210
|
-
maxRetries: 3,
|
|
4211
|
-
initialDelayMs: 100
|
|
4212
|
-
};
|
|
4213
|
-
const upsertResult = await BatchProcessor.processBatch(
|
|
4214
|
-
upsertDocs,
|
|
4215
|
-
(batch) => this.vectorDB.batchUpsert(batch, namespace),
|
|
4216
|
-
upsertBatchOptions
|
|
4217
|
-
);
|
|
4218
|
-
if (upsertResult.errors.length > 0) {
|
|
4219
|
-
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
|
|
4220
|
-
}
|
|
4221
|
-
return upsertResult.totalProcessed;
|
|
4222
|
-
}
|
|
4223
|
-
/**
|
|
4224
|
-
* Step 4: Optional graph-based entity extraction and ingestion.
|
|
4225
|
-
*/
|
|
4226
|
-
async processGraphIngestion(docId, chunks) {
|
|
4227
|
-
console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
|
|
4228
|
-
const extractionOptions = {
|
|
4229
|
-
batchSize: 2,
|
|
4230
|
-
// Low concurrency for LLM extraction
|
|
4231
|
-
maxRetries: 1,
|
|
4232
|
-
initialDelayMs: 500
|
|
4233
|
-
};
|
|
4234
|
-
await BatchProcessor.processBatch(
|
|
4235
|
-
chunks,
|
|
4236
|
-
async (batch) => {
|
|
4237
|
-
for (const chunk of batch) {
|
|
4238
|
-
try {
|
|
4239
|
-
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
4240
|
-
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
4241
|
-
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
4242
|
-
} catch (err) {
|
|
4243
|
-
console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
|
|
4244
|
-
}
|
|
4245
|
-
}
|
|
4246
|
-
},
|
|
4247
|
-
extractionOptions
|
|
4248
|
-
);
|
|
4249
|
-
}
|
|
4250
|
-
async ask(question, history = [], namespace) {
|
|
4251
|
-
var _a;
|
|
4252
|
-
await this.initialize();
|
|
4253
|
-
if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
|
|
4254
|
-
console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
|
|
4255
|
-
const agentReply = await this.agent.run(question, history);
|
|
4256
|
-
return { reply: agentReply, sources: [] };
|
|
4257
|
-
}
|
|
4258
|
-
const stream = this.askStream(question, history, namespace);
|
|
4259
|
-
let reply = "";
|
|
4260
|
-
let sources = [];
|
|
4261
|
-
let graphData;
|
|
4262
|
-
try {
|
|
4263
|
-
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
4264
|
-
const chunk = temp.value;
|
|
4265
|
-
if (typeof chunk === "string") {
|
|
4266
|
-
reply += chunk;
|
|
4267
|
-
} else if ("sources" in chunk) {
|
|
4268
|
-
sources = chunk.sources;
|
|
4269
|
-
graphData = chunk.graphData;
|
|
4270
|
-
}
|
|
4271
|
-
}
|
|
4272
|
-
} catch (temp) {
|
|
4273
|
-
error = [temp];
|
|
4274
|
-
} finally {
|
|
4275
|
-
try {
|
|
4276
|
-
more && (temp = iter.return) && await temp.call(iter);
|
|
4277
|
-
} finally {
|
|
4278
|
-
if (error)
|
|
4279
|
-
throw error[0];
|
|
4280
|
-
}
|
|
4281
|
-
}
|
|
4282
|
-
return { reply, sources, graphData };
|
|
4283
|
-
}
|
|
4284
|
-
/**
|
|
4285
|
-
* High-performance streaming RAG flow.
|
|
4286
|
-
* Yields text chunks first, then the retrieval metadata at the end.
|
|
4287
|
-
*/
|
|
4288
|
-
askStream(_0) {
|
|
4289
|
-
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
4290
|
-
var _a, _b, _c, _d, _e, _f, _g;
|
|
4291
|
-
yield new __await(this.initialize());
|
|
4292
|
-
const ns = namespace != null ? namespace : this.config.projectId;
|
|
4293
|
-
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
4294
|
-
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
4295
|
-
try {
|
|
4296
|
-
let searchQuery = question;
|
|
4297
|
-
if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
|
|
4298
|
-
searchQuery = yield new __await(this.rewriteQuery(question, history));
|
|
4299
|
-
}
|
|
4300
|
-
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
4301
|
-
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
4302
|
-
console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
|
|
4303
|
-
const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
|
|
4304
|
-
namespace: ns,
|
|
4305
|
-
topK: topK * 2,
|
|
4306
|
-
filter
|
|
4307
|
-
}));
|
|
4308
|
-
let sources = rawSources.filter((m) => m.score >= scoreThreshold);
|
|
4309
|
-
if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
|
|
4310
|
-
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
4311
|
-
} else {
|
|
4312
|
-
sources = sources.slice(0, topK);
|
|
4313
|
-
}
|
|
4314
|
-
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
4315
|
-
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
4316
|
-
if (graphData && graphData.nodes.length > 0) {
|
|
4317
|
-
const graphContext = graphData.nodes.map(
|
|
4318
|
-
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
4319
|
-
).join("\n");
|
|
4320
|
-
context = `GRAPH KNOWLEDGE:
|
|
4321
|
-
${graphContext}
|
|
4322
|
-
|
|
4323
|
-
VECTOR CONTEXT:
|
|
4324
|
-
${context}`;
|
|
4325
|
-
}
|
|
4326
|
-
const messages = [...history, { role: "user", content: question }];
|
|
4327
|
-
if (this.llmProvider.chatStream) {
|
|
4328
|
-
const stream = this.llmProvider.chatStream(messages, context);
|
|
4329
|
-
if (!stream) {
|
|
4330
|
-
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
4331
|
-
}
|
|
4332
|
-
try {
|
|
4333
|
-
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
4334
|
-
const chunk = temp.value;
|
|
4335
|
-
yield chunk;
|
|
4336
|
-
}
|
|
4337
|
-
} catch (temp) {
|
|
4338
|
-
error = [temp];
|
|
4339
|
-
} finally {
|
|
4340
|
-
try {
|
|
4341
|
-
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
4342
|
-
} finally {
|
|
4343
|
-
if (error)
|
|
4344
|
-
throw error[0];
|
|
4345
|
-
}
|
|
4346
|
-
}
|
|
4347
|
-
} else {
|
|
4348
|
-
const reply = yield new __await(this.llmProvider.chat(messages, context));
|
|
4349
|
-
yield reply;
|
|
4350
|
-
}
|
|
4351
|
-
yield { reply: "", sources, graphData };
|
|
4352
|
-
} catch (error2) {
|
|
4353
|
-
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
4354
|
-
}
|
|
4355
|
-
});
|
|
4356
|
-
}
|
|
4357
|
-
/**
|
|
4358
|
-
* Universal retrieval method combining all enabled providers.
|
|
4359
|
-
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
4360
|
-
*/
|
|
4361
|
-
async retrieve(query, options) {
|
|
4362
|
-
var _a, _b, _c;
|
|
4363
|
-
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
4364
|
-
const topK = (_b = options.topK) != null ? _b : 5;
|
|
4365
|
-
const cacheKey = `${ns}::${query}`;
|
|
4366
|
-
let queryVector = this.embeddingCache.get(cacheKey);
|
|
4367
|
-
const [retrievedVector, graphData] = await Promise.all([
|
|
4368
|
-
queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
|
|
4369
|
-
this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
|
|
4370
|
-
]);
|
|
4371
|
-
if (!queryVector) {
|
|
4372
|
-
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
4373
|
-
queryVector = retrievedVector;
|
|
4374
|
-
}
|
|
4375
|
-
const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
|
|
4376
|
-
return { sources, graphData };
|
|
4377
|
-
}
|
|
4378
|
-
/**
|
|
4379
|
-
* Rewrite the user query for better retrieval performance.
|
|
4380
|
-
*/
|
|
4381
|
-
async rewriteQuery(question, history) {
|
|
4382
|
-
const prompt = `
|
|
4383
|
-
Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
|
|
4384
|
-
Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
|
|
4385
|
-
|
|
4386
|
-
History:
|
|
4387
|
-
${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
|
|
4388
|
-
|
|
4389
|
-
New Question: ${question}
|
|
4390
|
-
|
|
4391
|
-
Optimized Search Query:`;
|
|
4392
|
-
const rewrite = await this.llmProvider.chat(
|
|
4393
|
-
[
|
|
4394
|
-
{ role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
|
|
4395
|
-
{ role: "user", content: prompt }
|
|
4396
|
-
],
|
|
4397
|
-
""
|
|
4398
|
-
);
|
|
4399
|
-
return rewrite.trim() || question;
|
|
4400
|
-
}
|
|
4401
|
-
/**
|
|
4402
|
-
* Generate 3-5 short, relevant questions based on the vector database content.
|
|
4403
|
-
*/
|
|
4404
|
-
async getSuggestions(query, namespace) {
|
|
4405
|
-
if (!query || query.trim().length < 3) {
|
|
4406
|
-
return [];
|
|
4407
|
-
}
|
|
4408
|
-
await this.initialize();
|
|
4409
|
-
const ns = namespace != null ? namespace : this.config.projectId;
|
|
4410
|
-
try {
|
|
4411
|
-
const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
|
|
4412
|
-
if (sources.length === 0) {
|
|
4413
|
-
return [];
|
|
4414
|
-
}
|
|
4415
|
-
const context = sources.map((s) => s.content).join("\n\n---\n\n");
|
|
4416
|
-
const prompt = `
|
|
4417
|
-
Based on the following snippets from a document, what are 3 short, relevant questions a user might ask?
|
|
4418
|
-
Focus on questions that can be answered by the context.
|
|
4419
|
-
Keep each question under 10 words and make them very specific to the content.
|
|
4420
|
-
Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
|
|
4421
|
-
|
|
4422
|
-
Context:
|
|
4423
|
-
${context}
|
|
4424
|
-
|
|
4425
|
-
Suggestions:`;
|
|
4426
|
-
const response = await this.llmProvider.chat(
|
|
4427
|
-
[
|
|
4428
|
-
{ role: "system", content: "You are a helpful assistant that generates search suggestions." },
|
|
4429
|
-
{ role: "user", content: prompt }
|
|
4430
|
-
],
|
|
4431
|
-
""
|
|
4432
|
-
);
|
|
4433
|
-
const match = response.match(/\[[\s\S]*\]/);
|
|
4434
|
-
if (match) {
|
|
4435
|
-
const suggestions = JSON.parse(match[0]);
|
|
4436
|
-
if (Array.isArray(suggestions)) {
|
|
4437
|
-
return suggestions.map((s) => String(s)).slice(0, 3);
|
|
4438
|
-
}
|
|
4439
|
-
}
|
|
4440
|
-
} catch (error) {
|
|
4441
|
-
console.error("[Pipeline] Failed to generate suggestions:", error);
|
|
4442
|
-
}
|
|
4443
|
-
return [];
|
|
4444
|
-
}
|
|
4445
|
-
};
|
|
4446
|
-
|
|
4447
|
-
// src/core/ProviderHealthCheck.ts
|
|
4448
|
-
var ProviderHealthCheck = class {
|
|
4449
|
-
/**
|
|
4450
|
-
* Validates vector database configuration before initialization.
|
|
4451
|
-
*/
|
|
4452
|
-
static async checkVectorProvider(config) {
|
|
4453
|
-
const timestamp = Date.now();
|
|
4454
|
-
const configErrors = await ConfigValidator.validate({
|
|
4455
|
-
projectId: "health-check",
|
|
4456
|
-
vectorDb: config,
|
|
4457
|
-
llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
|
|
4458
|
-
embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
|
|
4459
|
-
});
|
|
4460
|
-
const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
|
|
4461
|
-
if (vectorDbErrors.length > 0) {
|
|
4462
|
-
return {
|
|
4463
|
-
healthy: false,
|
|
4464
|
-
provider: config.provider,
|
|
4465
|
-
error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
|
|
4466
|
-
timestamp
|
|
4467
|
-
};
|
|
4468
|
-
}
|
|
4469
|
-
try {
|
|
4470
|
-
const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
|
|
4471
|
-
if (checker) {
|
|
4472
|
-
return await checker.check(config);
|
|
4473
|
-
}
|
|
4474
|
-
return await this.fallbackVectorHealthCheck(config, timestamp);
|
|
4475
|
-
} catch (error) {
|
|
4476
|
-
return {
|
|
4477
|
-
healthy: false,
|
|
4478
|
-
provider: config.provider,
|
|
4479
|
-
error: error instanceof Error ? error.message : String(error),
|
|
4480
|
-
timestamp
|
|
4481
|
-
};
|
|
4482
|
-
}
|
|
4483
|
-
}
|
|
4484
|
-
static async fallbackVectorHealthCheck(config, timestamp) {
|
|
4485
|
-
const opts = config.options || {};
|
|
4486
|
-
const baseUrl = opts.baseUrl || opts.uri;
|
|
4487
|
-
if (baseUrl && baseUrl.startsWith("http")) {
|
|
4488
|
-
try {
|
|
4489
|
-
const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
|
|
4490
|
-
return {
|
|
4491
|
-
healthy: response.ok,
|
|
4492
|
-
provider: config.provider,
|
|
4493
|
-
error: response.ok ? void 0 : `Health check failed: ${response.status}`,
|
|
4494
|
-
timestamp
|
|
4495
|
-
};
|
|
4496
|
-
} catch (e) {
|
|
4497
|
-
return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
|
|
4498
|
-
}
|
|
4499
|
-
}
|
|
4500
|
-
return {
|
|
4501
|
-
healthy: true,
|
|
4502
|
-
provider: config.provider,
|
|
4503
|
-
error: "Pluggable health check not implemented; basic reachability assumed.",
|
|
4504
|
-
timestamp
|
|
4505
|
-
};
|
|
4506
|
-
}
|
|
4507
|
-
/**
|
|
4508
|
-
* Validates LLM provider configuration.
|
|
4509
|
-
*/
|
|
4510
|
-
static async checkLLMProvider(config) {
|
|
4511
|
-
const timestamp = Date.now();
|
|
4512
|
-
try {
|
|
4513
|
-
const checker = LLMFactory.getHealthChecker(config.provider);
|
|
4514
|
-
if (checker) {
|
|
4515
|
-
return await checker.check(config);
|
|
4516
|
-
}
|
|
4517
|
-
return {
|
|
4518
|
-
healthy: true,
|
|
4519
|
-
provider: config.provider,
|
|
4520
|
-
error: "Pluggable health check not implemented; basic reachability assumed.",
|
|
4521
|
-
timestamp
|
|
4522
|
-
};
|
|
4523
|
-
} catch (error) {
|
|
4524
|
-
return {
|
|
4525
|
-
healthy: false,
|
|
4526
|
-
provider: config.provider,
|
|
4527
|
-
error: error instanceof Error ? error.message : String(error),
|
|
4528
|
-
timestamp
|
|
4529
|
-
};
|
|
4530
|
-
}
|
|
4531
|
-
}
|
|
4532
|
-
/**
|
|
4533
|
-
* Runs comprehensive health checks on all configured providers in parallel.
|
|
4534
|
-
*/
|
|
4535
|
-
static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
|
|
4536
|
-
const [vectorDb, llm, embedding] = await Promise.all([
|
|
4537
|
-
this.checkVectorProvider(vectorDbConfig),
|
|
4538
|
-
this.checkLLMProvider(llmConfig),
|
|
4539
|
-
embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
|
|
4540
|
-
]);
|
|
4541
|
-
return {
|
|
4542
|
-
vectorDb,
|
|
4543
|
-
llm,
|
|
4544
|
-
embedding,
|
|
4545
|
-
allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
|
|
4546
|
-
};
|
|
4547
|
-
}
|
|
4548
|
-
};
|
|
4549
|
-
|
|
4550
|
-
// src/core/VectorPlugin.ts
|
|
4551
|
-
var VectorPlugin = class {
|
|
4552
|
-
constructor(hostConfig) {
|
|
4553
|
-
this.config = ConfigResolver.resolve(hostConfig);
|
|
4554
|
-
this.validationPromise = ConfigValidator.validateAndThrow(this.config);
|
|
4555
|
-
this.pipeline = new Pipeline(this.config);
|
|
4556
|
-
}
|
|
4557
|
-
/**
|
|
4558
|
-
* Get the current resolved configuration.
|
|
4559
|
-
*/
|
|
4560
|
-
getConfig() {
|
|
4561
|
-
return this.config;
|
|
4562
|
-
}
|
|
4563
|
-
/**
|
|
4564
|
-
* Perform pre-flight health checks on all configured providers.
|
|
4565
|
-
* Useful to verify connectivity before running operations.
|
|
4566
|
-
*
|
|
4567
|
-
* @returns Health status for vector DB, LLM, and embedding providers
|
|
4568
|
-
*/
|
|
4569
|
-
async checkHealth() {
|
|
4570
|
-
return ProviderHealthCheck.checkAll(
|
|
4571
|
-
this.config.vectorDb,
|
|
4572
|
-
this.config.llm,
|
|
4573
|
-
this.config.embedding
|
|
4574
|
-
);
|
|
4575
|
-
}
|
|
4576
|
-
/**
|
|
4577
|
-
* Run a chat query.
|
|
4578
|
-
*/
|
|
4579
|
-
async chat(message, history = [], namespace) {
|
|
4580
|
-
await this.validationPromise;
|
|
4581
|
-
return this.pipeline.ask(message, history, namespace);
|
|
4582
|
-
}
|
|
4583
|
-
/**
|
|
4584
|
-
* Run a streaming chat query.
|
|
4585
|
-
*/
|
|
4586
|
-
chatStream(_0) {
|
|
4587
|
-
return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
|
|
4588
|
-
yield new __await(this.validationPromise);
|
|
4589
|
-
yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
|
|
4590
|
-
});
|
|
4451
|
+
`;
|
|
4452
|
+
if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
|
|
4453
|
+
let cleanPrompt = this.config.llm.systemPrompt || "";
|
|
4454
|
+
cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
|
|
4455
|
+
cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
|
|
4456
|
+
this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
|
|
4457
|
+
}
|
|
4458
|
+
console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
|
|
4459
|
+
this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
|
|
4460
|
+
const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
|
|
4461
|
+
this.config.llm,
|
|
4462
|
+
this.config.embedding
|
|
4463
|
+
);
|
|
4464
|
+
this.llmProvider = llmProvider;
|
|
4465
|
+
this.embeddingProvider = embeddingProvider;
|
|
4466
|
+
if (this.config.graphDb) {
|
|
4467
|
+
this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
|
|
4468
|
+
await this.graphDB.initialize();
|
|
4469
|
+
this.entityExtractor = new EntityExtractor(this.llmProvider);
|
|
4470
|
+
}
|
|
4471
|
+
await this.vectorDB.initialize();
|
|
4472
|
+
if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
|
|
4473
|
+
this.agent = new LangChainAgent(this, this.config);
|
|
4474
|
+
await this.agent.initialize(this.llmProvider);
|
|
4475
|
+
}
|
|
4476
|
+
this.initialised = true;
|
|
4591
4477
|
}
|
|
4592
4478
|
/**
|
|
4593
|
-
* Ingest documents
|
|
4479
|
+
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
4480
|
+
* Handles retries for transient failures.
|
|
4594
4481
|
*/
|
|
4595
4482
|
async ingest(documents, namespace) {
|
|
4596
|
-
await this.
|
|
4597
|
-
|
|
4598
|
-
|
|
4599
|
-
|
|
4600
|
-
|
|
4601
|
-
|
|
4602
|
-
|
|
4603
|
-
|
|
4604
|
-
|
|
4605
|
-
|
|
4606
|
-
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
|
|
4614
|
-
|
|
4615
|
-
|
|
4616
|
-
|
|
4617
|
-
|
|
4618
|
-
|
|
4619
|
-
|
|
4620
|
-
|
|
4621
|
-
var _a;
|
|
4622
|
-
if (provider === "auto") {
|
|
4623
|
-
this._vectorDb = this._autoDetectVectorDb();
|
|
4624
|
-
} else {
|
|
4625
|
-
this._vectorDb = {
|
|
4626
|
-
provider,
|
|
4627
|
-
indexName: (_a = options == null ? void 0 : options.indexName) != null ? _a : "default",
|
|
4628
|
-
options: __spreadValues({}, options)
|
|
4629
|
-
};
|
|
4483
|
+
await this.initialize();
|
|
4484
|
+
const ns = namespace != null ? namespace : this.config.projectId;
|
|
4485
|
+
const results = [];
|
|
4486
|
+
for (const doc of documents) {
|
|
4487
|
+
try {
|
|
4488
|
+
const chunks = await this.prepareChunks(doc);
|
|
4489
|
+
const vectors = await this.processEmbeddings(chunks);
|
|
4490
|
+
const upsertDocs = chunks.map((chunk, i) => ({
|
|
4491
|
+
id: chunk.id,
|
|
4492
|
+
vector: vectors[i],
|
|
4493
|
+
content: chunk.content,
|
|
4494
|
+
metadata: chunk.metadata
|
|
4495
|
+
}));
|
|
4496
|
+
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
4497
|
+
results.push({
|
|
4498
|
+
docId: doc.docId,
|
|
4499
|
+
chunksIngested: totalProcessed
|
|
4500
|
+
});
|
|
4501
|
+
if (this.graphDB && this.entityExtractor) {
|
|
4502
|
+
await this.processGraphIngestion(doc.docId, chunks);
|
|
4503
|
+
}
|
|
4504
|
+
} catch (error) {
|
|
4505
|
+
console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
|
|
4506
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
4507
|
+
}
|
|
4630
4508
|
}
|
|
4631
|
-
return
|
|
4509
|
+
return results;
|
|
4632
4510
|
}
|
|
4633
4511
|
/**
|
|
4634
|
-
*
|
|
4512
|
+
* Step 1: Chunk the document content.
|
|
4635
4513
|
*/
|
|
4636
|
-
|
|
4514
|
+
async prepareChunks(doc) {
|
|
4637
4515
|
var _a, _b;
|
|
4638
|
-
if (
|
|
4639
|
-
|
|
4640
|
-
|
|
4641
|
-
|
|
4642
|
-
|
|
4643
|
-
|
|
4644
|
-
|
|
4645
|
-
systemPrompt: options == null ? void 0 : options.systemPrompt,
|
|
4646
|
-
maxTokens: (_a = options == null ? void 0 : options.maxTokens) != null ? _a : 1024,
|
|
4647
|
-
temperature: (_b = options == null ? void 0 : options.temperature) != null ? _b : 0.7,
|
|
4648
|
-
baseUrl: options == null ? void 0 : options.baseUrl,
|
|
4649
|
-
options
|
|
4650
|
-
};
|
|
4651
|
-
}
|
|
4652
|
-
return this;
|
|
4653
|
-
}
|
|
4654
|
-
/**
|
|
4655
|
-
* Configure the embedding provider
|
|
4656
|
-
*/
|
|
4657
|
-
embedding(provider, model, apiKey, options) {
|
|
4658
|
-
if (provider === "auto") {
|
|
4659
|
-
this._embedding = this._autoDetectEmbedding();
|
|
4660
|
-
} else {
|
|
4661
|
-
this._embedding = {
|
|
4662
|
-
provider,
|
|
4663
|
-
model: model != null ? model : "default-embedding",
|
|
4664
|
-
apiKey,
|
|
4665
|
-
baseUrl: options == null ? void 0 : options.baseUrl,
|
|
4666
|
-
options
|
|
4667
|
-
};
|
|
4516
|
+
if (this.llamaIngestor) {
|
|
4517
|
+
return await this.llamaIngestor.chunk(doc.content, {
|
|
4518
|
+
docId: doc.docId,
|
|
4519
|
+
metadata: doc.metadata,
|
|
4520
|
+
chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
|
|
4521
|
+
chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
|
|
4522
|
+
});
|
|
4668
4523
|
}
|
|
4669
|
-
return this
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
*/
|
|
4674
|
-
rag(options) {
|
|
4675
|
-
var _a;
|
|
4676
|
-
this._rag = __spreadValues(__spreadValues({}, (_a = this._rag) != null ? _a : {}), options);
|
|
4677
|
-
return this;
|
|
4678
|
-
}
|
|
4679
|
-
/**
|
|
4680
|
-
* Set UI branding and appearance options.
|
|
4681
|
-
* Accepts the full UIConfig interface.
|
|
4682
|
-
*/
|
|
4683
|
-
ui(options) {
|
|
4684
|
-
this._ui = options;
|
|
4685
|
-
return this;
|
|
4524
|
+
return this.chunker.chunk(doc.content, {
|
|
4525
|
+
docId: doc.docId,
|
|
4526
|
+
metadata: doc.metadata
|
|
4527
|
+
});
|
|
4686
4528
|
}
|
|
4687
4529
|
/**
|
|
4688
|
-
*
|
|
4689
|
-
*
|
|
4530
|
+
* Step 2: Generate embeddings for chunks with retry logic.
|
|
4531
|
+
* Uses batchEmbed when available for efficiency; falls back to sequential embedding.
|
|
4690
4532
|
*/
|
|
4691
|
-
|
|
4692
|
-
|
|
4693
|
-
|
|
4694
|
-
|
|
4695
|
-
|
|
4696
|
-
|
|
4697
|
-
|
|
4698
|
-
|
|
4533
|
+
async processEmbeddings(chunks) {
|
|
4534
|
+
const embedBatchOptions = {
|
|
4535
|
+
batchSize: 50,
|
|
4536
|
+
maxRetries: 3,
|
|
4537
|
+
initialDelayMs: 100
|
|
4538
|
+
};
|
|
4539
|
+
const vectors = await BatchProcessor.mapWithRetry(
|
|
4540
|
+
chunks.map((c) => c.content),
|
|
4541
|
+
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
4542
|
+
embedBatchOptions
|
|
4543
|
+
);
|
|
4544
|
+
if (vectors.length !== chunks.length) {
|
|
4699
4545
|
throw new Error(
|
|
4700
|
-
`[
|
|
4701
|
-
${missing.join("\n ")}`
|
|
4546
|
+
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
|
|
4702
4547
|
);
|
|
4703
4548
|
}
|
|
4704
|
-
return
|
|
4705
|
-
projectId: this._projectId,
|
|
4706
|
-
vectorDb: this._vectorDb,
|
|
4707
|
-
llm: this._llm,
|
|
4708
|
-
embedding: this._embedding,
|
|
4709
|
-
ui: this._rag ? this._ui : void 0,
|
|
4710
|
-
rag: (_a = this._rag) != null ? _a : { chunkSize: 1e3, chunkOverlap: 200, topK: 5 }
|
|
4711
|
-
}, this._ui ? { ui: this._ui } : {});
|
|
4549
|
+
return vectors;
|
|
4712
4550
|
}
|
|
4713
4551
|
/**
|
|
4714
|
-
*
|
|
4715
|
-
*/
|
|
4716
|
-
|
|
4717
|
-
|
|
4718
|
-
|
|
4719
|
-
|
|
4720
|
-
|
|
4721
|
-
|
|
4722
|
-
|
|
4723
|
-
|
|
4724
|
-
|
|
4725
|
-
|
|
4726
|
-
|
|
4727
|
-
|
|
4728
|
-
|
|
4729
|
-
|
|
4730
|
-
|
|
4731
|
-
|
|
4732
|
-
if (process.env.MONGODB_URI) {
|
|
4733
|
-
return { provider: "mongodb", indexName: process.env.MONGODB_COLLECTION || "documents", options: { uri: process.env.MONGODB_URI, database: process.env.MONGODB_DB || "ai_db", collection: process.env.MONGODB_COLLECTION || "documents" } };
|
|
4734
|
-
}
|
|
4735
|
-
if (process.env.REDIS_URL) {
|
|
4736
|
-
return { provider: "redis", indexName: process.env.REDIS_INDEX || "documents", options: { url: process.env.REDIS_URL } };
|
|
4737
|
-
}
|
|
4738
|
-
return { provider: "qdrant", indexName: "documents", options: { url: process.env.QDRANT_URL || "http://localhost:6333" } };
|
|
4739
|
-
}
|
|
4740
|
-
_autoDetectLLM() {
|
|
4741
|
-
if (process.env.OPENAI_API_KEY) {
|
|
4742
|
-
return { provider: "openai", model: process.env.OPENAI_MODEL || "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY, maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OPENAI_TEMPERATURE || "0.7") };
|
|
4743
|
-
}
|
|
4744
|
-
if (process.env.ANTHROPIC_API_KEY) {
|
|
4745
|
-
return { provider: "anthropic", model: process.env.ANTHROPIC_MODEL || "claude-3-haiku-20240307", apiKey: process.env.ANTHROPIC_API_KEY, maxTokens: parseInt(process.env.ANTHROPIC_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.ANTHROPIC_TEMPERATURE || "0.7") };
|
|
4746
|
-
}
|
|
4747
|
-
if (process.env.OLLAMA_BASE_URL) {
|
|
4748
|
-
return { provider: "ollama", model: process.env.OLLAMA_MODEL || "mistral", baseUrl: process.env.OLLAMA_BASE_URL, maxTokens: parseInt(process.env.OLLAMA_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OLLAMA_TEMPERATURE || "0.7") };
|
|
4749
|
-
}
|
|
4750
|
-
return { provider: "openai", model: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY };
|
|
4751
|
-
}
|
|
4752
|
-
_autoDetectEmbedding() {
|
|
4753
|
-
if (process.env.EMBEDDING_PROVIDER) {
|
|
4754
|
-
return { provider: process.env.EMBEDDING_PROVIDER, model: process.env.EMBEDDING_MODEL || "default", apiKey: process.env.EMBEDDING_API_KEY, baseUrl: process.env.EMBEDDING_BASE_URL };
|
|
4755
|
-
}
|
|
4756
|
-
return { provider: "openai", model: "text-embedding-3-small", apiKey: process.env.OPENAI_API_KEY };
|
|
4757
|
-
}
|
|
4758
|
-
};
|
|
4759
|
-
var PRESETS = {
|
|
4760
|
-
/** OpenAI + Pinecone: Production-ready cloud setup */
|
|
4761
|
-
"openai-pinecone": { vectorDb: "pinecone", llm: "openai", embedding: "openai" },
|
|
4762
|
-
/** Claude + Qdrant: Open-source vector DB + proprietary LLM */
|
|
4763
|
-
"claude-qdrant": { vectorDb: "qdrant", llm: "anthropic", embedding: "openai" },
|
|
4764
|
-
/** Local development: Ollama + local Qdrant */
|
|
4765
|
-
"local-dev": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
|
|
4766
|
-
/** Fully open-source: Ollama LLM + Qdrant + Ollama embeddings */
|
|
4767
|
-
"fully-open-source": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
|
|
4768
|
-
/** PostgreSQL stack: pgvector + OpenAI */
|
|
4769
|
-
"postgres-openai": { vectorDb: "postgresql", llm: "openai", embedding: "openai" },
|
|
4770
|
-
/** Enterprise MongoDB: MongoDB Atlas with OpenAI */
|
|
4771
|
-
"mongodb-openai": { vectorDb: "mongodb", llm: "openai", embedding: "openai" },
|
|
4772
|
-
/** Redis stack for caching + search */
|
|
4773
|
-
"redis-openai": { vectorDb: "redis", llm: "openai", embedding: "openai" }
|
|
4774
|
-
};
|
|
4775
|
-
function createFromPreset(presetName) {
|
|
4776
|
-
const preset = PRESETS[presetName];
|
|
4777
|
-
return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
|
|
4778
|
-
}
|
|
4779
|
-
|
|
4780
|
-
// src/utils/DocumentParser.ts
|
|
4781
|
-
var DocumentParser = class {
|
|
4552
|
+
* Step 3: Upsert chunks to vector database with retry logic.
|
|
4553
|
+
*/
|
|
4554
|
+
async processUpserts(upsertDocs, namespace) {
|
|
4555
|
+
const upsertBatchOptions = {
|
|
4556
|
+
batchSize: 100,
|
|
4557
|
+
maxRetries: 3,
|
|
4558
|
+
initialDelayMs: 100
|
|
4559
|
+
};
|
|
4560
|
+
const upsertResult = await BatchProcessor.processBatch(
|
|
4561
|
+
upsertDocs,
|
|
4562
|
+
(batch) => this.vectorDB.batchUpsert(batch, namespace),
|
|
4563
|
+
upsertBatchOptions
|
|
4564
|
+
);
|
|
4565
|
+
if (upsertResult.errors.length > 0) {
|
|
4566
|
+
console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
|
|
4567
|
+
}
|
|
4568
|
+
return upsertResult.totalProcessed;
|
|
4569
|
+
}
|
|
4782
4570
|
/**
|
|
4783
|
-
*
|
|
4571
|
+
* Step 4: Optional graph-based entity extraction and ingestion.
|
|
4784
4572
|
*/
|
|
4785
|
-
|
|
4573
|
+
async processGraphIngestion(docId, chunks) {
|
|
4574
|
+
console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
|
|
4575
|
+
const extractionOptions = {
|
|
4576
|
+
batchSize: 2,
|
|
4577
|
+
// Low concurrency for LLM extraction
|
|
4578
|
+
maxRetries: 1,
|
|
4579
|
+
initialDelayMs: 500
|
|
4580
|
+
};
|
|
4581
|
+
await BatchProcessor.processBatch(
|
|
4582
|
+
chunks,
|
|
4583
|
+
async (batch) => {
|
|
4584
|
+
for (const chunk of batch) {
|
|
4585
|
+
try {
|
|
4586
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
4587
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
4588
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
4589
|
+
} catch (err) {
|
|
4590
|
+
console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
|
|
4591
|
+
}
|
|
4592
|
+
}
|
|
4593
|
+
},
|
|
4594
|
+
extractionOptions
|
|
4595
|
+
);
|
|
4596
|
+
}
|
|
4597
|
+
async ask(question, history = [], namespace) {
|
|
4786
4598
|
var _a;
|
|
4787
|
-
|
|
4788
|
-
if (
|
|
4789
|
-
|
|
4790
|
-
|
|
4791
|
-
|
|
4792
|
-
const text = await this.readAsText(file);
|
|
4793
|
-
try {
|
|
4794
|
-
const obj = JSON.parse(text);
|
|
4795
|
-
return JSON.stringify(obj, null, 2);
|
|
4796
|
-
} catch (e) {
|
|
4797
|
-
return text;
|
|
4798
|
-
}
|
|
4799
|
-
}
|
|
4800
|
-
if (extension === "csv" || mimeType === "text/csv") {
|
|
4801
|
-
return this.readAsText(file);
|
|
4599
|
+
await this.initialize();
|
|
4600
|
+
if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
|
|
4601
|
+
console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
|
|
4602
|
+
const agentReply = await this.agent.run(question, history);
|
|
4603
|
+
return { reply: agentReply, sources: [] };
|
|
4802
4604
|
}
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
4810
|
-
|
|
4811
|
-
|
|
4605
|
+
const stream = this.askStream(question, history, namespace);
|
|
4606
|
+
let reply = "";
|
|
4607
|
+
let sources = [];
|
|
4608
|
+
let graphData;
|
|
4609
|
+
try {
|
|
4610
|
+
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
|
|
4611
|
+
const chunk = temp.value;
|
|
4612
|
+
if (typeof chunk === "string") {
|
|
4613
|
+
reply += chunk;
|
|
4614
|
+
} else if ("sources" in chunk) {
|
|
4615
|
+
sources = chunk.sources;
|
|
4616
|
+
graphData = chunk.graphData;
|
|
4617
|
+
}
|
|
4812
4618
|
}
|
|
4813
|
-
}
|
|
4814
|
-
|
|
4619
|
+
} catch (temp) {
|
|
4620
|
+
error = [temp];
|
|
4621
|
+
} finally {
|
|
4815
4622
|
try {
|
|
4816
|
-
|
|
4817
|
-
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
} catch (err) {
|
|
4821
|
-
console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
|
|
4822
|
-
return `[DOCX Parsing Error for ${fileName}]`;
|
|
4623
|
+
more && (temp = iter.return) && await temp.call(iter);
|
|
4624
|
+
} finally {
|
|
4625
|
+
if (error)
|
|
4626
|
+
throw error[0];
|
|
4823
4627
|
}
|
|
4824
4628
|
}
|
|
4825
|
-
|
|
4826
|
-
return await this.readAsText(file);
|
|
4827
|
-
} catch (e) {
|
|
4828
|
-
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
4829
|
-
}
|
|
4830
|
-
}
|
|
4831
|
-
static async readAsText(file) {
|
|
4832
|
-
if (Buffer.isBuffer(file)) {
|
|
4833
|
-
return file.toString("utf-8");
|
|
4834
|
-
}
|
|
4835
|
-
return await file.text();
|
|
4629
|
+
return { reply, sources, graphData };
|
|
4836
4630
|
}
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
|
|
4841
|
-
|
|
4842
|
-
|
|
4631
|
+
/**
|
|
4632
|
+
* High-performance streaming RAG flow.
|
|
4633
|
+
* Yields text chunks first, then the retrieval metadata at the end.
|
|
4634
|
+
*/
|
|
4635
|
+
askStream(_0) {
|
|
4636
|
+
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
4637
|
+
var _a, _b, _c, _d, _e, _f, _g;
|
|
4638
|
+
yield new __await(this.initialize());
|
|
4639
|
+
const ns = namespace != null ? namespace : this.config.projectId;
|
|
4640
|
+
const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
|
|
4641
|
+
const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
|
|
4642
|
+
try {
|
|
4643
|
+
let searchQuery = question;
|
|
4644
|
+
if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
|
|
4645
|
+
searchQuery = yield new __await(this.rewriteQuery(question, history));
|
|
4646
|
+
}
|
|
4647
|
+
const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
|
|
4648
|
+
const filter = QueryProcessor.buildQueryFilter(question, hints);
|
|
4649
|
+
console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
|
|
4650
|
+
const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
|
|
4651
|
+
namespace: ns,
|
|
4652
|
+
topK: topK * 2,
|
|
4653
|
+
filter
|
|
4654
|
+
}));
|
|
4655
|
+
let sources = rawSources.filter((m) => m.score >= scoreThreshold);
|
|
4656
|
+
if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
|
|
4657
|
+
sources = yield new __await(this.reranker.rerank(sources, question, topK));
|
|
4658
|
+
} else {
|
|
4659
|
+
sources = sources.slice(0, topK);
|
|
4660
|
+
}
|
|
4661
|
+
let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
|
|
4662
|
+
${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
|
|
4663
|
+
if (graphData && graphData.nodes.length > 0) {
|
|
4664
|
+
const graphContext = graphData.nodes.map(
|
|
4665
|
+
(n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
|
|
4666
|
+
).join("\n");
|
|
4667
|
+
context = `GRAPH KNOWLEDGE:
|
|
4668
|
+
${graphContext}
|
|
4843
4669
|
|
|
4844
|
-
|
|
4845
|
-
|
|
4846
|
-
init_BaseVectorProvider();
|
|
4847
|
-
var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
4848
|
-
constructor(config) {
|
|
4849
|
-
var _a, _b, _c, _d, _e;
|
|
4850
|
-
super(config);
|
|
4851
|
-
const opts = config.options || {};
|
|
4852
|
-
if (!opts.connectionString) {
|
|
4853
|
-
throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
|
|
4854
|
-
}
|
|
4855
|
-
this.connectionString = opts.connectionString;
|
|
4856
|
-
this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
|
|
4857
|
-
const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
|
|
4858
|
-
this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
|
|
4859
|
-
if (this.tables.length === 0) {
|
|
4860
|
-
console.warn(
|
|
4861
|
-
"[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
|
|
4862
|
-
);
|
|
4863
|
-
}
|
|
4864
|
-
const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
|
|
4865
|
-
this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
|
|
4866
|
-
}
|
|
4867
|
-
async initialize() {
|
|
4868
|
-
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
4869
|
-
const client = await this.pool.connect();
|
|
4870
|
-
try {
|
|
4871
|
-
const res = await client.query(`
|
|
4872
|
-
SELECT table_name
|
|
4873
|
-
FROM information_schema.columns
|
|
4874
|
-
WHERE column_name = 'embedding'
|
|
4875
|
-
AND table_schema = 'public'
|
|
4876
|
-
`);
|
|
4877
|
-
const discoveredTables = res.rows.map((r) => r.table_name);
|
|
4878
|
-
if (this.tables.length === 0) {
|
|
4879
|
-
this.tables = discoveredTables;
|
|
4880
|
-
} else {
|
|
4881
|
-
const staticTables = [...this.tables];
|
|
4882
|
-
this.tables = staticTables.filter((t) => discoveredTables.includes(t));
|
|
4883
|
-
if (this.tables.length < staticTables.length) {
|
|
4884
|
-
const missing = staticTables.filter((t) => !discoveredTables.includes(t));
|
|
4885
|
-
console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
|
|
4670
|
+
VECTOR CONTEXT:
|
|
4671
|
+
${context}`;
|
|
4886
4672
|
}
|
|
4673
|
+
const messages = [...history, { role: "user", content: question }];
|
|
4674
|
+
if (this.llmProvider.chatStream) {
|
|
4675
|
+
const stream = this.llmProvider.chatStream(messages, context);
|
|
4676
|
+
if (!stream) {
|
|
4677
|
+
throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
|
|
4678
|
+
}
|
|
4679
|
+
try {
|
|
4680
|
+
for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
|
|
4681
|
+
const chunk = temp.value;
|
|
4682
|
+
yield chunk;
|
|
4683
|
+
}
|
|
4684
|
+
} catch (temp) {
|
|
4685
|
+
error = [temp];
|
|
4686
|
+
} finally {
|
|
4687
|
+
try {
|
|
4688
|
+
more && (temp = iter.return) && (yield new __await(temp.call(iter)));
|
|
4689
|
+
} finally {
|
|
4690
|
+
if (error)
|
|
4691
|
+
throw error[0];
|
|
4692
|
+
}
|
|
4693
|
+
}
|
|
4694
|
+
} else {
|
|
4695
|
+
const reply = yield new __await(this.llmProvider.chat(messages, context));
|
|
4696
|
+
yield reply;
|
|
4697
|
+
}
|
|
4698
|
+
const uiTransformation = UITransformer.transform(question, sources);
|
|
4699
|
+
yield {
|
|
4700
|
+
reply: "",
|
|
4701
|
+
sources,
|
|
4702
|
+
graphData,
|
|
4703
|
+
ui_transformation: uiTransformation
|
|
4704
|
+
};
|
|
4705
|
+
} catch (error2) {
|
|
4706
|
+
throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
|
|
4887
4707
|
}
|
|
4888
|
-
|
|
4889
|
-
|
|
4890
|
-
|
|
4891
|
-
|
|
4892
|
-
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4708
|
+
});
|
|
4709
|
+
}
|
|
4710
|
+
/**
|
|
4711
|
+
* Universal retrieval method combining all enabled providers.
|
|
4712
|
+
* Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
|
|
4713
|
+
*/
|
|
4714
|
+
async retrieve(query, options) {
|
|
4715
|
+
var _a, _b, _c;
|
|
4716
|
+
const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
|
|
4717
|
+
const topK = (_b = options.topK) != null ? _b : 5;
|
|
4718
|
+
const cacheKey = `${ns}::${query}`;
|
|
4719
|
+
let queryVector = this.embeddingCache.get(cacheKey);
|
|
4720
|
+
const [retrievedVector, graphData] = await Promise.all([
|
|
4721
|
+
queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
|
|
4722
|
+
this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
|
|
4723
|
+
]);
|
|
4724
|
+
if (!queryVector) {
|
|
4725
|
+
this.embeddingCache.set(cacheKey, retrievedVector);
|
|
4726
|
+
queryVector = retrievedVector;
|
|
4897
4727
|
}
|
|
4728
|
+
const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
|
|
4729
|
+
return { sources, graphData };
|
|
4898
4730
|
}
|
|
4899
4731
|
/**
|
|
4900
|
-
*
|
|
4901
|
-
* searching across pre-existing tables with varying schemas.
|
|
4902
|
-
*/
|
|
4903
|
-
async upsert(_doc, _namespace) {
|
|
4904
|
-
throw new Error(
|
|
4905
|
-
"[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
|
|
4906
|
-
);
|
|
4907
|
-
}
|
|
4908
|
-
/**
|
|
4909
|
-
* Batch upsert is not supported for MultiTablePostgresProvider.
|
|
4732
|
+
* Rewrite the user query for better retrieval performance.
|
|
4910
4733
|
*/
|
|
4911
|
-
async
|
|
4912
|
-
|
|
4913
|
-
|
|
4734
|
+
async rewriteQuery(question, history) {
|
|
4735
|
+
const prompt = `
|
|
4736
|
+
Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
|
|
4737
|
+
Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
|
|
4738
|
+
|
|
4739
|
+
History:
|
|
4740
|
+
${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
|
|
4741
|
+
|
|
4742
|
+
New Question: ${question}
|
|
4743
|
+
|
|
4744
|
+
Optimized Search Query:`;
|
|
4745
|
+
const rewrite = await this.llmProvider.chat(
|
|
4746
|
+
[
|
|
4747
|
+
{ role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
|
|
4748
|
+
{ role: "user", content: prompt }
|
|
4749
|
+
],
|
|
4750
|
+
""
|
|
4914
4751
|
);
|
|
4752
|
+
return rewrite.trim() || question;
|
|
4915
4753
|
}
|
|
4916
4754
|
/**
|
|
4917
|
-
*
|
|
4755
|
+
* Generate 3-5 short, relevant questions based on the vector database content.
|
|
4918
4756
|
*/
|
|
4919
|
-
async query
|
|
4920
|
-
|
|
4921
|
-
|
|
4922
|
-
throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
|
|
4757
|
+
async getSuggestions(query, namespace) {
|
|
4758
|
+
if (!query || query.trim().length < 3) {
|
|
4759
|
+
return [];
|
|
4923
4760
|
}
|
|
4924
|
-
|
|
4925
|
-
const
|
|
4926
|
-
|
|
4927
|
-
|
|
4928
|
-
|
|
4929
|
-
|
|
4930
|
-
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
4931
|
-
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
4932
|
-
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
4933
|
-
const getDynamicKeywordQuery = () => {
|
|
4934
|
-
if (entityHints.length > 0) {
|
|
4935
|
-
return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
|
|
4936
|
-
}
|
|
4937
|
-
if (queryText) {
|
|
4938
|
-
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
4761
|
+
await this.initialize();
|
|
4762
|
+
const ns = namespace != null ? namespace : this.config.projectId;
|
|
4763
|
+
try {
|
|
4764
|
+
const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
|
|
4765
|
+
if (sources.length === 0) {
|
|
4766
|
+
return [];
|
|
4939
4767
|
}
|
|
4940
|
-
|
|
4941
|
-
|
|
4942
|
-
|
|
4943
|
-
|
|
4944
|
-
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
|
|
4949
|
-
|
|
4950
|
-
|
|
4951
|
-
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4958
|
-
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
ORDER BY hybrid_score DESC
|
|
4964
|
-
LIMIT 50
|
|
4965
|
-
`;
|
|
4966
|
-
params = [vectorLiteral, dynamicKeywordQuery];
|
|
4967
|
-
} else {
|
|
4968
|
-
sqlQuery = `
|
|
4969
|
-
SELECT *,
|
|
4970
|
-
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
4971
|
-
FROM "${table}" t
|
|
4972
|
-
ORDER BY hybrid_score DESC
|
|
4973
|
-
LIMIT 50
|
|
4974
|
-
`;
|
|
4975
|
-
params = [vectorLiteral];
|
|
4976
|
-
}
|
|
4977
|
-
const result = await this.pool.query(sqlQuery, params);
|
|
4978
|
-
if (result.rowCount && result.rowCount > 0) {
|
|
4979
|
-
console.log(` \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
|
|
4980
|
-
} else {
|
|
4981
|
-
console.log(` \u26AA Table "${table}": No relevant matches found.`);
|
|
4982
|
-
}
|
|
4983
|
-
const tableResults = [];
|
|
4984
|
-
for (const row of result.rows) {
|
|
4985
|
-
const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
|
|
4986
|
-
delete rest.embedding;
|
|
4987
|
-
delete rest.vector_score;
|
|
4988
|
-
delete rest.keyword_score;
|
|
4989
|
-
delete rest.exact_name_score;
|
|
4990
|
-
const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
|
|
4991
|
-
` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
|
|
4992
|
-
tableResults.push({
|
|
4993
|
-
id: `${table}-${id}`,
|
|
4994
|
-
score: parseFloat(String(hybrid_score)),
|
|
4995
|
-
content,
|
|
4996
|
-
metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
|
|
4997
|
-
});
|
|
4768
|
+
const context = sources.map((s) => s.content).join("\n\n---\n\n");
|
|
4769
|
+
const prompt = `
|
|
4770
|
+
Based on the following snippets from a document, what are 3 short, relevant questions a user might ask?
|
|
4771
|
+
Focus on questions that can be answered by the context.
|
|
4772
|
+
Keep each question under 10 words and make them very specific to the content.
|
|
4773
|
+
Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
|
|
4774
|
+
|
|
4775
|
+
Context:
|
|
4776
|
+
${context}
|
|
4777
|
+
|
|
4778
|
+
Suggestions:`;
|
|
4779
|
+
const response = await this.llmProvider.chat(
|
|
4780
|
+
[
|
|
4781
|
+
{ role: "system", content: "You are a helpful assistant that generates search suggestions." },
|
|
4782
|
+
{ role: "user", content: prompt }
|
|
4783
|
+
],
|
|
4784
|
+
""
|
|
4785
|
+
);
|
|
4786
|
+
const match = response.match(/\[[\s\S]*\]/);
|
|
4787
|
+
if (match) {
|
|
4788
|
+
const suggestions = JSON.parse(match[0]);
|
|
4789
|
+
if (Array.isArray(suggestions)) {
|
|
4790
|
+
return suggestions.map((s) => String(s)).slice(0, 3);
|
|
4998
4791
|
}
|
|
4999
|
-
return tableResults;
|
|
5000
|
-
} catch (err) {
|
|
5001
|
-
console.error(
|
|
5002
|
-
`[MultiTablePostgresProvider] Error querying table "${table}":`,
|
|
5003
|
-
err.message
|
|
5004
|
-
);
|
|
5005
|
-
return [];
|
|
5006
4792
|
}
|
|
5007
|
-
})
|
|
5008
|
-
|
|
5009
|
-
for (const tableResults of resultsArray) {
|
|
5010
|
-
allResults.push(...tableResults);
|
|
5011
|
-
}
|
|
5012
|
-
const resultsByTable = {};
|
|
5013
|
-
for (const res of allResults) {
|
|
5014
|
-
const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
|
|
5015
|
-
if (!resultsByTable[table]) resultsByTable[table] = [];
|
|
5016
|
-
resultsByTable[table].push(res);
|
|
4793
|
+
} catch (error) {
|
|
4794
|
+
console.error("[Pipeline] Failed to generate suggestions:", error);
|
|
5017
4795
|
}
|
|
5018
|
-
|
|
5019
|
-
|
|
5020
|
-
|
|
5021
|
-
|
|
4796
|
+
return [];
|
|
4797
|
+
}
|
|
4798
|
+
};
|
|
4799
|
+
|
|
4800
|
+
// src/core/ProviderHealthCheck.ts
|
|
4801
|
+
var ProviderHealthCheck = class {
|
|
4802
|
+
/**
|
|
4803
|
+
* Validates vector database configuration before initialization.
|
|
4804
|
+
*/
|
|
4805
|
+
static async checkVectorProvider(config) {
|
|
4806
|
+
const timestamp = Date.now();
|
|
4807
|
+
const configErrors = await ConfigValidator.validate({
|
|
4808
|
+
projectId: "health-check",
|
|
4809
|
+
vectorDb: config,
|
|
4810
|
+
llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
|
|
4811
|
+
embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
|
|
4812
|
+
});
|
|
4813
|
+
const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
|
|
4814
|
+
if (vectorDbErrors.length > 0) {
|
|
4815
|
+
return {
|
|
4816
|
+
healthy: false,
|
|
4817
|
+
provider: config.provider,
|
|
4818
|
+
error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
|
|
4819
|
+
timestamp
|
|
4820
|
+
};
|
|
5022
4821
|
}
|
|
5023
|
-
|
|
5024
|
-
|
|
5025
|
-
|
|
5026
|
-
|
|
4822
|
+
try {
|
|
4823
|
+
const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
|
|
4824
|
+
if (checker) {
|
|
4825
|
+
return await checker.check(config);
|
|
4826
|
+
}
|
|
4827
|
+
return await this.fallbackVectorHealthCheck(config, timestamp);
|
|
4828
|
+
} catch (error) {
|
|
4829
|
+
return {
|
|
4830
|
+
healthy: false,
|
|
4831
|
+
provider: config.provider,
|
|
4832
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4833
|
+
timestamp
|
|
4834
|
+
};
|
|
5027
4835
|
}
|
|
5028
|
-
return finalSorted.slice(0, Math.max(topK, 15));
|
|
5029
|
-
}
|
|
5030
|
-
async delete(_id, _namespace) {
|
|
5031
|
-
console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
|
|
5032
4836
|
}
|
|
5033
|
-
async
|
|
5034
|
-
|
|
4837
|
+
static async fallbackVectorHealthCheck(config, timestamp) {
|
|
4838
|
+
const opts = config.options || {};
|
|
4839
|
+
const baseUrl = opts.baseUrl || opts.uri;
|
|
4840
|
+
if (baseUrl && baseUrl.startsWith("http")) {
|
|
4841
|
+
try {
|
|
4842
|
+
const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
|
|
4843
|
+
return {
|
|
4844
|
+
healthy: response.ok,
|
|
4845
|
+
provider: config.provider,
|
|
4846
|
+
error: response.ok ? void 0 : `Health check failed: ${response.status}`,
|
|
4847
|
+
timestamp
|
|
4848
|
+
};
|
|
4849
|
+
} catch (e) {
|
|
4850
|
+
return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
|
|
4851
|
+
}
|
|
4852
|
+
}
|
|
4853
|
+
return {
|
|
4854
|
+
healthy: true,
|
|
4855
|
+
provider: config.provider,
|
|
4856
|
+
error: "Pluggable health check not implemented; basic reachability assumed.",
|
|
4857
|
+
timestamp
|
|
4858
|
+
};
|
|
5035
4859
|
}
|
|
5036
|
-
|
|
4860
|
+
/**
|
|
4861
|
+
* Validates LLM provider configuration.
|
|
4862
|
+
*/
|
|
4863
|
+
static async checkLLMProvider(config) {
|
|
4864
|
+
const timestamp = Date.now();
|
|
5037
4865
|
try {
|
|
5038
|
-
|
|
5039
|
-
|
|
5040
|
-
|
|
5041
|
-
|
|
4866
|
+
const checker = LLMFactory.getHealthChecker(config.provider);
|
|
4867
|
+
if (checker) {
|
|
4868
|
+
return await checker.check(config);
|
|
4869
|
+
}
|
|
4870
|
+
return {
|
|
4871
|
+
healthy: true,
|
|
4872
|
+
provider: config.provider,
|
|
4873
|
+
error: "Pluggable health check not implemented; basic reachability assumed.",
|
|
4874
|
+
timestamp
|
|
4875
|
+
};
|
|
4876
|
+
} catch (error) {
|
|
4877
|
+
return {
|
|
4878
|
+
healthy: false,
|
|
4879
|
+
provider: config.provider,
|
|
4880
|
+
error: error instanceof Error ? error.message : String(error),
|
|
4881
|
+
timestamp
|
|
4882
|
+
};
|
|
5042
4883
|
}
|
|
5043
4884
|
}
|
|
5044
|
-
|
|
5045
|
-
|
|
5046
|
-
|
|
5047
|
-
|
|
4885
|
+
/**
|
|
4886
|
+
* Runs comprehensive health checks on all configured providers in parallel.
|
|
4887
|
+
*/
|
|
4888
|
+
static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
|
|
4889
|
+
const [vectorDb, llm, embedding] = await Promise.all([
|
|
4890
|
+
this.checkVectorProvider(vectorDbConfig),
|
|
4891
|
+
this.checkLLMProvider(llmConfig),
|
|
4892
|
+
embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
|
|
4893
|
+
]);
|
|
4894
|
+
return {
|
|
4895
|
+
vectorDb,
|
|
4896
|
+
llm,
|
|
4897
|
+
embedding,
|
|
4898
|
+
allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
|
|
4899
|
+
};
|
|
5048
4900
|
}
|
|
5049
4901
|
};
|
|
5050
4902
|
|
|
5051
|
-
// src/
|
|
5052
|
-
|
|
5053
|
-
|
|
5054
|
-
|
|
5055
|
-
|
|
5056
|
-
|
|
5057
|
-
|
|
5058
|
-
init_UniversalVectorProvider();
|
|
5059
|
-
|
|
5060
|
-
// src/handlers/index.ts
|
|
5061
|
-
var import_server = require("next/server");
|
|
5062
|
-
|
|
5063
|
-
// src/utils/UITransformer.ts
|
|
5064
|
-
var UITransformer = class {
|
|
4903
|
+
// src/core/VectorPlugin.ts
|
|
4904
|
+
var VectorPlugin = class {
|
|
4905
|
+
constructor(hostConfig) {
|
|
4906
|
+
this.config = ConfigResolver.resolve(hostConfig);
|
|
4907
|
+
this.validationPromise = ConfigValidator.validateAndThrow(this.config);
|
|
4908
|
+
this.pipeline = new Pipeline(this.config);
|
|
4909
|
+
}
|
|
5065
4910
|
/**
|
|
5066
|
-
*
|
|
5067
|
-
* Analyzes user query and retrieved data to determine the best visualization format
|
|
5068
|
-
*
|
|
5069
|
-
* @param userQuery - The original user question/query
|
|
5070
|
-
* @param retrievedData - Vector database retrieval results
|
|
5071
|
-
* @returns Structured JSON response for UI rendering
|
|
4911
|
+
* Get the current resolved configuration.
|
|
5072
4912
|
*/
|
|
5073
|
-
|
|
5074
|
-
|
|
5075
|
-
return this.createTextResponse("No data available", "No relevant data found for your query.");
|
|
5076
|
-
}
|
|
5077
|
-
const analysis = this.analyzeData(userQuery, retrievedData);
|
|
5078
|
-
switch (analysis.suggestedType) {
|
|
5079
|
-
case "product_carousel":
|
|
5080
|
-
return this.transformToProductCarousel(retrievedData);
|
|
5081
|
-
case "pie_chart":
|
|
5082
|
-
return this.transformToPieChart(retrievedData);
|
|
5083
|
-
case "bar_chart":
|
|
5084
|
-
return this.transformToBarChart(retrievedData);
|
|
5085
|
-
case "line_chart":
|
|
5086
|
-
return this.transformToLineChart(retrievedData);
|
|
5087
|
-
case "table":
|
|
5088
|
-
return this.transformToTable(retrievedData);
|
|
5089
|
-
default:
|
|
5090
|
-
return this.transformToText(retrievedData);
|
|
5091
|
-
}
|
|
4913
|
+
getConfig() {
|
|
4914
|
+
return this.config;
|
|
5092
4915
|
}
|
|
5093
4916
|
/**
|
|
5094
|
-
*
|
|
4917
|
+
* Perform pre-flight health checks on all configured providers.
|
|
4918
|
+
* Useful to verify connectivity before running operations.
|
|
4919
|
+
*
|
|
4920
|
+
* @returns Health status for vector DB, LLM, and embedding providers
|
|
5095
4921
|
*/
|
|
5096
|
-
|
|
5097
|
-
|
|
5098
|
-
|
|
5099
|
-
|
|
5100
|
-
|
|
5101
|
-
const hasTimeSeries = data.some(
|
|
5102
|
-
(item) => this.isTimeSeriesData(item)
|
|
4922
|
+
async checkHealth() {
|
|
4923
|
+
return ProviderHealthCheck.checkAll(
|
|
4924
|
+
this.config.vectorDb,
|
|
4925
|
+
this.config.llm,
|
|
4926
|
+
this.config.embedding
|
|
5103
4927
|
);
|
|
5104
|
-
const hasCategories = this.detectCategories(data).length > 0;
|
|
5105
|
-
const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
|
|
5106
|
-
const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
|
|
5107
|
-
const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
|
|
5108
|
-
if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
|
|
5109
|
-
return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
|
|
5110
|
-
}
|
|
5111
|
-
if (isTrendQuery && hasTimeSeries) {
|
|
5112
|
-
return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
|
|
5113
|
-
}
|
|
5114
|
-
if (isDistributionQuery && hasCategories) {
|
|
5115
|
-
return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
|
|
5116
|
-
}
|
|
5117
|
-
if (isComparisonQuery && hasCategories && data.length > 2) {
|
|
5118
|
-
return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
|
|
5119
|
-
}
|
|
5120
|
-
if (data.length > 5 && this.hasMultipleFields(data)) {
|
|
5121
|
-
return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
|
|
5122
|
-
}
|
|
5123
|
-
return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
|
|
5124
4928
|
}
|
|
5125
4929
|
/**
|
|
5126
|
-
*
|
|
4930
|
+
* Run a chat query.
|
|
5127
4931
|
*/
|
|
5128
|
-
|
|
5129
|
-
|
|
5130
|
-
return
|
|
5131
|
-
type: "product_carousel",
|
|
5132
|
-
title: "Recommended Products",
|
|
5133
|
-
description: `Found ${products.length} relevant products`,
|
|
5134
|
-
data: products
|
|
5135
|
-
};
|
|
4932
|
+
async chat(message, history = [], namespace) {
|
|
4933
|
+
await this.validationPromise;
|
|
4934
|
+
return this.pipeline.ask(message, history, namespace);
|
|
5136
4935
|
}
|
|
5137
4936
|
/**
|
|
5138
|
-
*
|
|
4937
|
+
* Run a streaming chat query.
|
|
5139
4938
|
*/
|
|
5140
|
-
|
|
5141
|
-
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
value: count,
|
|
5146
|
-
inStock: this.checkStockStatus(label, data)
|
|
5147
|
-
}));
|
|
5148
|
-
return {
|
|
5149
|
-
type: "pie_chart",
|
|
5150
|
-
title: "Distribution by Category",
|
|
5151
|
-
description: `Showing breakdown across ${categories.length} categories`,
|
|
5152
|
-
data: pieData
|
|
5153
|
-
};
|
|
4939
|
+
chatStream(_0) {
|
|
4940
|
+
return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
|
|
4941
|
+
yield new __await(this.validationPromise);
|
|
4942
|
+
yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
|
|
4943
|
+
});
|
|
5154
4944
|
}
|
|
5155
4945
|
/**
|
|
5156
|
-
*
|
|
4946
|
+
* Ingest documents into the vector database.
|
|
5157
4947
|
*/
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5161
|
-
const barData = Object.entries(categoryData).map(([category, value]) => ({
|
|
5162
|
-
category,
|
|
5163
|
-
value,
|
|
5164
|
-
inStock: this.checkStockStatus(category, data)
|
|
5165
|
-
}));
|
|
5166
|
-
return {
|
|
5167
|
-
type: "bar_chart",
|
|
5168
|
-
title: "Comparison by Category",
|
|
5169
|
-
description: `Comparing ${categories.length} categories`,
|
|
5170
|
-
data: barData
|
|
5171
|
-
};
|
|
4948
|
+
async ingest(documents, namespace) {
|
|
4949
|
+
await this.validationPromise;
|
|
4950
|
+
return this.pipeline.ingest(documents, namespace);
|
|
5172
4951
|
}
|
|
5173
4952
|
/**
|
|
5174
|
-
*
|
|
4953
|
+
* Get auto-suggestions based on a query prefix.
|
|
5175
4954
|
*/
|
|
5176
|
-
|
|
5177
|
-
|
|
5178
|
-
|
|
5179
|
-
|
|
5180
|
-
|
|
5181
|
-
|
|
5182
|
-
|
|
5183
|
-
|
|
5184
|
-
|
|
5185
|
-
|
|
5186
|
-
|
|
5187
|
-
|
|
5188
|
-
|
|
4955
|
+
async getSuggestions(query, namespace) {
|
|
4956
|
+
await this.validationPromise;
|
|
4957
|
+
return this.pipeline.getSuggestions(query, namespace);
|
|
4958
|
+
}
|
|
4959
|
+
};
|
|
4960
|
+
|
|
4961
|
+
// src/config/ConfigBuilder.ts
|
|
4962
|
+
var ConfigBuilder = class {
|
|
4963
|
+
/**
|
|
4964
|
+
* Set the project/application ID for namespacing
|
|
4965
|
+
*/
|
|
4966
|
+
projectId(id) {
|
|
4967
|
+
this._projectId = id;
|
|
4968
|
+
return this;
|
|
5189
4969
|
}
|
|
5190
4970
|
/**
|
|
5191
|
-
*
|
|
4971
|
+
* Configure the vector database provider
|
|
5192
4972
|
*/
|
|
5193
|
-
|
|
5194
|
-
|
|
5195
|
-
|
|
5196
|
-
|
|
5197
|
-
|
|
5198
|
-
|
|
5199
|
-
|
|
5200
|
-
|
|
5201
|
-
|
|
5202
|
-
|
|
5203
|
-
|
|
5204
|
-
|
|
5205
|
-
|
|
4973
|
+
vectorDb(provider, options) {
|
|
4974
|
+
var _a;
|
|
4975
|
+
if (provider === "auto") {
|
|
4976
|
+
this._vectorDb = this._autoDetectVectorDb();
|
|
4977
|
+
} else {
|
|
4978
|
+
this._vectorDb = {
|
|
4979
|
+
provider,
|
|
4980
|
+
indexName: (_a = options == null ? void 0 : options.indexName) != null ? _a : "default",
|
|
4981
|
+
options: __spreadValues({}, options)
|
|
4982
|
+
};
|
|
4983
|
+
}
|
|
4984
|
+
return this;
|
|
4985
|
+
}
|
|
4986
|
+
/**
|
|
4987
|
+
* Configure the LLM provider for chat
|
|
4988
|
+
*/
|
|
4989
|
+
llm(provider, model, apiKey, options) {
|
|
4990
|
+
var _a, _b;
|
|
4991
|
+
if (provider === "auto") {
|
|
4992
|
+
this._llm = this._autoDetectLLM();
|
|
4993
|
+
} else {
|
|
4994
|
+
this._llm = {
|
|
4995
|
+
provider,
|
|
4996
|
+
model: model != null ? model : "default-model",
|
|
4997
|
+
apiKey,
|
|
4998
|
+
systemPrompt: options == null ? void 0 : options.systemPrompt,
|
|
4999
|
+
maxTokens: (_a = options == null ? void 0 : options.maxTokens) != null ? _a : 1024,
|
|
5000
|
+
temperature: (_b = options == null ? void 0 : options.temperature) != null ? _b : 0.7,
|
|
5001
|
+
baseUrl: options == null ? void 0 : options.baseUrl,
|
|
5002
|
+
options
|
|
5003
|
+
};
|
|
5004
|
+
}
|
|
5005
|
+
return this;
|
|
5006
|
+
}
|
|
5007
|
+
/**
|
|
5008
|
+
* Configure the embedding provider
|
|
5009
|
+
*/
|
|
5010
|
+
embedding(provider, model, apiKey, options) {
|
|
5011
|
+
if (provider === "auto") {
|
|
5012
|
+
this._embedding = this._autoDetectEmbedding();
|
|
5013
|
+
} else {
|
|
5014
|
+
this._embedding = {
|
|
5015
|
+
provider,
|
|
5016
|
+
model: model != null ? model : "default-embedding",
|
|
5017
|
+
apiKey,
|
|
5018
|
+
baseUrl: options == null ? void 0 : options.baseUrl,
|
|
5019
|
+
options
|
|
5020
|
+
};
|
|
5021
|
+
}
|
|
5022
|
+
return this;
|
|
5206
5023
|
}
|
|
5207
5024
|
/**
|
|
5208
|
-
*
|
|
5025
|
+
* Set RAG-specific pipeline parameters
|
|
5209
5026
|
*/
|
|
5210
|
-
|
|
5211
|
-
|
|
5212
|
-
|
|
5213
|
-
|
|
5214
|
-
textContent,
|
|
5215
|
-
`Found ${data.length} relevant results`
|
|
5216
|
-
);
|
|
5027
|
+
rag(options) {
|
|
5028
|
+
var _a;
|
|
5029
|
+
this._rag = __spreadValues(__spreadValues({}, (_a = this._rag) != null ? _a : {}), options);
|
|
5030
|
+
return this;
|
|
5217
5031
|
}
|
|
5218
5032
|
/**
|
|
5219
|
-
*
|
|
5033
|
+
* Set UI branding and appearance options.
|
|
5034
|
+
* Accepts the full UIConfig interface.
|
|
5220
5035
|
*/
|
|
5221
|
-
|
|
5222
|
-
|
|
5223
|
-
|
|
5224
|
-
title,
|
|
5225
|
-
description,
|
|
5226
|
-
data: { content }
|
|
5227
|
-
};
|
|
5036
|
+
ui(options) {
|
|
5037
|
+
this._ui = options;
|
|
5038
|
+
return this;
|
|
5228
5039
|
}
|
|
5229
5040
|
/**
|
|
5230
|
-
*
|
|
5041
|
+
* Build and return the final RagConfig.
|
|
5042
|
+
* Throws if required fields (projectId, vectorDb, llm, embedding) are not set.
|
|
5231
5043
|
*/
|
|
5232
|
-
|
|
5233
|
-
|
|
5234
|
-
const
|
|
5235
|
-
|
|
5236
|
-
|
|
5237
|
-
);
|
|
5044
|
+
build() {
|
|
5045
|
+
var _a;
|
|
5046
|
+
const missing = [];
|
|
5047
|
+
if (!this._projectId) missing.push('projectId (call .projectId("my-app"))');
|
|
5048
|
+
if (!this._vectorDb) missing.push('vectorDb (call .vectorDb("pinecone", { ... }))');
|
|
5049
|
+
if (!this._llm) missing.push('llm (call .llm("openai", "gpt-4o", apiKey))');
|
|
5050
|
+
if (!this._embedding) missing.push('embedding (call .embedding("openai", "text-embedding-3-small"))');
|
|
5051
|
+
if (missing.length > 0) {
|
|
5052
|
+
throw new Error(
|
|
5053
|
+
`[ConfigBuilder] Cannot build \u2014 required fields are missing:
|
|
5054
|
+
${missing.join("\n ")}`
|
|
5055
|
+
);
|
|
5056
|
+
}
|
|
5057
|
+
return __spreadValues({
|
|
5058
|
+
projectId: this._projectId,
|
|
5059
|
+
vectorDb: this._vectorDb,
|
|
5060
|
+
llm: this._llm,
|
|
5061
|
+
embedding: this._embedding,
|
|
5062
|
+
ui: this._rag ? this._ui : void 0,
|
|
5063
|
+
rag: (_a = this._rag) != null ? _a : { chunkSize: 1e3, chunkOverlap: 200, topK: 5 }
|
|
5064
|
+
}, this._ui ? { ui: this._ui } : {});
|
|
5238
5065
|
}
|
|
5239
5066
|
/**
|
|
5240
|
-
*
|
|
5067
|
+
* Build and return as JSON for serialization
|
|
5241
5068
|
*/
|
|
5242
|
-
|
|
5243
|
-
|
|
5244
|
-
const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
|
|
5245
|
-
return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
|
|
5246
|
-
(k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
|
|
5247
|
-
);
|
|
5069
|
+
toJSON() {
|
|
5070
|
+
return JSON.stringify(this.build());
|
|
5248
5071
|
}
|
|
5249
|
-
|
|
5250
|
-
|
|
5251
|
-
|
|
5252
|
-
|
|
5253
|
-
|
|
5254
|
-
|
|
5255
|
-
return {
|
|
5256
|
-
id: item.id,
|
|
5257
|
-
name: meta.name || meta.product,
|
|
5258
|
-
price: meta.price,
|
|
5259
|
-
image: meta.image || meta.imageUrl,
|
|
5260
|
-
brand: meta.brand,
|
|
5261
|
-
description: item.content,
|
|
5262
|
-
inStock: this.determineStockStatus(item)
|
|
5263
|
-
};
|
|
5072
|
+
// ============================================================================
|
|
5073
|
+
// Private helper methods for auto-detection
|
|
5074
|
+
// ============================================================================
|
|
5075
|
+
_autoDetectVectorDb() {
|
|
5076
|
+
if (process.env.PINECONE_API_KEY && process.env.PINECONE_INDEX) {
|
|
5077
|
+
return { provider: "pinecone", indexName: process.env.PINECONE_INDEX, options: { apiKey: process.env.PINECONE_API_KEY } };
|
|
5264
5078
|
}
|
|
5265
|
-
|
|
5266
|
-
|
|
5267
|
-
if (nameMatch) {
|
|
5268
|
-
return {
|
|
5269
|
-
id: item.id,
|
|
5270
|
-
name: nameMatch[1],
|
|
5271
|
-
price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
|
|
5272
|
-
description: item.content,
|
|
5273
|
-
inStock: this.determineStockStatus(item)
|
|
5274
|
-
};
|
|
5079
|
+
if (process.env.QDRANT_URL) {
|
|
5080
|
+
return { provider: "qdrant", indexName: process.env.QDRANT_COLLECTION || "documents", options: { url: process.env.QDRANT_URL, apiKey: process.env.QDRANT_API_KEY } };
|
|
5275
5081
|
}
|
|
5276
|
-
|
|
5082
|
+
if (process.env.DATABASE_URL) {
|
|
5083
|
+
return { provider: "postgresql", indexName: process.env.PG_TABLE || "documents", options: { connectionString: process.env.DATABASE_URL } };
|
|
5084
|
+
}
|
|
5085
|
+
if (process.env.MONGODB_URI) {
|
|
5086
|
+
return { provider: "mongodb", indexName: process.env.MONGODB_COLLECTION || "documents", options: { uri: process.env.MONGODB_URI, database: process.env.MONGODB_DB || "ai_db", collection: process.env.MONGODB_COLLECTION || "documents" } };
|
|
5087
|
+
}
|
|
5088
|
+
if (process.env.REDIS_URL) {
|
|
5089
|
+
return { provider: "redis", indexName: process.env.REDIS_INDEX || "documents", options: { url: process.env.REDIS_URL } };
|
|
5090
|
+
}
|
|
5091
|
+
return { provider: "qdrant", indexName: "documents", options: { url: process.env.QDRANT_URL || "http://localhost:6333" } };
|
|
5092
|
+
}
|
|
5093
|
+
_autoDetectLLM() {
|
|
5094
|
+
if (process.env.OPENAI_API_KEY) {
|
|
5095
|
+
return { provider: "openai", model: process.env.OPENAI_MODEL || "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY, maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OPENAI_TEMPERATURE || "0.7") };
|
|
5096
|
+
}
|
|
5097
|
+
if (process.env.ANTHROPIC_API_KEY) {
|
|
5098
|
+
return { provider: "anthropic", model: process.env.ANTHROPIC_MODEL || "claude-3-haiku-20240307", apiKey: process.env.ANTHROPIC_API_KEY, maxTokens: parseInt(process.env.ANTHROPIC_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.ANTHROPIC_TEMPERATURE || "0.7") };
|
|
5099
|
+
}
|
|
5100
|
+
if (process.env.OLLAMA_BASE_URL) {
|
|
5101
|
+
return { provider: "ollama", model: process.env.OLLAMA_MODEL || "mistral", baseUrl: process.env.OLLAMA_BASE_URL, maxTokens: parseInt(process.env.OLLAMA_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OLLAMA_TEMPERATURE || "0.7") };
|
|
5102
|
+
}
|
|
5103
|
+
return { provider: "openai", model: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY };
|
|
5104
|
+
}
|
|
5105
|
+
_autoDetectEmbedding() {
|
|
5106
|
+
if (process.env.EMBEDDING_PROVIDER) {
|
|
5107
|
+
return { provider: process.env.EMBEDDING_PROVIDER, model: process.env.EMBEDDING_MODEL || "default", apiKey: process.env.EMBEDDING_API_KEY, baseUrl: process.env.EMBEDDING_BASE_URL };
|
|
5108
|
+
}
|
|
5109
|
+
return { provider: "openai", model: "text-embedding-3-small", apiKey: process.env.OPENAI_API_KEY };
|
|
5277
5110
|
}
|
|
5111
|
+
};
|
|
5112
|
+
var PRESETS = {
|
|
5113
|
+
/** OpenAI + Pinecone: Production-ready cloud setup */
|
|
5114
|
+
"openai-pinecone": { vectorDb: "pinecone", llm: "openai", embedding: "openai" },
|
|
5115
|
+
/** Claude + Qdrant: Open-source vector DB + proprietary LLM */
|
|
5116
|
+
"claude-qdrant": { vectorDb: "qdrant", llm: "anthropic", embedding: "openai" },
|
|
5117
|
+
/** Local development: Ollama + local Qdrant */
|
|
5118
|
+
"local-dev": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
|
|
5119
|
+
/** Fully open-source: Ollama LLM + Qdrant + Ollama embeddings */
|
|
5120
|
+
"fully-open-source": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
|
|
5121
|
+
/** PostgreSQL stack: pgvector + OpenAI */
|
|
5122
|
+
"postgres-openai": { vectorDb: "postgresql", llm: "openai", embedding: "openai" },
|
|
5123
|
+
/** Enterprise MongoDB: MongoDB Atlas with OpenAI */
|
|
5124
|
+
"mongodb-openai": { vectorDb: "mongodb", llm: "openai", embedding: "openai" },
|
|
5125
|
+
/** Redis stack for caching + search */
|
|
5126
|
+
"redis-openai": { vectorDb: "redis", llm: "openai", embedding: "openai" }
|
|
5127
|
+
};
|
|
5128
|
+
function createFromPreset(presetName) {
|
|
5129
|
+
const preset = PRESETS[presetName];
|
|
5130
|
+
return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
|
|
5131
|
+
}
|
|
5132
|
+
|
|
5133
|
+
// src/utils/DocumentParser.ts
|
|
5134
|
+
var DocumentParser = class {
|
|
5278
5135
|
/**
|
|
5279
|
-
*
|
|
5136
|
+
* Extract text from a File or Buffer based on its type.
|
|
5280
5137
|
*/
|
|
5281
|
-
static
|
|
5282
|
-
|
|
5283
|
-
|
|
5284
|
-
|
|
5285
|
-
|
|
5286
|
-
|
|
5287
|
-
|
|
5288
|
-
|
|
5289
|
-
|
|
5138
|
+
static async parse(file, fileName, mimeType) {
|
|
5139
|
+
var _a;
|
|
5140
|
+
const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
|
|
5141
|
+
if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
5142
|
+
return this.readAsText(file);
|
|
5143
|
+
}
|
|
5144
|
+
if (extension === "json" || mimeType === "application/json") {
|
|
5145
|
+
const text = await this.readAsText(file);
|
|
5146
|
+
try {
|
|
5147
|
+
const obj = JSON.parse(text);
|
|
5148
|
+
return JSON.stringify(obj, null, 2);
|
|
5149
|
+
} catch (e) {
|
|
5150
|
+
return text;
|
|
5290
5151
|
}
|
|
5291
|
-
|
|
5292
|
-
|
|
5293
|
-
|
|
5152
|
+
}
|
|
5153
|
+
if (extension === "csv" || mimeType === "text/csv") {
|
|
5154
|
+
return this.readAsText(file);
|
|
5155
|
+
}
|
|
5156
|
+
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
5157
|
+
try {
|
|
5158
|
+
const pdf = await import("pdf-parse");
|
|
5159
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
5160
|
+
const data = await pdf.default(buffer);
|
|
5161
|
+
return data.text;
|
|
5162
|
+
} catch (err) {
|
|
5163
|
+
console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
|
|
5164
|
+
return `[PDF Parsing Error for ${fileName}]`;
|
|
5294
5165
|
}
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
|
|
5166
|
+
}
|
|
5167
|
+
if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
|
|
5168
|
+
try {
|
|
5169
|
+
const mammoth = await import("mammoth");
|
|
5170
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
5171
|
+
const result = await mammoth.extractRawText({ buffer });
|
|
5172
|
+
return result.value;
|
|
5173
|
+
} catch (err) {
|
|
5174
|
+
console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
|
|
5175
|
+
return `[DOCX Parsing Error for ${fileName}]`;
|
|
5298
5176
|
}
|
|
5299
|
-
}
|
|
5300
|
-
|
|
5177
|
+
}
|
|
5178
|
+
try {
|
|
5179
|
+
return await this.readAsText(file);
|
|
5180
|
+
} catch (e) {
|
|
5181
|
+
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
5182
|
+
}
|
|
5183
|
+
}
|
|
5184
|
+
static async readAsText(file) {
|
|
5185
|
+
if (Buffer.isBuffer(file)) {
|
|
5186
|
+
return file.toString("utf-8");
|
|
5187
|
+
}
|
|
5188
|
+
return await file.text();
|
|
5189
|
+
}
|
|
5190
|
+
};
|
|
5191
|
+
|
|
5192
|
+
// src/server.ts
|
|
5193
|
+
init_BaseVectorProvider();
|
|
5194
|
+
init_PineconeProvider();
|
|
5195
|
+
init_PostgreSQLProvider();
|
|
5196
|
+
|
|
5197
|
+
// src/providers/vectordb/MultiTablePostgresProvider.ts
|
|
5198
|
+
var import_pg2 = require("pg");
|
|
5199
|
+
init_BaseVectorProvider();
|
|
5200
|
+
var MultiTablePostgresProvider = class extends BaseVectorProvider {
|
|
5201
|
+
constructor(config) {
|
|
5202
|
+
var _a, _b, _c, _d, _e;
|
|
5203
|
+
super(config);
|
|
5204
|
+
const opts = config.options || {};
|
|
5205
|
+
if (!opts.connectionString) {
|
|
5206
|
+
throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
|
|
5207
|
+
}
|
|
5208
|
+
this.connectionString = opts.connectionString;
|
|
5209
|
+
this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
|
|
5210
|
+
const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
|
|
5211
|
+
this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
|
|
5212
|
+
if (this.tables.length === 0) {
|
|
5213
|
+
console.warn(
|
|
5214
|
+
"[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
|
|
5215
|
+
);
|
|
5216
|
+
}
|
|
5217
|
+
const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
|
|
5218
|
+
this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
|
|
5301
5219
|
}
|
|
5302
|
-
|
|
5303
|
-
|
|
5304
|
-
|
|
5305
|
-
|
|
5306
|
-
|
|
5307
|
-
|
|
5308
|
-
|
|
5309
|
-
|
|
5310
|
-
|
|
5311
|
-
|
|
5312
|
-
const
|
|
5313
|
-
if (
|
|
5314
|
-
|
|
5220
|
+
async initialize() {
|
|
5221
|
+
this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
|
|
5222
|
+
const client = await this.pool.connect();
|
|
5223
|
+
try {
|
|
5224
|
+
const res = await client.query(`
|
|
5225
|
+
SELECT table_name
|
|
5226
|
+
FROM information_schema.columns
|
|
5227
|
+
WHERE column_name = 'embedding'
|
|
5228
|
+
AND table_schema = 'public'
|
|
5229
|
+
`);
|
|
5230
|
+
const discoveredTables = res.rows.map((r) => r.table_name);
|
|
5231
|
+
if (this.tables.length === 0) {
|
|
5232
|
+
this.tables = discoveredTables;
|
|
5315
5233
|
} else {
|
|
5316
|
-
|
|
5234
|
+
const staticTables = [...this.tables];
|
|
5235
|
+
this.tables = staticTables.filter((t) => discoveredTables.includes(t));
|
|
5236
|
+
if (this.tables.length < staticTables.length) {
|
|
5237
|
+
const missing = staticTables.filter((t) => !discoveredTables.includes(t));
|
|
5238
|
+
console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
|
|
5239
|
+
}
|
|
5317
5240
|
}
|
|
5318
|
-
|
|
5319
|
-
|
|
5241
|
+
if (this.tables.length === 0) {
|
|
5242
|
+
console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
|
|
5243
|
+
} else {
|
|
5244
|
+
console.log(
|
|
5245
|
+
`[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(", ")}`
|
|
5246
|
+
);
|
|
5247
|
+
}
|
|
5248
|
+
} finally {
|
|
5249
|
+
client.release();
|
|
5250
|
+
}
|
|
5320
5251
|
}
|
|
5321
5252
|
/**
|
|
5322
|
-
*
|
|
5253
|
+
* Upsert is not supported for MultiTablePostgresProvider as it's designed for
|
|
5254
|
+
* searching across pre-existing tables with varying schemas.
|
|
5323
5255
|
*/
|
|
5324
|
-
|
|
5325
|
-
|
|
5326
|
-
|
|
5327
|
-
|
|
5328
|
-
timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
|
|
5329
|
-
value: meta.value || item.score || 0,
|
|
5330
|
-
label: meta.label || item.content.substring(0, 50)
|
|
5331
|
-
};
|
|
5332
|
-
});
|
|
5256
|
+
async upsert(_doc, _namespace) {
|
|
5257
|
+
throw new Error(
|
|
5258
|
+
"[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
|
|
5259
|
+
);
|
|
5333
5260
|
}
|
|
5334
5261
|
/**
|
|
5335
|
-
*
|
|
5262
|
+
* Batch upsert is not supported for MultiTablePostgresProvider.
|
|
5336
5263
|
*/
|
|
5337
|
-
|
|
5338
|
-
|
|
5339
|
-
|
|
5340
|
-
|
|
5341
|
-
Object.keys(item.metadata || {}).forEach((key) => {
|
|
5342
|
-
columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
|
|
5343
|
-
});
|
|
5344
|
-
});
|
|
5345
|
-
return Array.from(columnSet);
|
|
5264
|
+
async batchUpsert(_docs, _namespace) {
|
|
5265
|
+
throw new Error(
|
|
5266
|
+
"[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
|
|
5267
|
+
);
|
|
5346
5268
|
}
|
|
5347
5269
|
/**
|
|
5348
|
-
*
|
|
5270
|
+
* Query all configured tables and merge results, sorted by cosine similarity score.
|
|
5349
5271
|
*/
|
|
5350
|
-
|
|
5351
|
-
|
|
5352
|
-
|
|
5353
|
-
|
|
5354
|
-
|
|
5272
|
+
async query(vector, topK, _namespace, _filter) {
|
|
5273
|
+
var _a, _b, _c;
|
|
5274
|
+
if (!this.pool) {
|
|
5275
|
+
throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
|
|
5276
|
+
}
|
|
5277
|
+
const vectorLiteral = `[${vector.join(",")}]`;
|
|
5278
|
+
const allResults = [];
|
|
5279
|
+
console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
|
|
5280
|
+
const queryText = _filter == null ? void 0 : _filter.queryText;
|
|
5281
|
+
const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter(
|
|
5282
|
+
(hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
|
|
5283
|
+
).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
|
|
5284
|
+
console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
|
|
5285
|
+
console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
|
|
5286
|
+
const getDynamicKeywordQuery = () => {
|
|
5287
|
+
if (entityHints.length > 0) {
|
|
5288
|
+
return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
|
|
5289
|
+
}
|
|
5290
|
+
if (queryText) {
|
|
5291
|
+
return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
|
|
5292
|
+
}
|
|
5293
|
+
return "";
|
|
5294
|
+
};
|
|
5295
|
+
const dynamicKeywordQuery = getDynamicKeywordQuery();
|
|
5296
|
+
const queryPromises = this.tables.map(async (table) => {
|
|
5297
|
+
try {
|
|
5298
|
+
let sqlQuery = "";
|
|
5299
|
+
let params = [];
|
|
5300
|
+
if (queryText) {
|
|
5301
|
+
const hasEntityHints = entityHints.length > 0;
|
|
5302
|
+
const exactNameScoreExpr = hasEntityHints ? `+ (
|
|
5303
|
+
SELECT COALESCE(MAX(
|
|
5304
|
+
CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
|
|
5305
|
+
THEN 1.0 ELSE 0.0 END
|
|
5306
|
+
), 0)
|
|
5307
|
+
FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
|
|
5308
|
+
WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
|
|
5309
|
+
) * 3.0` : "";
|
|
5310
|
+
sqlQuery = `
|
|
5311
|
+
SELECT *,
|
|
5312
|
+
(1 - (embedding <=> $1::vector)) AS vector_score,
|
|
5313
|
+
COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
|
|
5314
|
+
((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
|
|
5315
|
+
FROM "${table}" t
|
|
5316
|
+
ORDER BY hybrid_score DESC
|
|
5317
|
+
LIMIT 50
|
|
5318
|
+
`;
|
|
5319
|
+
params = [vectorLiteral, dynamicKeywordQuery];
|
|
5320
|
+
} else {
|
|
5321
|
+
sqlQuery = `
|
|
5322
|
+
SELECT *,
|
|
5323
|
+
(1 - (embedding <=> $1::vector)) AS hybrid_score
|
|
5324
|
+
FROM "${table}" t
|
|
5325
|
+
ORDER BY hybrid_score DESC
|
|
5326
|
+
LIMIT 50
|
|
5327
|
+
`;
|
|
5328
|
+
params = [vectorLiteral];
|
|
5329
|
+
}
|
|
5330
|
+
const result = await this.pool.query(sqlQuery, params);
|
|
5331
|
+
if (result.rowCount && result.rowCount > 0) {
|
|
5332
|
+
console.log(` \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
|
|
5333
|
+
} else {
|
|
5334
|
+
console.log(` \u26AA Table "${table}": No relevant matches found.`);
|
|
5335
|
+
}
|
|
5336
|
+
const tableResults = [];
|
|
5337
|
+
for (const row of result.rows) {
|
|
5338
|
+
const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
|
|
5339
|
+
delete rest.embedding;
|
|
5340
|
+
delete rest.vector_score;
|
|
5341
|
+
delete rest.keyword_score;
|
|
5342
|
+
delete rest.exact_name_score;
|
|
5343
|
+
const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
|
|
5344
|
+
` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
|
|
5345
|
+
tableResults.push({
|
|
5346
|
+
id: `${table}-${id}`,
|
|
5347
|
+
score: parseFloat(String(hybrid_score)),
|
|
5348
|
+
content,
|
|
5349
|
+
metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
|
|
5350
|
+
});
|
|
5351
|
+
}
|
|
5352
|
+
return tableResults;
|
|
5353
|
+
} catch (err) {
|
|
5354
|
+
console.error(
|
|
5355
|
+
`[MultiTablePostgresProvider] Error querying table "${table}":`,
|
|
5356
|
+
err.message
|
|
5357
|
+
);
|
|
5358
|
+
return [];
|
|
5355
5359
|
}
|
|
5356
|
-
const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
|
|
5357
|
-
const value = meta[metaKey];
|
|
5358
|
-
return value !== void 0 ? String(value) : "";
|
|
5359
|
-
});
|
|
5360
|
-
}
|
|
5361
|
-
/**
|
|
5362
|
-
* Helper: Check stock status
|
|
5363
|
-
*/
|
|
5364
|
-
static checkStockStatus(category, data) {
|
|
5365
|
-
const item = data.find((d) => {
|
|
5366
|
-
const meta = d.metadata || {};
|
|
5367
|
-
return meta.category === category || meta.type === category;
|
|
5368
5360
|
});
|
|
5369
|
-
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
* Helper: Determine if item is in stock
|
|
5373
|
-
*/
|
|
5374
|
-
static determineStockStatus(item) {
|
|
5375
|
-
const meta = item.metadata || {};
|
|
5376
|
-
if (meta.inStock !== void 0) {
|
|
5377
|
-
return Boolean(meta.inStock);
|
|
5361
|
+
const resultsArray = await Promise.all(queryPromises);
|
|
5362
|
+
for (const tableResults of resultsArray) {
|
|
5363
|
+
allResults.push(...tableResults);
|
|
5378
5364
|
}
|
|
5379
|
-
|
|
5380
|
-
|
|
5365
|
+
const resultsByTable = {};
|
|
5366
|
+
for (const res of allResults) {
|
|
5367
|
+
const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
|
|
5368
|
+
if (!resultsByTable[table]) resultsByTable[table] = [];
|
|
5369
|
+
resultsByTable[table].push(res);
|
|
5381
5370
|
}
|
|
5382
|
-
|
|
5383
|
-
|
|
5371
|
+
const balancedResults = [];
|
|
5372
|
+
const tables = Object.keys(resultsByTable);
|
|
5373
|
+
for (const table of tables) {
|
|
5374
|
+
balancedResults.push(...resultsByTable[table].slice(0, 3));
|
|
5384
5375
|
}
|
|
5385
|
-
const
|
|
5386
|
-
if (
|
|
5387
|
-
|
|
5376
|
+
const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
|
|
5377
|
+
if (finalSorted.length > 0) {
|
|
5378
|
+
console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
|
|
5379
|
+
console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
|
|
5388
5380
|
}
|
|
5389
|
-
|
|
5381
|
+
return finalSorted.slice(0, Math.max(topK, 15));
|
|
5382
|
+
}
|
|
5383
|
+
async delete(_id, _namespace) {
|
|
5384
|
+
console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
|
|
5385
|
+
}
|
|
5386
|
+
async deleteNamespace(_namespace) {
|
|
5387
|
+
console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
|
|
5388
|
+
}
|
|
5389
|
+
async ping() {
|
|
5390
|
+
try {
|
|
5391
|
+
await this.pool.query("SELECT 1");
|
|
5390
5392
|
return true;
|
|
5393
|
+
} catch (e) {
|
|
5394
|
+
return false;
|
|
5391
5395
|
}
|
|
5392
|
-
return true;
|
|
5393
5396
|
}
|
|
5394
|
-
|
|
5395
|
-
|
|
5396
|
-
|
|
5397
|
-
|
|
5398
|
-
const fieldCount = /* @__PURE__ */ new Set();
|
|
5399
|
-
data.forEach((item) => {
|
|
5400
|
-
Object.keys(item.metadata || {}).forEach((key) => {
|
|
5401
|
-
fieldCount.add(key);
|
|
5402
|
-
});
|
|
5403
|
-
});
|
|
5404
|
-
return fieldCount.size > 2;
|
|
5397
|
+
async disconnect() {
|
|
5398
|
+
if (this.pool) {
|
|
5399
|
+
await this.pool.end();
|
|
5400
|
+
}
|
|
5405
5401
|
}
|
|
5406
5402
|
};
|
|
5407
5403
|
|
|
5404
|
+
// src/server.ts
|
|
5405
|
+
init_MongoDBProvider();
|
|
5406
|
+
init_MilvusProvider();
|
|
5407
|
+
init_QdrantProvider();
|
|
5408
|
+
init_ChromaDBProvider();
|
|
5409
|
+
init_RedisProvider();
|
|
5410
|
+
init_WeaviateProvider();
|
|
5411
|
+
init_UniversalVectorProvider();
|
|
5412
|
+
|
|
5408
5413
|
// src/handlers/index.ts
|
|
5414
|
+
var import_server = require("next/server");
|
|
5409
5415
|
function sseFrame(payload) {
|
|
5410
5416
|
return `data: ${JSON.stringify(payload)}
|
|
5411
5417
|
|