@retrivora-ai/rag-engine 1.7.6 → 1.7.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.js CHANGED
@@ -3865,65 +3865,410 @@ var QueryProcessor = class {
3865
3865
  }
3866
3866
  };
3867
3867
 
3868
- // src/core/Pipeline.ts
3869
- var LRUEmbeddingCache = class {
3870
- constructor(maxSize = 500) {
3871
- this.cache = /* @__PURE__ */ new Map();
3872
- this.maxSize = maxSize;
3873
- }
3874
- get(key) {
3875
- const value = this.cache.get(key);
3876
- if (value !== void 0) {
3877
- this.cache.delete(key);
3878
- this.cache.set(key, value);
3868
+ // src/utils/UITransformer.ts
3869
+ var UITransformer = class {
3870
+ /**
3871
+ * Main transformation method
3872
+ * Analyzes user query and retrieved data to determine the best visualization format
3873
+ *
3874
+ * @param userQuery - The original user question/query
3875
+ * @param retrievedData - Vector database retrieval results
3876
+ * @returns Structured JSON response for UI rendering
3877
+ */
3878
+ static transform(userQuery, retrievedData) {
3879
+ if (!retrievedData || retrievedData.length === 0) {
3880
+ return this.createTextResponse("No data available", "No relevant data found for your query.");
3881
+ }
3882
+ const analysis = this.analyzeData(userQuery, retrievedData);
3883
+ switch (analysis.suggestedType) {
3884
+ case "product_carousel":
3885
+ return this.transformToProductCarousel(retrievedData);
3886
+ case "pie_chart":
3887
+ return this.transformToPieChart(retrievedData);
3888
+ case "bar_chart":
3889
+ return this.transformToBarChart(retrievedData);
3890
+ case "line_chart":
3891
+ return this.transformToLineChart(retrievedData);
3892
+ case "table":
3893
+ return this.transformToTable(retrievedData);
3894
+ default:
3895
+ return this.transformToText(retrievedData);
3879
3896
  }
3880
- return value;
3881
3897
  }
3882
- set(key, value) {
3883
- if (this.cache.has(key)) {
3884
- this.cache.delete(key);
3885
- } else if (this.cache.size >= this.maxSize) {
3886
- const oldestKey = this.cache.keys().next().value;
3887
- if (oldestKey !== void 0) this.cache.delete(oldestKey);
3898
+ /**
3899
+ * Analyzes the data and query to suggest the best visualization type
3900
+ */
3901
+ static analyzeData(query, data) {
3902
+ const queryLower = query.toLowerCase();
3903
+ const hasProducts = data.some(
3904
+ (item) => this.isProductData(item)
3905
+ );
3906
+ const hasTimeSeries = data.some(
3907
+ (item) => this.isTimeSeriesData(item)
3908
+ );
3909
+ const hasCategories = this.detectCategories(data).length > 0;
3910
+ const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
3911
+ const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
3912
+ const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
3913
+ if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
3914
+ return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
3888
3915
  }
3889
- this.cache.set(key, value);
3916
+ if (isTrendQuery && hasTimeSeries) {
3917
+ return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
3918
+ }
3919
+ if (isDistributionQuery && hasCategories) {
3920
+ return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
3921
+ }
3922
+ if (isComparisonQuery && hasCategories && data.length > 2) {
3923
+ return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
3924
+ }
3925
+ if (data.length > 5 && this.hasMultipleFields(data)) {
3926
+ return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
3927
+ }
3928
+ return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
3890
3929
  }
3891
- clear() {
3892
- this.cache.clear();
3930
+ /**
3931
+ * Transform data to product carousel format
3932
+ */
3933
+ static transformToProductCarousel(data) {
3934
+ const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
3935
+ return {
3936
+ type: "product_carousel",
3937
+ title: "Recommended Products",
3938
+ description: `Found ${products.length} relevant products`,
3939
+ data: products
3940
+ };
3893
3941
  }
3894
- get size() {
3895
- return this.cache.size;
3942
+ /**
3943
+ * Transform data to pie chart format
3944
+ */
3945
+ static transformToPieChart(data) {
3946
+ const categories = this.detectCategories(data);
3947
+ const categoryData = this.aggregateByCategory(data, categories);
3948
+ const pieData = Object.entries(categoryData).map(([label, count]) => ({
3949
+ label,
3950
+ value: count,
3951
+ inStock: this.checkStockStatus(label, data)
3952
+ }));
3953
+ return {
3954
+ type: "pie_chart",
3955
+ title: "Distribution by Category",
3956
+ description: `Showing breakdown across ${categories.length} categories`,
3957
+ data: pieData
3958
+ };
3896
3959
  }
3897
- };
3898
- var Pipeline = class {
3899
- constructor(config) {
3900
- this.config = config;
3901
- /** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
3902
- this.embeddingCache = new LRUEmbeddingCache(500);
3903
- this.initialised = false;
3904
- var _a, _b, _c, _d, _e;
3905
- this.chunker = new DocumentChunker(
3906
- (_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
3907
- (_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
3960
+ /**
3961
+ * Transform data to bar chart format
3962
+ */
3963
+ static transformToBarChart(data) {
3964
+ const categories = this.detectCategories(data);
3965
+ const categoryData = this.aggregateByCategory(data, categories);
3966
+ const barData = Object.entries(categoryData).map(([category, value]) => ({
3967
+ category,
3968
+ value,
3969
+ inStock: this.checkStockStatus(category, data)
3970
+ }));
3971
+ return {
3972
+ type: "bar_chart",
3973
+ title: "Comparison by Category",
3974
+ description: `Comparing ${categories.length} categories`,
3975
+ data: barData
3976
+ };
3977
+ }
3978
+ /**
3979
+ * Transform data to line chart format
3980
+ */
3981
+ static transformToLineChart(data) {
3982
+ const timePoints = this.extractTimeSeriesData(data);
3983
+ const lineData = timePoints.map((point) => ({
3984
+ timestamp: point.timestamp,
3985
+ value: point.value,
3986
+ label: point.label
3987
+ }));
3988
+ return {
3989
+ type: "line_chart",
3990
+ title: "Trend Over Time",
3991
+ description: `Showing ${lineData.length} data points`,
3992
+ data: lineData
3993
+ };
3994
+ }
3995
+ /**
3996
+ * Transform data to table format
3997
+ */
3998
+ static transformToTable(data) {
3999
+ const columns = this.extractTableColumns(data);
4000
+ const rows = data.map((item) => this.extractTableRow(item, columns));
4001
+ const tableData = {
4002
+ columns,
4003
+ rows
4004
+ };
4005
+ return {
4006
+ type: "table",
4007
+ title: "Detailed Results",
4008
+ description: `Showing ${data.length} results`,
4009
+ data: tableData
4010
+ };
4011
+ }
4012
+ /**
4013
+ * Transform data to text format (fallback)
4014
+ */
4015
+ static transformToText(data) {
4016
+ const textContent = data.map((item) => item.content).join("\n\n");
4017
+ return this.createTextResponse(
4018
+ "Search Results",
4019
+ textContent,
4020
+ `Found ${data.length} relevant results`
3908
4021
  );
3909
- if (((_e = config.rag) == null ? void 0 : _e.chunkingStrategy) === "llamaindex") {
3910
- this.llamaIngestor = new LlamaIndexIngestor();
4022
+ }
4023
+ /**
4024
+ * Helper: Create text response
4025
+ */
4026
+ static createTextResponse(title, content, description) {
4027
+ return {
4028
+ type: "text",
4029
+ title,
4030
+ description,
4031
+ data: { content }
4032
+ };
4033
+ }
4034
+ /**
4035
+ * Helper: Check if data item is product-related
4036
+ */
4037
+ static isProductData(item) {
4038
+ const content = (item.content || "").toLowerCase();
4039
+ const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
4040
+ return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
4041
+ (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
4042
+ );
4043
+ }
4044
+ /**
4045
+ * Helper: Check if data contains time series
4046
+ */
4047
+ static isTimeSeriesData(item) {
4048
+ const content = (item.content || "").toLowerCase();
4049
+ const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
4050
+ return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
4051
+ (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
4052
+ );
4053
+ }
4054
+ /**
4055
+ * Helper: Extract product information from vector match
4056
+ */
4057
+ static extractProductInfo(item) {
4058
+ const meta = item.metadata || {};
4059
+ if (meta.name || meta.product) {
4060
+ return {
4061
+ id: item.id,
4062
+ name: meta.name || meta.product,
4063
+ price: meta.price,
4064
+ image: meta.image || meta.imageUrl,
4065
+ brand: meta.brand,
4066
+ description: item.content,
4067
+ inStock: this.determineStockStatus(item)
4068
+ };
3911
4069
  }
3912
- this.reranker = new Reranker();
4070
+ const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
4071
+ const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
4072
+ if (nameMatch) {
4073
+ return {
4074
+ id: item.id,
4075
+ name: nameMatch[1],
4076
+ price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
4077
+ description: item.content,
4078
+ inStock: this.determineStockStatus(item)
4079
+ };
4080
+ }
4081
+ return null;
3913
4082
  }
3914
- async initialize() {
3915
- var _a, _b;
3916
- if (this.initialised) return;
3917
- const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
3918
- const chartInstruction = `
3919
-
3920
- ${CHART_MARKER}
3921
- ### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
3922
-
3923
- You are responsible for returning a SINGLE structured UI block when visualization is required.
3924
-
3925
- ---
3926
-
4083
+ /**
4084
+ * Helper: Detect categories in data
4085
+ */
4086
+ static detectCategories(data) {
4087
+ const categories = /* @__PURE__ */ new Set();
4088
+ data.forEach((item) => {
4089
+ const meta = item.metadata || {};
4090
+ if (meta.category) {
4091
+ categories.add(String(meta.category));
4092
+ }
4093
+ if (meta.type) {
4094
+ categories.add(String(meta.type));
4095
+ }
4096
+ if (meta.tag) {
4097
+ const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
4098
+ tags.forEach((t) => categories.add(String(t)));
4099
+ }
4100
+ const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
4101
+ if (categoryMatch) {
4102
+ categories.add(categoryMatch[1].trim());
4103
+ }
4104
+ });
4105
+ return Array.from(categories);
4106
+ }
4107
+ /**
4108
+ * Helper: Aggregate data by category
4109
+ */
4110
+ static aggregateByCategory(data, categories) {
4111
+ const result = {};
4112
+ categories.forEach((cat) => {
4113
+ result[cat] = 0;
4114
+ });
4115
+ data.forEach((item) => {
4116
+ const meta = item.metadata || {};
4117
+ const itemCategory = meta.category || meta.type || "Other";
4118
+ if (result.hasOwnProperty(itemCategory)) {
4119
+ result[itemCategory]++;
4120
+ } else {
4121
+ result["Other"] = (result["Other"] || 0) + 1;
4122
+ }
4123
+ });
4124
+ return result;
4125
+ }
4126
+ /**
4127
+ * Helper: Extract time series data
4128
+ */
4129
+ static extractTimeSeriesData(data) {
4130
+ return data.map((item) => {
4131
+ const meta = item.metadata || {};
4132
+ return {
4133
+ timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
4134
+ value: meta.value || item.score || 0,
4135
+ label: meta.label || item.content.substring(0, 50)
4136
+ };
4137
+ });
4138
+ }
4139
+ /**
4140
+ * Helper: Extract table columns
4141
+ */
4142
+ static extractTableColumns(data) {
4143
+ const columnSet = /* @__PURE__ */ new Set();
4144
+ columnSet.add("Content");
4145
+ data.forEach((item) => {
4146
+ Object.keys(item.metadata || {}).forEach((key) => {
4147
+ columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
4148
+ });
4149
+ });
4150
+ return Array.from(columnSet);
4151
+ }
4152
+ /**
4153
+ * Helper: Extract table row
4154
+ */
4155
+ static extractTableRow(item, columns) {
4156
+ const meta = item.metadata || {};
4157
+ return columns.map((col) => {
4158
+ if (col === "Content") {
4159
+ return item.content.substring(0, 100);
4160
+ }
4161
+ const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
4162
+ const value = meta[metaKey];
4163
+ return value !== void 0 ? String(value) : "";
4164
+ });
4165
+ }
4166
+ /**
4167
+ * Helper: Check stock status
4168
+ */
4169
+ static checkStockStatus(category, data) {
4170
+ const item = data.find((d) => {
4171
+ const meta = d.metadata || {};
4172
+ return meta.category === category || meta.type === category;
4173
+ });
4174
+ return this.determineStockStatus(item || {});
4175
+ }
4176
+ /**
4177
+ * Helper: Determine if item is in stock
4178
+ */
4179
+ static determineStockStatus(item) {
4180
+ const meta = item.metadata || {};
4181
+ if (meta.inStock !== void 0) {
4182
+ return Boolean(meta.inStock);
4183
+ }
4184
+ if (meta.stock !== void 0) {
4185
+ return Boolean(meta.stock);
4186
+ }
4187
+ if (meta.available !== void 0) {
4188
+ return Boolean(meta.available);
4189
+ }
4190
+ const content = (item.content || "").toLowerCase();
4191
+ if (content.includes("out of stock") || content.includes("unavailable")) {
4192
+ return false;
4193
+ }
4194
+ if (content.includes("in stock") || content.includes("available")) {
4195
+ return true;
4196
+ }
4197
+ return true;
4198
+ }
4199
+ /**
4200
+ * Helper: Check if data has multiple fields
4201
+ */
4202
+ static hasMultipleFields(data) {
4203
+ const fieldCount = /* @__PURE__ */ new Set();
4204
+ data.forEach((item) => {
4205
+ Object.keys(item.metadata || {}).forEach((key) => {
4206
+ fieldCount.add(key);
4207
+ });
4208
+ });
4209
+ return fieldCount.size > 2;
4210
+ }
4211
+ };
4212
+
4213
+ // src/core/Pipeline.ts
4214
+ var LRUEmbeddingCache = class {
4215
+ constructor(maxSize = 500) {
4216
+ this.cache = /* @__PURE__ */ new Map();
4217
+ this.maxSize = maxSize;
4218
+ }
4219
+ get(key) {
4220
+ const value = this.cache.get(key);
4221
+ if (value !== void 0) {
4222
+ this.cache.delete(key);
4223
+ this.cache.set(key, value);
4224
+ }
4225
+ return value;
4226
+ }
4227
+ set(key, value) {
4228
+ if (this.cache.has(key)) {
4229
+ this.cache.delete(key);
4230
+ } else if (this.cache.size >= this.maxSize) {
4231
+ const oldestKey = this.cache.keys().next().value;
4232
+ if (oldestKey !== void 0) this.cache.delete(oldestKey);
4233
+ }
4234
+ this.cache.set(key, value);
4235
+ }
4236
+ clear() {
4237
+ this.cache.clear();
4238
+ }
4239
+ get size() {
4240
+ return this.cache.size;
4241
+ }
4242
+ };
4243
+ var Pipeline = class {
4244
+ constructor(config) {
4245
+ this.config = config;
4246
+ /** LRU-bounded cache: avoids re-embedding identical queries within the same process. */
4247
+ this.embeddingCache = new LRUEmbeddingCache(500);
4248
+ this.initialised = false;
4249
+ var _a, _b, _c, _d, _e;
4250
+ this.chunker = new DocumentChunker(
4251
+ (_b = (_a = config.rag) == null ? void 0 : _a.chunkSize) != null ? _b : 1e3,
4252
+ (_d = (_c = config.rag) == null ? void 0 : _c.chunkOverlap) != null ? _d : 200
4253
+ );
4254
+ if (((_e = config.rag) == null ? void 0 : _e.chunkingStrategy) === "llamaindex") {
4255
+ this.llamaIngestor = new LlamaIndexIngestor();
4256
+ }
4257
+ this.reranker = new Reranker();
4258
+ }
4259
+ async initialize() {
4260
+ var _a, _b;
4261
+ if (this.initialised) return;
4262
+ const CHART_MARKER = "<!-- UI_PROTOCOL_V7 -->";
4263
+ const chartInstruction = `
4264
+
4265
+ ${CHART_MARKER}
4266
+ ### UNIVERSAL UI PROTOCOL V7 (STRICT MODE)
4267
+
4268
+ You are responsible for returning a SINGLE structured UI block when visualization is required.
4269
+
4270
+ ---
4271
+
3927
4272
  ## 1. INTENT CLASSIFICATION (MANDATORY)
3928
4273
 
3929
4274
  Classify the query into ONE of the following intents:
@@ -4043,6 +4388,8 @@ Classify the query into ONE of the following intents:
4043
4388
  - ALWAYS return EXACTLY ONE \`\`\`ui\`\`\` block
4044
4389
  - JSON must be VALID
4045
4390
  - No explanation inside block
4391
+ - NEVER use ASCII charts, Markdown tables, or text-based drawings for data.
4392
+ - If data is tabular or statistical, ALWAYS use the \`\`\`ui\`\`\` block.
4046
4393
 
4047
4394
  ---
4048
4395
 
@@ -4101,1311 +4448,970 @@ User: "distribution of products across categories"
4101
4448
  "insights": ["Beauty dominates inventory"]
4102
4449
  }
4103
4450
  \`\`\`
4104
- `;
4105
- if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
4106
- let cleanPrompt = this.config.llm.systemPrompt || "";
4107
- cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
4108
- cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
4109
- this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
4110
- }
4111
- console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
4112
- this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
4113
- const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
4114
- this.config.llm,
4115
- this.config.embedding
4116
- );
4117
- this.llmProvider = llmProvider;
4118
- this.embeddingProvider = embeddingProvider;
4119
- if (this.config.graphDb) {
4120
- this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
4121
- await this.graphDB.initialize();
4122
- this.entityExtractor = new EntityExtractor(this.llmProvider);
4123
- }
4124
- await this.vectorDB.initialize();
4125
- if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
4126
- this.agent = new LangChainAgent(this, this.config);
4127
- await this.agent.initialize(this.llmProvider);
4128
- }
4129
- this.initialised = true;
4130
- }
4131
- /**
4132
- * Ingest documents with automatic chunking, embedding, and batch upsert.
4133
- * Handles retries for transient failures.
4134
- */
4135
- async ingest(documents, namespace) {
4136
- await this.initialize();
4137
- const ns = namespace != null ? namespace : this.config.projectId;
4138
- const results = [];
4139
- for (const doc of documents) {
4140
- try {
4141
- const chunks = await this.prepareChunks(doc);
4142
- const vectors = await this.processEmbeddings(chunks);
4143
- const upsertDocs = chunks.map((chunk, i) => ({
4144
- id: chunk.id,
4145
- vector: vectors[i],
4146
- content: chunk.content,
4147
- metadata: chunk.metadata
4148
- }));
4149
- const totalProcessed = await this.processUpserts(upsertDocs, ns);
4150
- results.push({
4151
- docId: doc.docId,
4152
- chunksIngested: totalProcessed
4153
- });
4154
- if (this.graphDB && this.entityExtractor) {
4155
- await this.processGraphIngestion(doc.docId, chunks);
4156
- }
4157
- } catch (error) {
4158
- console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
4159
- results.push({ docId: doc.docId, chunksIngested: 0 });
4160
- }
4161
- }
4162
- return results;
4163
- }
4164
- /**
4165
- * Step 1: Chunk the document content.
4166
- */
4167
- async prepareChunks(doc) {
4168
- var _a, _b;
4169
- if (this.llamaIngestor) {
4170
- return await this.llamaIngestor.chunk(doc.content, {
4171
- docId: doc.docId,
4172
- metadata: doc.metadata,
4173
- chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
4174
- chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
4175
- });
4176
- }
4177
- return this.chunker.chunk(doc.content, {
4178
- docId: doc.docId,
4179
- metadata: doc.metadata
4180
- });
4181
- }
4182
- /**
4183
- * Step 2: Generate embeddings for chunks with retry logic.
4184
- * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
4185
- */
4186
- async processEmbeddings(chunks) {
4187
- const embedBatchOptions = {
4188
- batchSize: 50,
4189
- maxRetries: 3,
4190
- initialDelayMs: 100
4191
- };
4192
- const vectors = await BatchProcessor.mapWithRetry(
4193
- chunks.map((c) => c.content),
4194
- (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
4195
- embedBatchOptions
4196
- );
4197
- if (vectors.length !== chunks.length) {
4198
- throw new Error(
4199
- `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
4200
- );
4201
- }
4202
- return vectors;
4203
- }
4204
- /**
4205
- * Step 3: Upsert chunks to vector database with retry logic.
4206
- */
4207
- async processUpserts(upsertDocs, namespace) {
4208
- const upsertBatchOptions = {
4209
- batchSize: 100,
4210
- maxRetries: 3,
4211
- initialDelayMs: 100
4212
- };
4213
- const upsertResult = await BatchProcessor.processBatch(
4214
- upsertDocs,
4215
- (batch) => this.vectorDB.batchUpsert(batch, namespace),
4216
- upsertBatchOptions
4217
- );
4218
- if (upsertResult.errors.length > 0) {
4219
- console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
4220
- }
4221
- return upsertResult.totalProcessed;
4222
- }
4223
- /**
4224
- * Step 4: Optional graph-based entity extraction and ingestion.
4225
- */
4226
- async processGraphIngestion(docId, chunks) {
4227
- console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
4228
- const extractionOptions = {
4229
- batchSize: 2,
4230
- // Low concurrency for LLM extraction
4231
- maxRetries: 1,
4232
- initialDelayMs: 500
4233
- };
4234
- await BatchProcessor.processBatch(
4235
- chunks,
4236
- async (batch) => {
4237
- for (const chunk of batch) {
4238
- try {
4239
- const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
4240
- if (nodes.length > 0) await this.graphDB.addNodes(nodes);
4241
- if (edges.length > 0) await this.graphDB.addEdges(edges);
4242
- } catch (err) {
4243
- console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
4244
- }
4245
- }
4246
- },
4247
- extractionOptions
4248
- );
4249
- }
4250
- async ask(question, history = [], namespace) {
4251
- var _a;
4252
- await this.initialize();
4253
- if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
4254
- console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
4255
- const agentReply = await this.agent.run(question, history);
4256
- return { reply: agentReply, sources: [] };
4257
- }
4258
- const stream = this.askStream(question, history, namespace);
4259
- let reply = "";
4260
- let sources = [];
4261
- let graphData;
4262
- try {
4263
- for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
4264
- const chunk = temp.value;
4265
- if (typeof chunk === "string") {
4266
- reply += chunk;
4267
- } else if ("sources" in chunk) {
4268
- sources = chunk.sources;
4269
- graphData = chunk.graphData;
4270
- }
4271
- }
4272
- } catch (temp) {
4273
- error = [temp];
4274
- } finally {
4275
- try {
4276
- more && (temp = iter.return) && await temp.call(iter);
4277
- } finally {
4278
- if (error)
4279
- throw error[0];
4280
- }
4281
- }
4282
- return { reply, sources, graphData };
4283
- }
4284
- /**
4285
- * High-performance streaming RAG flow.
4286
- * Yields text chunks first, then the retrieval metadata at the end.
4287
- */
4288
- askStream(_0) {
4289
- return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
4290
- var _a, _b, _c, _d, _e, _f, _g;
4291
- yield new __await(this.initialize());
4292
- const ns = namespace != null ? namespace : this.config.projectId;
4293
- const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
4294
- const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
4295
- try {
4296
- let searchQuery = question;
4297
- if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
4298
- searchQuery = yield new __await(this.rewriteQuery(question, history));
4299
- }
4300
- const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
4301
- const filter = QueryProcessor.buildQueryFilter(question, hints);
4302
- console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
4303
- const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
4304
- namespace: ns,
4305
- topK: topK * 2,
4306
- filter
4307
- }));
4308
- let sources = rawSources.filter((m) => m.score >= scoreThreshold);
4309
- if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
4310
- sources = yield new __await(this.reranker.rerank(sources, question, topK));
4311
- } else {
4312
- sources = sources.slice(0, topK);
4313
- }
4314
- let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
4315
- ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
4316
- if (graphData && graphData.nodes.length > 0) {
4317
- const graphContext = graphData.nodes.map(
4318
- (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
4319
- ).join("\n");
4320
- context = `GRAPH KNOWLEDGE:
4321
- ${graphContext}
4322
-
4323
- VECTOR CONTEXT:
4324
- ${context}`;
4325
- }
4326
- const messages = [...history, { role: "user", content: question }];
4327
- if (this.llmProvider.chatStream) {
4328
- const stream = this.llmProvider.chatStream(messages, context);
4329
- if (!stream) {
4330
- throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
4331
- }
4332
- try {
4333
- for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
4334
- const chunk = temp.value;
4335
- yield chunk;
4336
- }
4337
- } catch (temp) {
4338
- error = [temp];
4339
- } finally {
4340
- try {
4341
- more && (temp = iter.return) && (yield new __await(temp.call(iter)));
4342
- } finally {
4343
- if (error)
4344
- throw error[0];
4345
- }
4346
- }
4347
- } else {
4348
- const reply = yield new __await(this.llmProvider.chat(messages, context));
4349
- yield reply;
4350
- }
4351
- yield { reply: "", sources, graphData };
4352
- } catch (error2) {
4353
- throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
4354
- }
4355
- });
4356
- }
4357
- /**
4358
- * Universal retrieval method combining all enabled providers.
4359
- * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
4360
- */
4361
- async retrieve(query, options) {
4362
- var _a, _b, _c;
4363
- const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
4364
- const topK = (_b = options.topK) != null ? _b : 5;
4365
- const cacheKey = `${ns}::${query}`;
4366
- let queryVector = this.embeddingCache.get(cacheKey);
4367
- const [retrievedVector, graphData] = await Promise.all([
4368
- queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
4369
- this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
4370
- ]);
4371
- if (!queryVector) {
4372
- this.embeddingCache.set(cacheKey, retrievedVector);
4373
- queryVector = retrievedVector;
4374
- }
4375
- const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
4376
- return { sources, graphData };
4377
- }
4378
- /**
4379
- * Rewrite the user query for better retrieval performance.
4380
- */
4381
- async rewriteQuery(question, history) {
4382
- const prompt = `
4383
- Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
4384
- Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
4385
-
4386
- History:
4387
- ${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
4388
-
4389
- New Question: ${question}
4390
-
4391
- Optimized Search Query:`;
4392
- const rewrite = await this.llmProvider.chat(
4393
- [
4394
- { role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
4395
- { role: "user", content: prompt }
4396
- ],
4397
- ""
4398
- );
4399
- return rewrite.trim() || question;
4400
- }
4401
- /**
4402
- * Generate 3-5 short, relevant questions based on the vector database content.
4403
- */
4404
- async getSuggestions(query, namespace) {
4405
- if (!query || query.trim().length < 3) {
4406
- return [];
4407
- }
4408
- await this.initialize();
4409
- const ns = namespace != null ? namespace : this.config.projectId;
4410
- try {
4411
- const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
4412
- if (sources.length === 0) {
4413
- return [];
4414
- }
4415
- const context = sources.map((s) => s.content).join("\n\n---\n\n");
4416
- const prompt = `
4417
- Based on the following snippets from a document, what are 3 short, relevant questions a user might ask?
4418
- Focus on questions that can be answered by the context.
4419
- Keep each question under 10 words and make them very specific to the content.
4420
- Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
4421
-
4422
- Context:
4423
- ${context}
4424
-
4425
- Suggestions:`;
4426
- const response = await this.llmProvider.chat(
4427
- [
4428
- { role: "system", content: "You are a helpful assistant that generates search suggestions." },
4429
- { role: "user", content: prompt }
4430
- ],
4431
- ""
4432
- );
4433
- const match = response.match(/\[[\s\S]*\]/);
4434
- if (match) {
4435
- const suggestions = JSON.parse(match[0]);
4436
- if (Array.isArray(suggestions)) {
4437
- return suggestions.map((s) => String(s)).slice(0, 3);
4438
- }
4439
- }
4440
- } catch (error) {
4441
- console.error("[Pipeline] Failed to generate suggestions:", error);
4442
- }
4443
- return [];
4444
- }
4445
- };
4446
-
4447
- // src/core/ProviderHealthCheck.ts
4448
- var ProviderHealthCheck = class {
4449
- /**
4450
- * Validates vector database configuration before initialization.
4451
- */
4452
- static async checkVectorProvider(config) {
4453
- const timestamp = Date.now();
4454
- const configErrors = await ConfigValidator.validate({
4455
- projectId: "health-check",
4456
- vectorDb: config,
4457
- llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
4458
- embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
4459
- });
4460
- const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
4461
- if (vectorDbErrors.length > 0) {
4462
- return {
4463
- healthy: false,
4464
- provider: config.provider,
4465
- error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
4466
- timestamp
4467
- };
4468
- }
4469
- try {
4470
- const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
4471
- if (checker) {
4472
- return await checker.check(config);
4473
- }
4474
- return await this.fallbackVectorHealthCheck(config, timestamp);
4475
- } catch (error) {
4476
- return {
4477
- healthy: false,
4478
- provider: config.provider,
4479
- error: error instanceof Error ? error.message : String(error),
4480
- timestamp
4481
- };
4482
- }
4483
- }
4484
- static async fallbackVectorHealthCheck(config, timestamp) {
4485
- const opts = config.options || {};
4486
- const baseUrl = opts.baseUrl || opts.uri;
4487
- if (baseUrl && baseUrl.startsWith("http")) {
4488
- try {
4489
- const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
4490
- return {
4491
- healthy: response.ok,
4492
- provider: config.provider,
4493
- error: response.ok ? void 0 : `Health check failed: ${response.status}`,
4494
- timestamp
4495
- };
4496
- } catch (e) {
4497
- return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
4498
- }
4499
- }
4500
- return {
4501
- healthy: true,
4502
- provider: config.provider,
4503
- error: "Pluggable health check not implemented; basic reachability assumed.",
4504
- timestamp
4505
- };
4506
- }
4507
- /**
4508
- * Validates LLM provider configuration.
4509
- */
4510
- static async checkLLMProvider(config) {
4511
- const timestamp = Date.now();
4512
- try {
4513
- const checker = LLMFactory.getHealthChecker(config.provider);
4514
- if (checker) {
4515
- return await checker.check(config);
4516
- }
4517
- return {
4518
- healthy: true,
4519
- provider: config.provider,
4520
- error: "Pluggable health check not implemented; basic reachability assumed.",
4521
- timestamp
4522
- };
4523
- } catch (error) {
4524
- return {
4525
- healthy: false,
4526
- provider: config.provider,
4527
- error: error instanceof Error ? error.message : String(error),
4528
- timestamp
4529
- };
4530
- }
4531
- }
4532
- /**
4533
- * Runs comprehensive health checks on all configured providers in parallel.
4534
- */
4535
- static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
4536
- const [vectorDb, llm, embedding] = await Promise.all([
4537
- this.checkVectorProvider(vectorDbConfig),
4538
- this.checkLLMProvider(llmConfig),
4539
- embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
4540
- ]);
4541
- return {
4542
- vectorDb,
4543
- llm,
4544
- embedding,
4545
- allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
4546
- };
4547
- }
4548
- };
4549
-
4550
- // src/core/VectorPlugin.ts
4551
- var VectorPlugin = class {
4552
- constructor(hostConfig) {
4553
- this.config = ConfigResolver.resolve(hostConfig);
4554
- this.validationPromise = ConfigValidator.validateAndThrow(this.config);
4555
- this.pipeline = new Pipeline(this.config);
4556
- }
4557
- /**
4558
- * Get the current resolved configuration.
4559
- */
4560
- getConfig() {
4561
- return this.config;
4562
- }
4563
- /**
4564
- * Perform pre-flight health checks on all configured providers.
4565
- * Useful to verify connectivity before running operations.
4566
- *
4567
- * @returns Health status for vector DB, LLM, and embedding providers
4568
- */
4569
- async checkHealth() {
4570
- return ProviderHealthCheck.checkAll(
4571
- this.config.vectorDb,
4572
- this.config.llm,
4573
- this.config.embedding
4574
- );
4575
- }
4576
- /**
4577
- * Run a chat query.
4578
- */
4579
- async chat(message, history = [], namespace) {
4580
- await this.validationPromise;
4581
- return this.pipeline.ask(message, history, namespace);
4582
- }
4583
- /**
4584
- * Run a streaming chat query.
4585
- */
4586
- chatStream(_0) {
4587
- return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
4588
- yield new __await(this.validationPromise);
4589
- yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
4590
- });
4451
+ `;
4452
+ if (!((_a = this.config.llm.systemPrompt) == null ? void 0 : _a.includes(CHART_MARKER))) {
4453
+ let cleanPrompt = this.config.llm.systemPrompt || "";
4454
+ cleanPrompt = cleanPrompt.replace(/\n\n### UNIVERSAL UI PROTOCOL[\s\S]*?(?=\n\n|$)/g, "");
4455
+ cleanPrompt = cleanPrompt.replace(/<!-- UI_PROTOCOL_V\d+ -->[\s\S]*?(?=\n\n|$)/g, "");
4456
+ this.config.llm.systemPrompt = cleanPrompt.trim() + chartInstruction;
4457
+ }
4458
+ console.log(`[Pipeline] Initializing with provider: ${this.config.vectorDb.provider}...`);
4459
+ this.vectorDB = await ProviderRegistry.createVectorProvider(this.config.vectorDb);
4460
+ const { llmProvider, embeddingProvider } = await EmbeddingStrategyResolver.resolve(
4461
+ this.config.llm,
4462
+ this.config.embedding
4463
+ );
4464
+ this.llmProvider = llmProvider;
4465
+ this.embeddingProvider = embeddingProvider;
4466
+ if (this.config.graphDb) {
4467
+ this.graphDB = await ProviderRegistry.createGraphProvider(this.config.graphDb);
4468
+ await this.graphDB.initialize();
4469
+ this.entityExtractor = new EntityExtractor(this.llmProvider);
4470
+ }
4471
+ await this.vectorDB.initialize();
4472
+ if (((_b = this.config.rag) == null ? void 0 : _b.architecture) === "agentic") {
4473
+ this.agent = new LangChainAgent(this, this.config);
4474
+ await this.agent.initialize(this.llmProvider);
4475
+ }
4476
+ this.initialised = true;
4591
4477
  }
4592
4478
  /**
4593
- * Ingest documents into the vector database.
4479
+ * Ingest documents with automatic chunking, embedding, and batch upsert.
4480
+ * Handles retries for transient failures.
4594
4481
  */
4595
4482
  async ingest(documents, namespace) {
4596
- await this.validationPromise;
4597
- return this.pipeline.ingest(documents, namespace);
4598
- }
4599
- /**
4600
- * Get auto-suggestions based on a query prefix.
4601
- */
4602
- async getSuggestions(query, namespace) {
4603
- await this.validationPromise;
4604
- return this.pipeline.getSuggestions(query, namespace);
4605
- }
4606
- };
4607
-
4608
- // src/config/ConfigBuilder.ts
4609
- var ConfigBuilder = class {
4610
- /**
4611
- * Set the project/application ID for namespacing
4612
- */
4613
- projectId(id) {
4614
- this._projectId = id;
4615
- return this;
4616
- }
4617
- /**
4618
- * Configure the vector database provider
4619
- */
4620
- vectorDb(provider, options) {
4621
- var _a;
4622
- if (provider === "auto") {
4623
- this._vectorDb = this._autoDetectVectorDb();
4624
- } else {
4625
- this._vectorDb = {
4626
- provider,
4627
- indexName: (_a = options == null ? void 0 : options.indexName) != null ? _a : "default",
4628
- options: __spreadValues({}, options)
4629
- };
4483
+ await this.initialize();
4484
+ const ns = namespace != null ? namespace : this.config.projectId;
4485
+ const results = [];
4486
+ for (const doc of documents) {
4487
+ try {
4488
+ const chunks = await this.prepareChunks(doc);
4489
+ const vectors = await this.processEmbeddings(chunks);
4490
+ const upsertDocs = chunks.map((chunk, i) => ({
4491
+ id: chunk.id,
4492
+ vector: vectors[i],
4493
+ content: chunk.content,
4494
+ metadata: chunk.metadata
4495
+ }));
4496
+ const totalProcessed = await this.processUpserts(upsertDocs, ns);
4497
+ results.push({
4498
+ docId: doc.docId,
4499
+ chunksIngested: totalProcessed
4500
+ });
4501
+ if (this.graphDB && this.entityExtractor) {
4502
+ await this.processGraphIngestion(doc.docId, chunks);
4503
+ }
4504
+ } catch (error) {
4505
+ console.error(`[Pipeline] Failed to ingest document ${doc.docId}:`, error);
4506
+ results.push({ docId: doc.docId, chunksIngested: 0 });
4507
+ }
4630
4508
  }
4631
- return this;
4509
+ return results;
4632
4510
  }
4633
4511
  /**
4634
- * Configure the LLM provider for chat
4512
+ * Step 1: Chunk the document content.
4635
4513
  */
4636
- llm(provider, model, apiKey, options) {
4514
+ async prepareChunks(doc) {
4637
4515
  var _a, _b;
4638
- if (provider === "auto") {
4639
- this._llm = this._autoDetectLLM();
4640
- } else {
4641
- this._llm = {
4642
- provider,
4643
- model: model != null ? model : "default-model",
4644
- apiKey,
4645
- systemPrompt: options == null ? void 0 : options.systemPrompt,
4646
- maxTokens: (_a = options == null ? void 0 : options.maxTokens) != null ? _a : 1024,
4647
- temperature: (_b = options == null ? void 0 : options.temperature) != null ? _b : 0.7,
4648
- baseUrl: options == null ? void 0 : options.baseUrl,
4649
- options
4650
- };
4651
- }
4652
- return this;
4653
- }
4654
- /**
4655
- * Configure the embedding provider
4656
- */
4657
- embedding(provider, model, apiKey, options) {
4658
- if (provider === "auto") {
4659
- this._embedding = this._autoDetectEmbedding();
4660
- } else {
4661
- this._embedding = {
4662
- provider,
4663
- model: model != null ? model : "default-embedding",
4664
- apiKey,
4665
- baseUrl: options == null ? void 0 : options.baseUrl,
4666
- options
4667
- };
4516
+ if (this.llamaIngestor) {
4517
+ return await this.llamaIngestor.chunk(doc.content, {
4518
+ docId: doc.docId,
4519
+ metadata: doc.metadata,
4520
+ chunkSize: (_a = this.config.rag) == null ? void 0 : _a.chunkSize,
4521
+ chunkOverlap: (_b = this.config.rag) == null ? void 0 : _b.chunkOverlap
4522
+ });
4668
4523
  }
4669
- return this;
4670
- }
4671
- /**
4672
- * Set RAG-specific pipeline parameters
4673
- */
4674
- rag(options) {
4675
- var _a;
4676
- this._rag = __spreadValues(__spreadValues({}, (_a = this._rag) != null ? _a : {}), options);
4677
- return this;
4678
- }
4679
- /**
4680
- * Set UI branding and appearance options.
4681
- * Accepts the full UIConfig interface.
4682
- */
4683
- ui(options) {
4684
- this._ui = options;
4685
- return this;
4524
+ return this.chunker.chunk(doc.content, {
4525
+ docId: doc.docId,
4526
+ metadata: doc.metadata
4527
+ });
4686
4528
  }
4687
4529
  /**
4688
- * Build and return the final RagConfig.
4689
- * Throws if required fields (projectId, vectorDb, llm, embedding) are not set.
4530
+ * Step 2: Generate embeddings for chunks with retry logic.
4531
+ * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
4690
4532
  */
4691
- build() {
4692
- var _a;
4693
- const missing = [];
4694
- if (!this._projectId) missing.push('projectId (call .projectId("my-app"))');
4695
- if (!this._vectorDb) missing.push('vectorDb (call .vectorDb("pinecone", { ... }))');
4696
- if (!this._llm) missing.push('llm (call .llm("openai", "gpt-4o", apiKey))');
4697
- if (!this._embedding) missing.push('embedding (call .embedding("openai", "text-embedding-3-small"))');
4698
- if (missing.length > 0) {
4533
+ async processEmbeddings(chunks) {
4534
+ const embedBatchOptions = {
4535
+ batchSize: 50,
4536
+ maxRetries: 3,
4537
+ initialDelayMs: 100
4538
+ };
4539
+ const vectors = await BatchProcessor.mapWithRetry(
4540
+ chunks.map((c) => c.content),
4541
+ (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
4542
+ embedBatchOptions
4543
+ );
4544
+ if (vectors.length !== chunks.length) {
4699
4545
  throw new Error(
4700
- `[ConfigBuilder] Cannot build \u2014 required fields are missing:
4701
- ${missing.join("\n ")}`
4546
+ `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
4702
4547
  );
4703
4548
  }
4704
- return __spreadValues({
4705
- projectId: this._projectId,
4706
- vectorDb: this._vectorDb,
4707
- llm: this._llm,
4708
- embedding: this._embedding,
4709
- ui: this._rag ? this._ui : void 0,
4710
- rag: (_a = this._rag) != null ? _a : { chunkSize: 1e3, chunkOverlap: 200, topK: 5 }
4711
- }, this._ui ? { ui: this._ui } : {});
4549
+ return vectors;
4712
4550
  }
4713
4551
  /**
4714
- * Build and return as JSON for serialization
4715
- */
4716
- toJSON() {
4717
- return JSON.stringify(this.build());
4718
- }
4719
- // ============================================================================
4720
- // Private helper methods for auto-detection
4721
- // ============================================================================
4722
- _autoDetectVectorDb() {
4723
- if (process.env.PINECONE_API_KEY && process.env.PINECONE_INDEX) {
4724
- return { provider: "pinecone", indexName: process.env.PINECONE_INDEX, options: { apiKey: process.env.PINECONE_API_KEY } };
4725
- }
4726
- if (process.env.QDRANT_URL) {
4727
- return { provider: "qdrant", indexName: process.env.QDRANT_COLLECTION || "documents", options: { url: process.env.QDRANT_URL, apiKey: process.env.QDRANT_API_KEY } };
4728
- }
4729
- if (process.env.DATABASE_URL) {
4730
- return { provider: "postgresql", indexName: process.env.PG_TABLE || "documents", options: { connectionString: process.env.DATABASE_URL } };
4731
- }
4732
- if (process.env.MONGODB_URI) {
4733
- return { provider: "mongodb", indexName: process.env.MONGODB_COLLECTION || "documents", options: { uri: process.env.MONGODB_URI, database: process.env.MONGODB_DB || "ai_db", collection: process.env.MONGODB_COLLECTION || "documents" } };
4734
- }
4735
- if (process.env.REDIS_URL) {
4736
- return { provider: "redis", indexName: process.env.REDIS_INDEX || "documents", options: { url: process.env.REDIS_URL } };
4737
- }
4738
- return { provider: "qdrant", indexName: "documents", options: { url: process.env.QDRANT_URL || "http://localhost:6333" } };
4739
- }
4740
- _autoDetectLLM() {
4741
- if (process.env.OPENAI_API_KEY) {
4742
- return { provider: "openai", model: process.env.OPENAI_MODEL || "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY, maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OPENAI_TEMPERATURE || "0.7") };
4743
- }
4744
- if (process.env.ANTHROPIC_API_KEY) {
4745
- return { provider: "anthropic", model: process.env.ANTHROPIC_MODEL || "claude-3-haiku-20240307", apiKey: process.env.ANTHROPIC_API_KEY, maxTokens: parseInt(process.env.ANTHROPIC_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.ANTHROPIC_TEMPERATURE || "0.7") };
4746
- }
4747
- if (process.env.OLLAMA_BASE_URL) {
4748
- return { provider: "ollama", model: process.env.OLLAMA_MODEL || "mistral", baseUrl: process.env.OLLAMA_BASE_URL, maxTokens: parseInt(process.env.OLLAMA_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OLLAMA_TEMPERATURE || "0.7") };
4749
- }
4750
- return { provider: "openai", model: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY };
4751
- }
4752
- _autoDetectEmbedding() {
4753
- if (process.env.EMBEDDING_PROVIDER) {
4754
- return { provider: process.env.EMBEDDING_PROVIDER, model: process.env.EMBEDDING_MODEL || "default", apiKey: process.env.EMBEDDING_API_KEY, baseUrl: process.env.EMBEDDING_BASE_URL };
4755
- }
4756
- return { provider: "openai", model: "text-embedding-3-small", apiKey: process.env.OPENAI_API_KEY };
4757
- }
4758
- };
4759
- var PRESETS = {
4760
- /** OpenAI + Pinecone: Production-ready cloud setup */
4761
- "openai-pinecone": { vectorDb: "pinecone", llm: "openai", embedding: "openai" },
4762
- /** Claude + Qdrant: Open-source vector DB + proprietary LLM */
4763
- "claude-qdrant": { vectorDb: "qdrant", llm: "anthropic", embedding: "openai" },
4764
- /** Local development: Ollama + local Qdrant */
4765
- "local-dev": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
4766
- /** Fully open-source: Ollama LLM + Qdrant + Ollama embeddings */
4767
- "fully-open-source": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
4768
- /** PostgreSQL stack: pgvector + OpenAI */
4769
- "postgres-openai": { vectorDb: "postgresql", llm: "openai", embedding: "openai" },
4770
- /** Enterprise MongoDB: MongoDB Atlas with OpenAI */
4771
- "mongodb-openai": { vectorDb: "mongodb", llm: "openai", embedding: "openai" },
4772
- /** Redis stack for caching + search */
4773
- "redis-openai": { vectorDb: "redis", llm: "openai", embedding: "openai" }
4774
- };
4775
- function createFromPreset(presetName) {
4776
- const preset = PRESETS[presetName];
4777
- return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
4778
- }
4779
-
4780
- // src/utils/DocumentParser.ts
4781
- var DocumentParser = class {
4552
+ * Step 3: Upsert chunks to vector database with retry logic.
4553
+ */
4554
+ async processUpserts(upsertDocs, namespace) {
4555
+ const upsertBatchOptions = {
4556
+ batchSize: 100,
4557
+ maxRetries: 3,
4558
+ initialDelayMs: 100
4559
+ };
4560
+ const upsertResult = await BatchProcessor.processBatch(
4561
+ upsertDocs,
4562
+ (batch) => this.vectorDB.batchUpsert(batch, namespace),
4563
+ upsertBatchOptions
4564
+ );
4565
+ if (upsertResult.errors.length > 0) {
4566
+ console.warn(`[Pipeline] Ingestion: ${upsertResult.errors.length} batches failed`);
4567
+ }
4568
+ return upsertResult.totalProcessed;
4569
+ }
4782
4570
  /**
4783
- * Extract text from a File or Buffer based on its type.
4571
+ * Step 4: Optional graph-based entity extraction and ingestion.
4784
4572
  */
4785
- static async parse(file, fileName, mimeType) {
4573
+ async processGraphIngestion(docId, chunks) {
4574
+ console.log(`[Pipeline] Extracting entities for doc ${docId} (${chunks.length} chunks)...`);
4575
+ const extractionOptions = {
4576
+ batchSize: 2,
4577
+ // Low concurrency for LLM extraction
4578
+ maxRetries: 1,
4579
+ initialDelayMs: 500
4580
+ };
4581
+ await BatchProcessor.processBatch(
4582
+ chunks,
4583
+ async (batch) => {
4584
+ for (const chunk of batch) {
4585
+ try {
4586
+ const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
4587
+ if (nodes.length > 0) await this.graphDB.addNodes(nodes);
4588
+ if (edges.length > 0) await this.graphDB.addEdges(edges);
4589
+ } catch (err) {
4590
+ console.warn(`[Pipeline] Entity extraction failed for chunk:`, err);
4591
+ }
4592
+ }
4593
+ },
4594
+ extractionOptions
4595
+ );
4596
+ }
4597
+ async ask(question, history = [], namespace) {
4786
4598
  var _a;
4787
- const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
4788
- if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
4789
- return this.readAsText(file);
4790
- }
4791
- if (extension === "json" || mimeType === "application/json") {
4792
- const text = await this.readAsText(file);
4793
- try {
4794
- const obj = JSON.parse(text);
4795
- return JSON.stringify(obj, null, 2);
4796
- } catch (e) {
4797
- return text;
4798
- }
4799
- }
4800
- if (extension === "csv" || mimeType === "text/csv") {
4801
- return this.readAsText(file);
4599
+ await this.initialize();
4600
+ if (((_a = this.config.rag) == null ? void 0 : _a.architecture) === "agentic" && this.agent) {
4601
+ console.log("[Pipeline] \u{1F916} Executing in Agentic Mode...");
4602
+ const agentReply = await this.agent.run(question, history);
4603
+ return { reply: agentReply, sources: [] };
4802
4604
  }
4803
- if (extension === "pdf" || mimeType === "application/pdf") {
4804
- try {
4805
- const pdf = await import("pdf-parse");
4806
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
4807
- const data = await pdf.default(buffer);
4808
- return data.text;
4809
- } catch (err) {
4810
- console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
4811
- return `[PDF Parsing Error for ${fileName}]`;
4605
+ const stream = this.askStream(question, history, namespace);
4606
+ let reply = "";
4607
+ let sources = [];
4608
+ let graphData;
4609
+ try {
4610
+ for (var iter = __forAwait(stream), more, temp, error; more = !(temp = await iter.next()).done; more = false) {
4611
+ const chunk = temp.value;
4612
+ if (typeof chunk === "string") {
4613
+ reply += chunk;
4614
+ } else if ("sources" in chunk) {
4615
+ sources = chunk.sources;
4616
+ graphData = chunk.graphData;
4617
+ }
4812
4618
  }
4813
- }
4814
- if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
4619
+ } catch (temp) {
4620
+ error = [temp];
4621
+ } finally {
4815
4622
  try {
4816
- const mammoth = await import("mammoth");
4817
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
4818
- const result = await mammoth.extractRawText({ buffer });
4819
- return result.value;
4820
- } catch (err) {
4821
- console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
4822
- return `[DOCX Parsing Error for ${fileName}]`;
4623
+ more && (temp = iter.return) && await temp.call(iter);
4624
+ } finally {
4625
+ if (error)
4626
+ throw error[0];
4823
4627
  }
4824
4628
  }
4825
- try {
4826
- return await this.readAsText(file);
4827
- } catch (e) {
4828
- throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
4829
- }
4830
- }
4831
- static async readAsText(file) {
4832
- if (Buffer.isBuffer(file)) {
4833
- return file.toString("utf-8");
4834
- }
4835
- return await file.text();
4629
+ return { reply, sources, graphData };
4836
4630
  }
4837
- };
4838
-
4839
- // src/server.ts
4840
- init_BaseVectorProvider();
4841
- init_PineconeProvider();
4842
- init_PostgreSQLProvider();
4631
+ /**
4632
+ * High-performance streaming RAG flow.
4633
+ * Yields text chunks first, then the retrieval metadata at the end.
4634
+ */
4635
+ askStream(_0) {
4636
+ return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
4637
+ var _a, _b, _c, _d, _e, _f, _g;
4638
+ yield new __await(this.initialize());
4639
+ const ns = namespace != null ? namespace : this.config.projectId;
4640
+ const topK = (_b = (_a = this.config.rag) == null ? void 0 : _a.topK) != null ? _b : 5;
4641
+ const scoreThreshold = (_d = (_c = this.config.rag) == null ? void 0 : _c.scoreThreshold) != null ? _d : 0;
4642
+ try {
4643
+ let searchQuery = question;
4644
+ if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
4645
+ searchQuery = yield new __await(this.rewriteQuery(question, history));
4646
+ }
4647
+ const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
4648
+ const filter = QueryProcessor.buildQueryFilter(question, hints);
4649
+ console.log(`[Pipeline] \u{1F9E9} Extracted filters:`, JSON.stringify(filter));
4650
+ const { sources: rawSources, graphData } = yield new __await(this.retrieve(searchQuery, {
4651
+ namespace: ns,
4652
+ topK: topK * 2,
4653
+ filter
4654
+ }));
4655
+ let sources = rawSources.filter((m) => m.score >= scoreThreshold);
4656
+ if ((_g = this.config.rag) == null ? void 0 : _g.useReranking) {
4657
+ sources = yield new __await(this.reranker.rerank(sources, question, topK));
4658
+ } else {
4659
+ sources = sources.slice(0, topK);
4660
+ }
4661
+ let context = sources.length ? sources.map((m, i) => `[Source ${i + 1}]
4662
+ ${m.content}`).join("\n\n---\n\n") : "No relevant context found.";
4663
+ if (graphData && graphData.nodes.length > 0) {
4664
+ const graphContext = graphData.nodes.map(
4665
+ (n) => `Entity: ${n.label} (${n.id})${n.properties ? " - " + JSON.stringify(n.properties) : ""}`
4666
+ ).join("\n");
4667
+ context = `GRAPH KNOWLEDGE:
4668
+ ${graphContext}
4843
4669
 
4844
- // src/providers/vectordb/MultiTablePostgresProvider.ts
4845
- var import_pg2 = require("pg");
4846
- init_BaseVectorProvider();
4847
- var MultiTablePostgresProvider = class extends BaseVectorProvider {
4848
- constructor(config) {
4849
- var _a, _b, _c, _d, _e;
4850
- super(config);
4851
- const opts = config.options || {};
4852
- if (!opts.connectionString) {
4853
- throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
4854
- }
4855
- this.connectionString = opts.connectionString;
4856
- this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
4857
- const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
4858
- this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
4859
- if (this.tables.length === 0) {
4860
- console.warn(
4861
- "[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
4862
- );
4863
- }
4864
- const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
4865
- this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
4866
- }
4867
- async initialize() {
4868
- this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
4869
- const client = await this.pool.connect();
4870
- try {
4871
- const res = await client.query(`
4872
- SELECT table_name
4873
- FROM information_schema.columns
4874
- WHERE column_name = 'embedding'
4875
- AND table_schema = 'public'
4876
- `);
4877
- const discoveredTables = res.rows.map((r) => r.table_name);
4878
- if (this.tables.length === 0) {
4879
- this.tables = discoveredTables;
4880
- } else {
4881
- const staticTables = [...this.tables];
4882
- this.tables = staticTables.filter((t) => discoveredTables.includes(t));
4883
- if (this.tables.length < staticTables.length) {
4884
- const missing = staticTables.filter((t) => !discoveredTables.includes(t));
4885
- console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
4670
+ VECTOR CONTEXT:
4671
+ ${context}`;
4886
4672
  }
4673
+ const messages = [...history, { role: "user", content: question }];
4674
+ if (this.llmProvider.chatStream) {
4675
+ const stream = this.llmProvider.chatStream(messages, context);
4676
+ if (!stream) {
4677
+ throw new Error(`[Pipeline] ${this.config.llm.provider} chatStream returned undefined`);
4678
+ }
4679
+ try {
4680
+ for (var iter = __forAwait(stream), more, temp, error; more = !(temp = yield new __await(iter.next())).done; more = false) {
4681
+ const chunk = temp.value;
4682
+ yield chunk;
4683
+ }
4684
+ } catch (temp) {
4685
+ error = [temp];
4686
+ } finally {
4687
+ try {
4688
+ more && (temp = iter.return) && (yield new __await(temp.call(iter)));
4689
+ } finally {
4690
+ if (error)
4691
+ throw error[0];
4692
+ }
4693
+ }
4694
+ } else {
4695
+ const reply = yield new __await(this.llmProvider.chat(messages, context));
4696
+ yield reply;
4697
+ }
4698
+ const uiTransformation = UITransformer.transform(question, sources);
4699
+ yield {
4700
+ reply: "",
4701
+ sources,
4702
+ graphData,
4703
+ ui_transformation: uiTransformation
4704
+ };
4705
+ } catch (error2) {
4706
+ throw new Error(`[Pipeline] Stream failed: ${error2 instanceof Error ? error2.message : String(error2)}`);
4887
4707
  }
4888
- if (this.tables.length === 0) {
4889
- console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
4890
- } else {
4891
- console.log(
4892
- `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(", ")}`
4893
- );
4894
- }
4895
- } finally {
4896
- client.release();
4708
+ });
4709
+ }
4710
+ /**
4711
+ * Universal retrieval method combining all enabled providers.
4712
+ * Uses an LRU-bounded embedding cache to avoid re-embedding the same query.
4713
+ */
4714
+ async retrieve(query, options) {
4715
+ var _a, _b, _c;
4716
+ const ns = (_a = options.namespace) != null ? _a : this.config.projectId;
4717
+ const topK = (_b = options.topK) != null ? _b : 5;
4718
+ const cacheKey = `${ns}::${query}`;
4719
+ let queryVector = this.embeddingCache.get(cacheKey);
4720
+ const [retrievedVector, graphData] = await Promise.all([
4721
+ queryVector ? Promise.resolve(queryVector) : this.embeddingProvider.embed(query, { taskType: "query" }),
4722
+ this.graphDB && ((_c = this.config.rag) == null ? void 0 : _c.useGraphRetrieval) ? this.graphDB.query(query) : Promise.resolve(void 0)
4723
+ ]);
4724
+ if (!queryVector) {
4725
+ this.embeddingCache.set(cacheKey, retrievedVector);
4726
+ queryVector = retrievedVector;
4897
4727
  }
4728
+ const sources = await this.vectorDB.query(queryVector, topK, ns, options.filter);
4729
+ return { sources, graphData };
4898
4730
  }
4899
4731
  /**
4900
- * Upsert is not supported for MultiTablePostgresProvider as it's designed for
4901
- * searching across pre-existing tables with varying schemas.
4902
- */
4903
- async upsert(_doc, _namespace) {
4904
- throw new Error(
4905
- "[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
4906
- );
4907
- }
4908
- /**
4909
- * Batch upsert is not supported for MultiTablePostgresProvider.
4732
+ * Rewrite the user query for better retrieval performance.
4910
4733
  */
4911
- async batchUpsert(_docs, _namespace) {
4912
- throw new Error(
4913
- "[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
4734
+ async rewriteQuery(question, history) {
4735
+ const prompt = `
4736
+ Given the following conversation history and a new question, rewrite the question to be a better search query for a vector database.
4737
+ Focus on extracting the core intent and entities. Do not answer the question, just rewrite it.
4738
+
4739
+ History:
4740
+ ${history.map((m) => `${m.role}: ${m.content}`).join("\n")}
4741
+
4742
+ New Question: ${question}
4743
+
4744
+ Optimized Search Query:`;
4745
+ const rewrite = await this.llmProvider.chat(
4746
+ [
4747
+ { role: "system", content: "You are an assistant that optimizes search queries for RAG systems." },
4748
+ { role: "user", content: prompt }
4749
+ ],
4750
+ ""
4914
4751
  );
4752
+ return rewrite.trim() || question;
4915
4753
  }
4916
4754
  /**
4917
- * Query all configured tables and merge results, sorted by cosine similarity score.
4755
+ * Generate 3-5 short, relevant questions based on the vector database content.
4918
4756
  */
4919
- async query(vector, topK, _namespace, _filter) {
4920
- var _a, _b, _c;
4921
- if (!this.pool) {
4922
- throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
4757
+ async getSuggestions(query, namespace) {
4758
+ if (!query || query.trim().length < 3) {
4759
+ return [];
4923
4760
  }
4924
- const vectorLiteral = `[${vector.join(",")}]`;
4925
- const allResults = [];
4926
- console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
4927
- const queryText = _filter == null ? void 0 : _filter.queryText;
4928
- const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter(
4929
- (hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
4930
- ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
4931
- console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
4932
- console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
4933
- const getDynamicKeywordQuery = () => {
4934
- if (entityHints.length > 0) {
4935
- return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
4936
- }
4937
- if (queryText) {
4938
- return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
4761
+ await this.initialize();
4762
+ const ns = namespace != null ? namespace : this.config.projectId;
4763
+ try {
4764
+ const { sources } = await this.retrieve(query, { namespace: ns, topK: 3 });
4765
+ if (sources.length === 0) {
4766
+ return [];
4939
4767
  }
4940
- return "";
4941
- };
4942
- const dynamicKeywordQuery = getDynamicKeywordQuery();
4943
- const queryPromises = this.tables.map(async (table) => {
4944
- try {
4945
- let sqlQuery = "";
4946
- let params = [];
4947
- if (queryText) {
4948
- const hasEntityHints = entityHints.length > 0;
4949
- const exactNameScoreExpr = hasEntityHints ? `+ (
4950
- SELECT COALESCE(MAX(
4951
- CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
4952
- THEN 1.0 ELSE 0.0 END
4953
- ), 0)
4954
- FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
4955
- WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
4956
- ) * 3.0` : "";
4957
- sqlQuery = `
4958
- SELECT *,
4959
- (1 - (embedding <=> $1::vector)) AS vector_score,
4960
- COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
4961
- ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
4962
- FROM "${table}" t
4963
- ORDER BY hybrid_score DESC
4964
- LIMIT 50
4965
- `;
4966
- params = [vectorLiteral, dynamicKeywordQuery];
4967
- } else {
4968
- sqlQuery = `
4969
- SELECT *,
4970
- (1 - (embedding <=> $1::vector)) AS hybrid_score
4971
- FROM "${table}" t
4972
- ORDER BY hybrid_score DESC
4973
- LIMIT 50
4974
- `;
4975
- params = [vectorLiteral];
4976
- }
4977
- const result = await this.pool.query(sqlQuery, params);
4978
- if (result.rowCount && result.rowCount > 0) {
4979
- console.log(` \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
4980
- } else {
4981
- console.log(` \u26AA Table "${table}": No relevant matches found.`);
4982
- }
4983
- const tableResults = [];
4984
- for (const row of result.rows) {
4985
- const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
4986
- delete rest.embedding;
4987
- delete rest.vector_score;
4988
- delete rest.keyword_score;
4989
- delete rest.exact_name_score;
4990
- const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
4991
- ` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
4992
- tableResults.push({
4993
- id: `${table}-${id}`,
4994
- score: parseFloat(String(hybrid_score)),
4995
- content,
4996
- metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
4997
- });
4768
+ const context = sources.map((s) => s.content).join("\n\n---\n\n");
4769
+ const prompt = `
4770
+ Based on the following snippets from a document, what are 3 short, relevant questions a user might ask?
4771
+ Focus on questions that can be answered by the context.
4772
+ Keep each question under 10 words and make them very specific to the content.
4773
+ Return ONLY a JSON array of strings like ["Question 1", "Question 2", "Question 3"].
4774
+
4775
+ Context:
4776
+ ${context}
4777
+
4778
+ Suggestions:`;
4779
+ const response = await this.llmProvider.chat(
4780
+ [
4781
+ { role: "system", content: "You are a helpful assistant that generates search suggestions." },
4782
+ { role: "user", content: prompt }
4783
+ ],
4784
+ ""
4785
+ );
4786
+ const match = response.match(/\[[\s\S]*\]/);
4787
+ if (match) {
4788
+ const suggestions = JSON.parse(match[0]);
4789
+ if (Array.isArray(suggestions)) {
4790
+ return suggestions.map((s) => String(s)).slice(0, 3);
4998
4791
  }
4999
- return tableResults;
5000
- } catch (err) {
5001
- console.error(
5002
- `[MultiTablePostgresProvider] Error querying table "${table}":`,
5003
- err.message
5004
- );
5005
- return [];
5006
4792
  }
5007
- });
5008
- const resultsArray = await Promise.all(queryPromises);
5009
- for (const tableResults of resultsArray) {
5010
- allResults.push(...tableResults);
5011
- }
5012
- const resultsByTable = {};
5013
- for (const res of allResults) {
5014
- const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
5015
- if (!resultsByTable[table]) resultsByTable[table] = [];
5016
- resultsByTable[table].push(res);
4793
+ } catch (error) {
4794
+ console.error("[Pipeline] Failed to generate suggestions:", error);
5017
4795
  }
5018
- const balancedResults = [];
5019
- const tables = Object.keys(resultsByTable);
5020
- for (const table of tables) {
5021
- balancedResults.push(...resultsByTable[table].slice(0, 3));
4796
+ return [];
4797
+ }
4798
+ };
4799
+
4800
+ // src/core/ProviderHealthCheck.ts
4801
+ var ProviderHealthCheck = class {
4802
+ /**
4803
+ * Validates vector database configuration before initialization.
4804
+ */
4805
+ static async checkVectorProvider(config) {
4806
+ const timestamp = Date.now();
4807
+ const configErrors = await ConfigValidator.validate({
4808
+ projectId: "health-check",
4809
+ vectorDb: config,
4810
+ llm: { provider: "openai", model: "gpt-4o", apiKey: "none" },
4811
+ embedding: { provider: "openai", model: "text-embedding-3-small", apiKey: "none" }
4812
+ });
4813
+ const vectorDbErrors = configErrors.filter((e) => e.field.startsWith("vectorDb"));
4814
+ if (vectorDbErrors.length > 0) {
4815
+ return {
4816
+ healthy: false,
4817
+ provider: config.provider,
4818
+ error: `Configuration validation failed: ${vectorDbErrors.map((e) => e.message).join("; ")}`,
4819
+ timestamp
4820
+ };
5022
4821
  }
5023
- const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
5024
- if (finalSorted.length > 0) {
5025
- console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
5026
- console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
4822
+ try {
4823
+ const checker = await ProviderRegistry.getVectorHealthChecker(config.provider);
4824
+ if (checker) {
4825
+ return await checker.check(config);
4826
+ }
4827
+ return await this.fallbackVectorHealthCheck(config, timestamp);
4828
+ } catch (error) {
4829
+ return {
4830
+ healthy: false,
4831
+ provider: config.provider,
4832
+ error: error instanceof Error ? error.message : String(error),
4833
+ timestamp
4834
+ };
5027
4835
  }
5028
- return finalSorted.slice(0, Math.max(topK, 15));
5029
- }
5030
- async delete(_id, _namespace) {
5031
- console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
5032
4836
  }
5033
- async deleteNamespace(_namespace) {
5034
- console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
4837
+ static async fallbackVectorHealthCheck(config, timestamp) {
4838
+ const opts = config.options || {};
4839
+ const baseUrl = opts.baseUrl || opts.uri;
4840
+ if (baseUrl && baseUrl.startsWith("http")) {
4841
+ try {
4842
+ const response = await fetch(`${baseUrl.replace(/\/$/, "")}/health`);
4843
+ return {
4844
+ healthy: response.ok,
4845
+ provider: config.provider,
4846
+ error: response.ok ? void 0 : `Health check failed: ${response.status}`,
4847
+ timestamp
4848
+ };
4849
+ } catch (e) {
4850
+ return { healthy: false, provider: config.provider, error: "Provider unreachable", timestamp };
4851
+ }
4852
+ }
4853
+ return {
4854
+ healthy: true,
4855
+ provider: config.provider,
4856
+ error: "Pluggable health check not implemented; basic reachability assumed.",
4857
+ timestamp
4858
+ };
5035
4859
  }
5036
- async ping() {
4860
+ /**
4861
+ * Validates LLM provider configuration.
4862
+ */
4863
+ static async checkLLMProvider(config) {
4864
+ const timestamp = Date.now();
5037
4865
  try {
5038
- await this.pool.query("SELECT 1");
5039
- return true;
5040
- } catch (e) {
5041
- return false;
4866
+ const checker = LLMFactory.getHealthChecker(config.provider);
4867
+ if (checker) {
4868
+ return await checker.check(config);
4869
+ }
4870
+ return {
4871
+ healthy: true,
4872
+ provider: config.provider,
4873
+ error: "Pluggable health check not implemented; basic reachability assumed.",
4874
+ timestamp
4875
+ };
4876
+ } catch (error) {
4877
+ return {
4878
+ healthy: false,
4879
+ provider: config.provider,
4880
+ error: error instanceof Error ? error.message : String(error),
4881
+ timestamp
4882
+ };
5042
4883
  }
5043
4884
  }
5044
- async disconnect() {
5045
- if (this.pool) {
5046
- await this.pool.end();
5047
- }
4885
+ /**
4886
+ * Runs comprehensive health checks on all configured providers in parallel.
4887
+ */
4888
+ static async checkAll(vectorDbConfig, llmConfig, embeddingConfig) {
4889
+ const [vectorDb, llm, embedding] = await Promise.all([
4890
+ this.checkVectorProvider(vectorDbConfig),
4891
+ this.checkLLMProvider(llmConfig),
4892
+ embeddingConfig ? this.checkLLMProvider(embeddingConfig) : Promise.resolve(void 0)
4893
+ ]);
4894
+ return {
4895
+ vectorDb,
4896
+ llm,
4897
+ embedding,
4898
+ allHealthy: vectorDb.healthy && llm.healthy && (!embedding || embedding.healthy)
4899
+ };
5048
4900
  }
5049
4901
  };
5050
4902
 
5051
- // src/server.ts
5052
- init_MongoDBProvider();
5053
- init_MilvusProvider();
5054
- init_QdrantProvider();
5055
- init_ChromaDBProvider();
5056
- init_RedisProvider();
5057
- init_WeaviateProvider();
5058
- init_UniversalVectorProvider();
5059
-
5060
- // src/handlers/index.ts
5061
- var import_server = require("next/server");
5062
-
5063
- // src/utils/UITransformer.ts
5064
- var UITransformer = class {
4903
+ // src/core/VectorPlugin.ts
4904
+ var VectorPlugin = class {
4905
+ constructor(hostConfig) {
4906
+ this.config = ConfigResolver.resolve(hostConfig);
4907
+ this.validationPromise = ConfigValidator.validateAndThrow(this.config);
4908
+ this.pipeline = new Pipeline(this.config);
4909
+ }
5065
4910
  /**
5066
- * Main transformation method
5067
- * Analyzes user query and retrieved data to determine the best visualization format
5068
- *
5069
- * @param userQuery - The original user question/query
5070
- * @param retrievedData - Vector database retrieval results
5071
- * @returns Structured JSON response for UI rendering
4911
+ * Get the current resolved configuration.
5072
4912
  */
5073
- static transform(userQuery, retrievedData) {
5074
- if (!retrievedData || retrievedData.length === 0) {
5075
- return this.createTextResponse("No data available", "No relevant data found for your query.");
5076
- }
5077
- const analysis = this.analyzeData(userQuery, retrievedData);
5078
- switch (analysis.suggestedType) {
5079
- case "product_carousel":
5080
- return this.transformToProductCarousel(retrievedData);
5081
- case "pie_chart":
5082
- return this.transformToPieChart(retrievedData);
5083
- case "bar_chart":
5084
- return this.transformToBarChart(retrievedData);
5085
- case "line_chart":
5086
- return this.transformToLineChart(retrievedData);
5087
- case "table":
5088
- return this.transformToTable(retrievedData);
5089
- default:
5090
- return this.transformToText(retrievedData);
5091
- }
4913
+ getConfig() {
4914
+ return this.config;
5092
4915
  }
5093
4916
  /**
5094
- * Analyzes the data and query to suggest the best visualization type
4917
+ * Perform pre-flight health checks on all configured providers.
4918
+ * Useful to verify connectivity before running operations.
4919
+ *
4920
+ * @returns Health status for vector DB, LLM, and embedding providers
5095
4921
  */
5096
- static analyzeData(query, data) {
5097
- const queryLower = query.toLowerCase();
5098
- const hasProducts = data.some(
5099
- (item) => this.isProductData(item)
5100
- );
5101
- const hasTimeSeries = data.some(
5102
- (item) => this.isTimeSeriesData(item)
4922
+ async checkHealth() {
4923
+ return ProviderHealthCheck.checkAll(
4924
+ this.config.vectorDb,
4925
+ this.config.llm,
4926
+ this.config.embedding
5103
4927
  );
5104
- const hasCategories = this.detectCategories(data).length > 0;
5105
- const isDistributionQuery = queryLower.includes("distribution") || queryLower.includes("breakdown") || queryLower.includes("percentage") || queryLower.includes("proportion");
5106
- const isComparisonQuery = queryLower.includes("compare") || queryLower.includes("vs") || queryLower.includes("difference");
5107
- const isTrendQuery = queryLower.includes("trend") || queryLower.includes("over time") || queryLower.includes("growth") || queryLower.includes("decline");
5108
- if (hasProducts && !hasTimeSeries && !isDistributionQuery) {
5109
- return { suggestedType: "product_carousel", confidence: 0.95, hasProducts, hasTimeSeries, hasCategories };
5110
- }
5111
- if (isTrendQuery && hasTimeSeries) {
5112
- return { suggestedType: "line_chart", confidence: 0.9, hasProducts, hasTimeSeries, hasCategories };
5113
- }
5114
- if (isDistributionQuery && hasCategories) {
5115
- return { suggestedType: "pie_chart", confidence: 0.85, hasProducts, hasTimeSeries, hasCategories };
5116
- }
5117
- if (isComparisonQuery && hasCategories && data.length > 2) {
5118
- return { suggestedType: "bar_chart", confidence: 0.8, hasProducts, hasTimeSeries, hasCategories };
5119
- }
5120
- if (data.length > 5 && this.hasMultipleFields(data)) {
5121
- return { suggestedType: "table", confidence: 0.75, hasProducts, hasTimeSeries, hasCategories };
5122
- }
5123
- return { suggestedType: "text", confidence: 0.5, hasProducts, hasTimeSeries, hasCategories };
5124
4928
  }
5125
4929
  /**
5126
- * Transform data to product carousel format
4930
+ * Run a chat query.
5127
4931
  */
5128
- static transformToProductCarousel(data) {
5129
- const products = data.filter((item) => this.isProductData(item)).map((item) => this.extractProductInfo(item)).filter((p) => p !== null);
5130
- return {
5131
- type: "product_carousel",
5132
- title: "Recommended Products",
5133
- description: `Found ${products.length} relevant products`,
5134
- data: products
5135
- };
4932
+ async chat(message, history = [], namespace) {
4933
+ await this.validationPromise;
4934
+ return this.pipeline.ask(message, history, namespace);
5136
4935
  }
5137
4936
  /**
5138
- * Transform data to pie chart format
4937
+ * Run a streaming chat query.
5139
4938
  */
5140
- static transformToPieChart(data) {
5141
- const categories = this.detectCategories(data);
5142
- const categoryData = this.aggregateByCategory(data, categories);
5143
- const pieData = Object.entries(categoryData).map(([label, count]) => ({
5144
- label,
5145
- value: count,
5146
- inStock: this.checkStockStatus(label, data)
5147
- }));
5148
- return {
5149
- type: "pie_chart",
5150
- title: "Distribution by Category",
5151
- description: `Showing breakdown across ${categories.length} categories`,
5152
- data: pieData
5153
- };
4939
+ chatStream(_0) {
4940
+ return __asyncGenerator(this, arguments, function* (message, history = [], namespace) {
4941
+ yield new __await(this.validationPromise);
4942
+ yield* __yieldStar(this.pipeline.askStream(message, history, namespace));
4943
+ });
5154
4944
  }
5155
4945
  /**
5156
- * Transform data to bar chart format
4946
+ * Ingest documents into the vector database.
5157
4947
  */
5158
- static transformToBarChart(data) {
5159
- const categories = this.detectCategories(data);
5160
- const categoryData = this.aggregateByCategory(data, categories);
5161
- const barData = Object.entries(categoryData).map(([category, value]) => ({
5162
- category,
5163
- value,
5164
- inStock: this.checkStockStatus(category, data)
5165
- }));
5166
- return {
5167
- type: "bar_chart",
5168
- title: "Comparison by Category",
5169
- description: `Comparing ${categories.length} categories`,
5170
- data: barData
5171
- };
4948
+ async ingest(documents, namespace) {
4949
+ await this.validationPromise;
4950
+ return this.pipeline.ingest(documents, namespace);
5172
4951
  }
5173
4952
  /**
5174
- * Transform data to line chart format
4953
+ * Get auto-suggestions based on a query prefix.
5175
4954
  */
5176
- static transformToLineChart(data) {
5177
- const timePoints = this.extractTimeSeriesData(data);
5178
- const lineData = timePoints.map((point) => ({
5179
- timestamp: point.timestamp,
5180
- value: point.value,
5181
- label: point.label
5182
- }));
5183
- return {
5184
- type: "line_chart",
5185
- title: "Trend Over Time",
5186
- description: `Showing ${lineData.length} data points`,
5187
- data: lineData
5188
- };
4955
+ async getSuggestions(query, namespace) {
4956
+ await this.validationPromise;
4957
+ return this.pipeline.getSuggestions(query, namespace);
4958
+ }
4959
+ };
4960
+
4961
+ // src/config/ConfigBuilder.ts
4962
+ var ConfigBuilder = class {
4963
+ /**
4964
+ * Set the project/application ID for namespacing
4965
+ */
4966
+ projectId(id) {
4967
+ this._projectId = id;
4968
+ return this;
5189
4969
  }
5190
4970
  /**
5191
- * Transform data to table format
4971
+ * Configure the vector database provider
5192
4972
  */
5193
- static transformToTable(data) {
5194
- const columns = this.extractTableColumns(data);
5195
- const rows = data.map((item) => this.extractTableRow(item, columns));
5196
- const tableData = {
5197
- columns,
5198
- rows
5199
- };
5200
- return {
5201
- type: "table",
5202
- title: "Detailed Results",
5203
- description: `Showing ${data.length} results`,
5204
- data: tableData
5205
- };
4973
+ vectorDb(provider, options) {
4974
+ var _a;
4975
+ if (provider === "auto") {
4976
+ this._vectorDb = this._autoDetectVectorDb();
4977
+ } else {
4978
+ this._vectorDb = {
4979
+ provider,
4980
+ indexName: (_a = options == null ? void 0 : options.indexName) != null ? _a : "default",
4981
+ options: __spreadValues({}, options)
4982
+ };
4983
+ }
4984
+ return this;
4985
+ }
4986
+ /**
4987
+ * Configure the LLM provider for chat
4988
+ */
4989
+ llm(provider, model, apiKey, options) {
4990
+ var _a, _b;
4991
+ if (provider === "auto") {
4992
+ this._llm = this._autoDetectLLM();
4993
+ } else {
4994
+ this._llm = {
4995
+ provider,
4996
+ model: model != null ? model : "default-model",
4997
+ apiKey,
4998
+ systemPrompt: options == null ? void 0 : options.systemPrompt,
4999
+ maxTokens: (_a = options == null ? void 0 : options.maxTokens) != null ? _a : 1024,
5000
+ temperature: (_b = options == null ? void 0 : options.temperature) != null ? _b : 0.7,
5001
+ baseUrl: options == null ? void 0 : options.baseUrl,
5002
+ options
5003
+ };
5004
+ }
5005
+ return this;
5006
+ }
5007
+ /**
5008
+ * Configure the embedding provider
5009
+ */
5010
+ embedding(provider, model, apiKey, options) {
5011
+ if (provider === "auto") {
5012
+ this._embedding = this._autoDetectEmbedding();
5013
+ } else {
5014
+ this._embedding = {
5015
+ provider,
5016
+ model: model != null ? model : "default-embedding",
5017
+ apiKey,
5018
+ baseUrl: options == null ? void 0 : options.baseUrl,
5019
+ options
5020
+ };
5021
+ }
5022
+ return this;
5206
5023
  }
5207
5024
  /**
5208
- * Transform data to text format (fallback)
5025
+ * Set RAG-specific pipeline parameters
5209
5026
  */
5210
- static transformToText(data) {
5211
- const textContent = data.map((item) => item.content).join("\n\n");
5212
- return this.createTextResponse(
5213
- "Search Results",
5214
- textContent,
5215
- `Found ${data.length} relevant results`
5216
- );
5027
+ rag(options) {
5028
+ var _a;
5029
+ this._rag = __spreadValues(__spreadValues({}, (_a = this._rag) != null ? _a : {}), options);
5030
+ return this;
5217
5031
  }
5218
5032
  /**
5219
- * Helper: Create text response
5033
+ * Set UI branding and appearance options.
5034
+ * Accepts the full UIConfig interface.
5220
5035
  */
5221
- static createTextResponse(title, content, description) {
5222
- return {
5223
- type: "text",
5224
- title,
5225
- description,
5226
- data: { content }
5227
- };
5036
+ ui(options) {
5037
+ this._ui = options;
5038
+ return this;
5228
5039
  }
5229
5040
  /**
5230
- * Helper: Check if data item is product-related
5041
+ * Build and return the final RagConfig.
5042
+ * Throws if required fields (projectId, vectorDb, llm, embedding) are not set.
5231
5043
  */
5232
- static isProductData(item) {
5233
- const content = (item.content || "").toLowerCase();
5234
- const productKeywords = ["product", "price", "stock", "item", "sku", "brand", "model"];
5235
- return productKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
5236
- (k) => ["name", "price", "product", "sku"].includes(k.toLowerCase())
5237
- );
5044
+ build() {
5045
+ var _a;
5046
+ const missing = [];
5047
+ if (!this._projectId) missing.push('projectId (call .projectId("my-app"))');
5048
+ if (!this._vectorDb) missing.push('vectorDb (call .vectorDb("pinecone", { ... }))');
5049
+ if (!this._llm) missing.push('llm (call .llm("openai", "gpt-4o", apiKey))');
5050
+ if (!this._embedding) missing.push('embedding (call .embedding("openai", "text-embedding-3-small"))');
5051
+ if (missing.length > 0) {
5052
+ throw new Error(
5053
+ `[ConfigBuilder] Cannot build \u2014 required fields are missing:
5054
+ ${missing.join("\n ")}`
5055
+ );
5056
+ }
5057
+ return __spreadValues({
5058
+ projectId: this._projectId,
5059
+ vectorDb: this._vectorDb,
5060
+ llm: this._llm,
5061
+ embedding: this._embedding,
5062
+ ui: this._rag ? this._ui : void 0,
5063
+ rag: (_a = this._rag) != null ? _a : { chunkSize: 1e3, chunkOverlap: 200, topK: 5 }
5064
+ }, this._ui ? { ui: this._ui } : {});
5238
5065
  }
5239
5066
  /**
5240
- * Helper: Check if data contains time series
5067
+ * Build and return as JSON for serialization
5241
5068
  */
5242
- static isTimeSeriesData(item) {
5243
- const content = (item.content || "").toLowerCase();
5244
- const timeKeywords = ["date", "time", "year", "month", "week", "day", "hour", "trend", "historical"];
5245
- return timeKeywords.some((kw) => content.includes(kw)) || Object.keys(item.metadata || {}).some(
5246
- (k) => ["date", "timestamp", "time", "period"].includes(k.toLowerCase())
5247
- );
5069
+ toJSON() {
5070
+ return JSON.stringify(this.build());
5248
5071
  }
5249
- /**
5250
- * Helper: Extract product information from vector match
5251
- */
5252
- static extractProductInfo(item) {
5253
- const meta = item.metadata || {};
5254
- if (meta.name || meta.product) {
5255
- return {
5256
- id: item.id,
5257
- name: meta.name || meta.product,
5258
- price: meta.price,
5259
- image: meta.image || meta.imageUrl,
5260
- brand: meta.brand,
5261
- description: item.content,
5262
- inStock: this.determineStockStatus(item)
5263
- };
5072
+ // ============================================================================
5073
+ // Private helper methods for auto-detection
5074
+ // ============================================================================
5075
+ _autoDetectVectorDb() {
5076
+ if (process.env.PINECONE_API_KEY && process.env.PINECONE_INDEX) {
5077
+ return { provider: "pinecone", indexName: process.env.PINECONE_INDEX, options: { apiKey: process.env.PINECONE_API_KEY } };
5264
5078
  }
5265
- const nameMatch = item.content.match(/(?:Product|Item|Name):\s*([^\n]+)/i);
5266
- const priceMatch = item.content.match(/(?:Price|Cost):\s*\$?([\d.]+)/i);
5267
- if (nameMatch) {
5268
- return {
5269
- id: item.id,
5270
- name: nameMatch[1],
5271
- price: priceMatch ? parseFloat(priceMatch[1]) : void 0,
5272
- description: item.content,
5273
- inStock: this.determineStockStatus(item)
5274
- };
5079
+ if (process.env.QDRANT_URL) {
5080
+ return { provider: "qdrant", indexName: process.env.QDRANT_COLLECTION || "documents", options: { url: process.env.QDRANT_URL, apiKey: process.env.QDRANT_API_KEY } };
5275
5081
  }
5276
- return null;
5082
+ if (process.env.DATABASE_URL) {
5083
+ return { provider: "postgresql", indexName: process.env.PG_TABLE || "documents", options: { connectionString: process.env.DATABASE_URL } };
5084
+ }
5085
+ if (process.env.MONGODB_URI) {
5086
+ return { provider: "mongodb", indexName: process.env.MONGODB_COLLECTION || "documents", options: { uri: process.env.MONGODB_URI, database: process.env.MONGODB_DB || "ai_db", collection: process.env.MONGODB_COLLECTION || "documents" } };
5087
+ }
5088
+ if (process.env.REDIS_URL) {
5089
+ return { provider: "redis", indexName: process.env.REDIS_INDEX || "documents", options: { url: process.env.REDIS_URL } };
5090
+ }
5091
+ return { provider: "qdrant", indexName: "documents", options: { url: process.env.QDRANT_URL || "http://localhost:6333" } };
5092
+ }
5093
+ _autoDetectLLM() {
5094
+ if (process.env.OPENAI_API_KEY) {
5095
+ return { provider: "openai", model: process.env.OPENAI_MODEL || "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY, maxTokens: parseInt(process.env.OPENAI_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OPENAI_TEMPERATURE || "0.7") };
5096
+ }
5097
+ if (process.env.ANTHROPIC_API_KEY) {
5098
+ return { provider: "anthropic", model: process.env.ANTHROPIC_MODEL || "claude-3-haiku-20240307", apiKey: process.env.ANTHROPIC_API_KEY, maxTokens: parseInt(process.env.ANTHROPIC_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.ANTHROPIC_TEMPERATURE || "0.7") };
5099
+ }
5100
+ if (process.env.OLLAMA_BASE_URL) {
5101
+ return { provider: "ollama", model: process.env.OLLAMA_MODEL || "mistral", baseUrl: process.env.OLLAMA_BASE_URL, maxTokens: parseInt(process.env.OLLAMA_MAX_TOKENS || "1024"), temperature: parseFloat(process.env.OLLAMA_TEMPERATURE || "0.7") };
5102
+ }
5103
+ return { provider: "openai", model: "gpt-4o-mini", apiKey: process.env.OPENAI_API_KEY };
5104
+ }
5105
+ _autoDetectEmbedding() {
5106
+ if (process.env.EMBEDDING_PROVIDER) {
5107
+ return { provider: process.env.EMBEDDING_PROVIDER, model: process.env.EMBEDDING_MODEL || "default", apiKey: process.env.EMBEDDING_API_KEY, baseUrl: process.env.EMBEDDING_BASE_URL };
5108
+ }
5109
+ return { provider: "openai", model: "text-embedding-3-small", apiKey: process.env.OPENAI_API_KEY };
5277
5110
  }
5111
+ };
5112
+ var PRESETS = {
5113
+ /** OpenAI + Pinecone: Production-ready cloud setup */
5114
+ "openai-pinecone": { vectorDb: "pinecone", llm: "openai", embedding: "openai" },
5115
+ /** Claude + Qdrant: Open-source vector DB + proprietary LLM */
5116
+ "claude-qdrant": { vectorDb: "qdrant", llm: "anthropic", embedding: "openai" },
5117
+ /** Local development: Ollama + local Qdrant */
5118
+ "local-dev": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
5119
+ /** Fully open-source: Ollama LLM + Qdrant + Ollama embeddings */
5120
+ "fully-open-source": { vectorDb: "qdrant", llm: "ollama", embedding: "ollama" },
5121
+ /** PostgreSQL stack: pgvector + OpenAI */
5122
+ "postgres-openai": { vectorDb: "postgresql", llm: "openai", embedding: "openai" },
5123
+ /** Enterprise MongoDB: MongoDB Atlas with OpenAI */
5124
+ "mongodb-openai": { vectorDb: "mongodb", llm: "openai", embedding: "openai" },
5125
+ /** Redis stack for caching + search */
5126
+ "redis-openai": { vectorDb: "redis", llm: "openai", embedding: "openai" }
5127
+ };
5128
+ function createFromPreset(presetName) {
5129
+ const preset = PRESETS[presetName];
5130
+ return new ConfigBuilder().vectorDb(preset.vectorDb, { apiKey: process.env[`${preset.vectorDb.toUpperCase()}_API_KEY`] }).llm(preset.llm, void 0, process.env[`${preset.llm.toUpperCase()}_API_KEY`]).embedding(preset.embedding, void 0, process.env[`${preset.embedding.toUpperCase()}_API_KEY`]);
5131
+ }
5132
+
5133
+ // src/utils/DocumentParser.ts
5134
+ var DocumentParser = class {
5278
5135
  /**
5279
- * Helper: Detect categories in data
5136
+ * Extract text from a File or Buffer based on its type.
5280
5137
  */
5281
- static detectCategories(data) {
5282
- const categories = /* @__PURE__ */ new Set();
5283
- data.forEach((item) => {
5284
- const meta = item.metadata || {};
5285
- if (meta.category) {
5286
- categories.add(String(meta.category));
5287
- }
5288
- if (meta.type) {
5289
- categories.add(String(meta.type));
5138
+ static async parse(file, fileName, mimeType) {
5139
+ var _a;
5140
+ const extension = (_a = fileName.split(".").pop()) == null ? void 0 : _a.toLowerCase();
5141
+ if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
5142
+ return this.readAsText(file);
5143
+ }
5144
+ if (extension === "json" || mimeType === "application/json") {
5145
+ const text = await this.readAsText(file);
5146
+ try {
5147
+ const obj = JSON.parse(text);
5148
+ return JSON.stringify(obj, null, 2);
5149
+ } catch (e) {
5150
+ return text;
5290
5151
  }
5291
- if (meta.tag) {
5292
- const tags = Array.isArray(meta.tag) ? meta.tag : [meta.tag];
5293
- tags.forEach((t) => categories.add(String(t)));
5152
+ }
5153
+ if (extension === "csv" || mimeType === "text/csv") {
5154
+ return this.readAsText(file);
5155
+ }
5156
+ if (extension === "pdf" || mimeType === "application/pdf") {
5157
+ try {
5158
+ const pdf = await import("pdf-parse");
5159
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
5160
+ const data = await pdf.default(buffer);
5161
+ return data.text;
5162
+ } catch (err) {
5163
+ console.warn('[DocumentParser] PDF parsing failed. Make sure "pdf-parse" is installed.', err);
5164
+ return `[PDF Parsing Error for ${fileName}]`;
5294
5165
  }
5295
- const categoryMatch = item.content.match(/(?:Category|Type|Class):\s*([^\n]+)/i);
5296
- if (categoryMatch) {
5297
- categories.add(categoryMatch[1].trim());
5166
+ }
5167
+ if (extension === "docx" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
5168
+ try {
5169
+ const mammoth = await import("mammoth");
5170
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
5171
+ const result = await mammoth.extractRawText({ buffer });
5172
+ return result.value;
5173
+ } catch (err) {
5174
+ console.warn('[DocumentParser] DOCX parsing failed. Make sure "mammoth" is installed.', err);
5175
+ return `[DOCX Parsing Error for ${fileName}]`;
5298
5176
  }
5299
- });
5300
- return Array.from(categories);
5177
+ }
5178
+ try {
5179
+ return await this.readAsText(file);
5180
+ } catch (e) {
5181
+ throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
5182
+ }
5183
+ }
5184
+ static async readAsText(file) {
5185
+ if (Buffer.isBuffer(file)) {
5186
+ return file.toString("utf-8");
5187
+ }
5188
+ return await file.text();
5189
+ }
5190
+ };
5191
+
5192
+ // src/server.ts
5193
+ init_BaseVectorProvider();
5194
+ init_PineconeProvider();
5195
+ init_PostgreSQLProvider();
5196
+
5197
+ // src/providers/vectordb/MultiTablePostgresProvider.ts
5198
+ var import_pg2 = require("pg");
5199
+ init_BaseVectorProvider();
5200
+ var MultiTablePostgresProvider = class extends BaseVectorProvider {
5201
+ constructor(config) {
5202
+ var _a, _b, _c, _d, _e;
5203
+ super(config);
5204
+ const opts = config.options || {};
5205
+ if (!opts.connectionString) {
5206
+ throw new Error("[MultiTablePostgresProvider] options.connectionString is required");
5207
+ }
5208
+ this.connectionString = opts.connectionString;
5209
+ this.dimensions = (_a = opts.dimensions) != null ? _a : 768;
5210
+ const rawTables = (_c = (_b = opts.tables) != null ? _b : process.env.VECTOR_DB_TABLES) != null ? _c : "";
5211
+ this.tables = typeof rawTables === "string" ? rawTables.split(",").map((t) => t.trim()).filter(Boolean) : rawTables;
5212
+ if (this.tables.length === 0) {
5213
+ console.warn(
5214
+ "[MultiTablePostgresProvider] No tables configured. Set VECTOR_DB_TABLES as a comma-separated list of table names or pass options.tables."
5215
+ );
5216
+ }
5217
+ const rawSearchFields = (_e = (_d = opts.searchFields) != null ? _d : process.env.VECTOR_DB_SEARCH_FIELDS) != null ? _e : ["name", "product_name", "productname", "title"];
5218
+ this.searchFields = typeof rawSearchFields === "string" ? rawSearchFields.split(",").map((f) => f.trim()).filter(Boolean) : rawSearchFields;
5301
5219
  }
5302
- /**
5303
- * Helper: Aggregate data by category
5304
- */
5305
- static aggregateByCategory(data, categories) {
5306
- const result = {};
5307
- categories.forEach((cat) => {
5308
- result[cat] = 0;
5309
- });
5310
- data.forEach((item) => {
5311
- const meta = item.metadata || {};
5312
- const itemCategory = meta.category || meta.type || "Other";
5313
- if (result.hasOwnProperty(itemCategory)) {
5314
- result[itemCategory]++;
5220
+ async initialize() {
5221
+ this.pool = new import_pg2.Pool({ connectionString: this.connectionString });
5222
+ const client = await this.pool.connect();
5223
+ try {
5224
+ const res = await client.query(`
5225
+ SELECT table_name
5226
+ FROM information_schema.columns
5227
+ WHERE column_name = 'embedding'
5228
+ AND table_schema = 'public'
5229
+ `);
5230
+ const discoveredTables = res.rows.map((r) => r.table_name);
5231
+ if (this.tables.length === 0) {
5232
+ this.tables = discoveredTables;
5315
5233
  } else {
5316
- result["Other"] = (result["Other"] || 0) + 1;
5234
+ const staticTables = [...this.tables];
5235
+ this.tables = staticTables.filter((t) => discoveredTables.includes(t));
5236
+ if (this.tables.length < staticTables.length) {
5237
+ const missing = staticTables.filter((t) => !discoveredTables.includes(t));
5238
+ console.warn(`[MultiTablePostgresProvider] Some configured tables were skipped (missing 'embedding' column): ${missing.join(", ")}`);
5239
+ }
5317
5240
  }
5318
- });
5319
- return result;
5241
+ if (this.tables.length === 0) {
5242
+ console.warn('[MultiTablePostgresProvider] No tables with "embedding" columns found in the database.');
5243
+ } else {
5244
+ console.log(
5245
+ `[MultiTablePostgresProvider] Connected. Searching across ${this.tables.length} table(s): ${this.tables.join(", ")}`
5246
+ );
5247
+ }
5248
+ } finally {
5249
+ client.release();
5250
+ }
5320
5251
  }
5321
5252
  /**
5322
- * Helper: Extract time series data
5253
+ * Upsert is not supported for MultiTablePostgresProvider as it's designed for
5254
+ * searching across pre-existing tables with varying schemas.
5323
5255
  */
5324
- static extractTimeSeriesData(data) {
5325
- return data.map((item) => {
5326
- const meta = item.metadata || {};
5327
- return {
5328
- timestamp: meta.timestamp || meta.date || (/* @__PURE__ */ new Date()).toISOString(),
5329
- value: meta.value || item.score || 0,
5330
- label: meta.label || item.content.substring(0, 50)
5331
- };
5332
- });
5256
+ async upsert(_doc, _namespace) {
5257
+ throw new Error(
5258
+ "[MultiTablePostgresProvider] upsert() is not supported in multi-table mode. Please use the standard PostgreSQLProvider for single-table managed indices."
5259
+ );
5333
5260
  }
5334
5261
  /**
5335
- * Helper: Extract table columns
5262
+ * Batch upsert is not supported for MultiTablePostgresProvider.
5336
5263
  */
5337
- static extractTableColumns(data) {
5338
- const columnSet = /* @__PURE__ */ new Set();
5339
- columnSet.add("Content");
5340
- data.forEach((item) => {
5341
- Object.keys(item.metadata || {}).forEach((key) => {
5342
- columnSet.add(key.charAt(0).toUpperCase() + key.slice(1));
5343
- });
5344
- });
5345
- return Array.from(columnSet);
5264
+ async batchUpsert(_docs, _namespace) {
5265
+ throw new Error(
5266
+ "[MultiTablePostgresProvider] batchUpsert() is not supported in multi-table mode."
5267
+ );
5346
5268
  }
5347
5269
  /**
5348
- * Helper: Extract table row
5270
+ * Query all configured tables and merge results, sorted by cosine similarity score.
5349
5271
  */
5350
- static extractTableRow(item, columns) {
5351
- const meta = item.metadata || {};
5352
- return columns.map((col) => {
5353
- if (col === "Content") {
5354
- return item.content.substring(0, 100);
5272
+ async query(vector, topK, _namespace, _filter) {
5273
+ var _a, _b, _c;
5274
+ if (!this.pool) {
5275
+ throw new Error("[MultiTablePostgresProvider] Provider not initialized. Call initialize() first.");
5276
+ }
5277
+ const vectorLiteral = `[${vector.join(",")}]`;
5278
+ const allResults = [];
5279
+ console.log(`[MultiTablePostgresProvider] --- Starting Multi-Table Search ---`);
5280
+ const queryText = _filter == null ? void 0 : _filter.queryText;
5281
+ const entityHints = Array.isArray(_filter == null ? void 0 : _filter.__entityHints) ? _filter.__entityHints.filter(
5282
+ (hint) => typeof hint === "object" && hint !== null && typeof hint.value === "string"
5283
+ ).map((hint) => hint.value.trim().toLowerCase()).filter(Boolean) : [];
5284
+ console.log(`[MultiTablePostgresProvider] queryText: "${queryText}"`);
5285
+ console.log(`[MultiTablePostgresProvider] entityHints: [${entityHints.join(", ")}]`);
5286
+ const getDynamicKeywordQuery = () => {
5287
+ if (entityHints.length > 0) {
5288
+ return entityHints.map((h) => h.includes(" ") ? `"${h}"` : h).join(" & ");
5289
+ }
5290
+ if (queryText) {
5291
+ return queryText.toLowerCase().replace(/\b(show|me|the|product[s]?|item[s]?|card[s]?|of|category|find|list|browse|what|are|display|all|under|for|in|with|about)\b/g, "").trim().replace(/\s+/g, " & ");
5292
+ }
5293
+ return "";
5294
+ };
5295
+ const dynamicKeywordQuery = getDynamicKeywordQuery();
5296
+ const queryPromises = this.tables.map(async (table) => {
5297
+ try {
5298
+ let sqlQuery = "";
5299
+ let params = [];
5300
+ if (queryText) {
5301
+ const hasEntityHints = entityHints.length > 0;
5302
+ const exactNameScoreExpr = hasEntityHints ? `+ (
5303
+ SELECT COALESCE(MAX(
5304
+ CASE WHEN LOWER(val) IN (${entityHints.map((h) => `'${h.replace(/'/g, "''")}'`).join(", ")})
5305
+ THEN 1.0 ELSE 0.0 END
5306
+ ), 0)
5307
+ FROM jsonb_each_text(to_jsonb(t)) AS kv(key, val)
5308
+ WHERE key IN (${this.searchFields.map((f) => `'${f}'`).join(", ")})
5309
+ ) * 3.0` : "";
5310
+ sqlQuery = `
5311
+ SELECT *,
5312
+ (1 - (embedding <=> $1::vector)) AS vector_score,
5313
+ COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) AS keyword_score,
5314
+ ((1 - (embedding <=> $1::vector)) + (COALESCE(ts_rank(to_tsvector('english', (to_jsonb(t) - 'embedding')::text), to_tsquery('english', $2)), 0) * 2.0) ${exactNameScoreExpr}) AS hybrid_score
5315
+ FROM "${table}" t
5316
+ ORDER BY hybrid_score DESC
5317
+ LIMIT 50
5318
+ `;
5319
+ params = [vectorLiteral, dynamicKeywordQuery];
5320
+ } else {
5321
+ sqlQuery = `
5322
+ SELECT *,
5323
+ (1 - (embedding <=> $1::vector)) AS hybrid_score
5324
+ FROM "${table}" t
5325
+ ORDER BY hybrid_score DESC
5326
+ LIMIT 50
5327
+ `;
5328
+ params = [vectorLiteral];
5329
+ }
5330
+ const result = await this.pool.query(sqlQuery, params);
5331
+ if (result.rowCount && result.rowCount > 0) {
5332
+ console.log(` \u2705 Table "${table}": Found ${result.rowCount} potential matches. Top score: ${result.rows[0].hybrid_score.toFixed(4)}`);
5333
+ } else {
5334
+ console.log(` \u26AA Table "${table}": No relevant matches found.`);
5335
+ }
5336
+ const tableResults = [];
5337
+ for (const row of result.rows) {
5338
+ const _a2 = row, { hybrid_score, id } = _a2, rest = __objRest(_a2, ["hybrid_score", "id"]);
5339
+ delete rest.embedding;
5340
+ delete rest.vector_score;
5341
+ delete rest.keyword_score;
5342
+ delete rest.exact_name_score;
5343
+ const content = `[TYPE: ${table.replace(/s$/, "").toUpperCase()}]
5344
+ ` + Object.entries(rest).filter(([k, v]) => v !== null && typeof v !== "object" && k !== "id").map(([k, v]) => `${k}: ${v}`).join("\n");
5345
+ tableResults.push({
5346
+ id: `${table}-${id}`,
5347
+ score: parseFloat(String(hybrid_score)),
5348
+ content,
5349
+ metadata: __spreadProps(__spreadValues({}, rest), { source_table: table })
5350
+ });
5351
+ }
5352
+ return tableResults;
5353
+ } catch (err) {
5354
+ console.error(
5355
+ `[MultiTablePostgresProvider] Error querying table "${table}":`,
5356
+ err.message
5357
+ );
5358
+ return [];
5355
5359
  }
5356
- const metaKey = col.charAt(0).toLowerCase() + col.slice(1);
5357
- const value = meta[metaKey];
5358
- return value !== void 0 ? String(value) : "";
5359
- });
5360
- }
5361
- /**
5362
- * Helper: Check stock status
5363
- */
5364
- static checkStockStatus(category, data) {
5365
- const item = data.find((d) => {
5366
- const meta = d.metadata || {};
5367
- return meta.category === category || meta.type === category;
5368
5360
  });
5369
- return this.determineStockStatus(item || {});
5370
- }
5371
- /**
5372
- * Helper: Determine if item is in stock
5373
- */
5374
- static determineStockStatus(item) {
5375
- const meta = item.metadata || {};
5376
- if (meta.inStock !== void 0) {
5377
- return Boolean(meta.inStock);
5361
+ const resultsArray = await Promise.all(queryPromises);
5362
+ for (const tableResults of resultsArray) {
5363
+ allResults.push(...tableResults);
5378
5364
  }
5379
- if (meta.stock !== void 0) {
5380
- return Boolean(meta.stock);
5365
+ const resultsByTable = {};
5366
+ for (const res of allResults) {
5367
+ const table = (_a = res.metadata) == null ? void 0 : _a.source_table;
5368
+ if (!resultsByTable[table]) resultsByTable[table] = [];
5369
+ resultsByTable[table].push(res);
5381
5370
  }
5382
- if (meta.available !== void 0) {
5383
- return Boolean(meta.available);
5371
+ const balancedResults = [];
5372
+ const tables = Object.keys(resultsByTable);
5373
+ for (const table of tables) {
5374
+ balancedResults.push(...resultsByTable[table].slice(0, 3));
5384
5375
  }
5385
- const content = (item.content || "").toLowerCase();
5386
- if (content.includes("out of stock") || content.includes("unavailable")) {
5387
- return false;
5376
+ const finalSorted = balancedResults.sort((a, b) => b.score - a.score);
5377
+ if (finalSorted.length > 0) {
5378
+ console.log(`[MultiTablePostgresProvider] --- Search Complete ---`);
5379
+ console.log(`[MultiTablePostgresProvider] Final top match from "${(_c = (_b = finalSorted[0].metadata) == null ? void 0 : _b.source_table) != null ? _c : "unknown"}" with score ${finalSorted[0].score.toFixed(4)}`);
5388
5380
  }
5389
- if (content.includes("in stock") || content.includes("available")) {
5381
+ return finalSorted.slice(0, Math.max(topK, 15));
5382
+ }
5383
+ async delete(_id, _namespace) {
5384
+ console.warn("[MultiTablePostgresProvider] delete() is a no-op for multi-table mode.");
5385
+ }
5386
+ async deleteNamespace(_namespace) {
5387
+ console.warn("[MultiTablePostgresProvider] deleteNamespace() is a no-op for multi-table mode.");
5388
+ }
5389
+ async ping() {
5390
+ try {
5391
+ await this.pool.query("SELECT 1");
5390
5392
  return true;
5393
+ } catch (e) {
5394
+ return false;
5391
5395
  }
5392
- return true;
5393
5396
  }
5394
- /**
5395
- * Helper: Check if data has multiple fields
5396
- */
5397
- static hasMultipleFields(data) {
5398
- const fieldCount = /* @__PURE__ */ new Set();
5399
- data.forEach((item) => {
5400
- Object.keys(item.metadata || {}).forEach((key) => {
5401
- fieldCount.add(key);
5402
- });
5403
- });
5404
- return fieldCount.size > 2;
5397
+ async disconnect() {
5398
+ if (this.pool) {
5399
+ await this.pool.end();
5400
+ }
5405
5401
  }
5406
5402
  };
5407
5403
 
5404
+ // src/server.ts
5405
+ init_MongoDBProvider();
5406
+ init_MilvusProvider();
5407
+ init_QdrantProvider();
5408
+ init_ChromaDBProvider();
5409
+ init_RedisProvider();
5410
+ init_WeaviateProvider();
5411
+ init_UniversalVectorProvider();
5412
+
5408
5413
  // src/handlers/index.ts
5414
+ var import_server = require("next/server");
5409
5415
  function sseFrame(payload) {
5410
5416
  return `data: ${JSON.stringify(payload)}
5411
5417