@retrivora-ai/rag-engine 2.5.7 → 2.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-DwFj0h2u.d.mts → ILLMProvider-B2N3lMBc.d.mts} +10 -1
- package/dist/{ILLMProvider-DwFj0h2u.d.ts → ILLMProvider-B2N3lMBc.d.ts} +10 -1
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +962 -193
- package/dist/handlers/index.mjs +12821 -24
- package/dist/{index-Cb0GsT16.d.ts → index-37V810a-.d.mts} +3 -3
- package/dist/{index-D1jN8KSF.d.mts → index-BxzDv2jd.d.ts} +3 -3
- package/dist/{index-9h_ENWQn.d.mts → index-CAGGPyI7.d.mts} +6 -2
- package/dist/{index-Xci6GfpI.d.ts → index-DdQxoIpu.d.ts} +6 -2
- package/dist/index.css +76 -4
- package/dist/index.d.mts +6 -6
- package/dist/index.d.ts +6 -6
- package/dist/index.js +138 -96
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +1554 -130
- package/dist/server.d.mts +100 -9
- package/dist/server.d.ts +100 -9
- package/dist/server.js +964 -193
- package/dist/server.mjs +13378 -372
- package/package.json +5 -2
- package/dist/ChromaDBProvider-4RPJN7Y3.mjs +0 -8
- package/dist/MilvusProvider-WJLLZ6TW.mjs +0 -8
- package/dist/MongoDBProvider-QSERELZX.mjs +0 -8
- package/dist/MultiTablePostgresProvider-JF2HEUDZ.mjs +0 -9
- package/dist/PineconeProvider-HLZL7HKU.mjs +0 -10
- package/dist/PostgreSQLProvider-JQMCNHRO.mjs +0 -8
- package/dist/QdrantProvider-6MNFPEVC.mjs +0 -8
- package/dist/RedisProvider-ERQL6AEB.mjs +0 -8
- package/dist/SimpleGraphProvider-OP3HAJJM.mjs +0 -70
- package/dist/UniversalVectorProvider-INM3KH5F.mjs +0 -9
- package/dist/WeaviateProvider-VHWWICCC.mjs +0 -8
- package/dist/chunk-366EFBZC.mjs +0 -165
- package/dist/chunk-6MGVQTU4.mjs +0 -8883
- package/dist/chunk-B4AXIUM7.mjs +0 -49
- package/dist/chunk-BEPLYQBS.mjs +0 -89
- package/dist/chunk-BNOPQCZW.mjs +0 -238
- package/dist/chunk-FJ43J4FW.mjs +0 -654
- package/dist/chunk-GP2AJQRQ.mjs +0 -107
- package/dist/chunk-IMP6FUCY.mjs +0 -30
- package/dist/chunk-JRQWSJNF.mjs +0 -202
- package/dist/chunk-LYHCKUNF.mjs +0 -112
- package/dist/chunk-MFIWWUC7.mjs +0 -152
- package/dist/chunk-MWICNCB2.mjs +0 -157
- package/dist/chunk-N54YTFBN.mjs +0 -76
- package/dist/chunk-POOWU2HR.mjs +0 -211
- package/dist/chunk-QGDGPOZH.mjs +0 -217
- package/dist/chunk-QI3FTN6G.mjs +0 -281
- package/dist/chunk-QJIDGVG6.mjs +0 -334
- package/dist/chunk-W2DKVD3G.mjs +0 -81
- package/dist/chunk-XHSDHPKI.mjs +0 -336
package/dist/server.js
CHANGED
|
@@ -94,8 +94,8 @@ function buildPayload(template, vars) {
|
|
|
94
94
|
let raw = template;
|
|
95
95
|
for (const [key, val] of Object.entries(vars)) {
|
|
96
96
|
const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
|
|
97
|
-
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
|
|
98
|
-
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
|
|
97
|
+
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
|
|
98
|
+
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
|
|
99
99
|
}
|
|
100
100
|
try {
|
|
101
101
|
return JSON.parse(raw);
|
|
@@ -332,7 +332,7 @@ var init_package = __esm({
|
|
|
332
332
|
"package.json"() {
|
|
333
333
|
package_default = {
|
|
334
334
|
name: "@retrivora-ai/rag-engine",
|
|
335
|
-
version: "2.5.
|
|
335
|
+
version: "2.5.9",
|
|
336
336
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
337
337
|
author: "Abhinav Alkuchi",
|
|
338
338
|
license: "UNLICENSED",
|
|
@@ -435,9 +435,11 @@ var init_package = __esm({
|
|
|
435
435
|
"@google/genai": "^0.8.0",
|
|
436
436
|
"@google/generative-ai": "^0.24.1",
|
|
437
437
|
"@pinecone-database/pinecone": "^7.2.0",
|
|
438
|
-
"@retrivora-ai/billing-guards": "
|
|
438
|
+
"@retrivora-ai/billing-guards": "file:../billing-guards",
|
|
439
439
|
"@types/papaparse": "^5.5.2",
|
|
440
|
+
exceljs: "^4.4.0",
|
|
440
441
|
"framer-motion": "^13.1.0",
|
|
442
|
+
jszip: "^3.10.1",
|
|
441
443
|
katex: "^0.18.4",
|
|
442
444
|
"lucide-react": "^1.8.0",
|
|
443
445
|
"next-themes": "^0.4.6",
|
|
@@ -450,7 +452,7 @@ var init_package = __esm({
|
|
|
450
452
|
"rehype-katex": "^7.0.1",
|
|
451
453
|
"remark-gfm": "^4.0.1",
|
|
452
454
|
"remark-math": "^6.0.0",
|
|
453
|
-
|
|
455
|
+
xlsx: "^0.18.5",
|
|
454
456
|
zod: "^4.4.3"
|
|
455
457
|
},
|
|
456
458
|
optionalDependencies: {
|
|
@@ -468,6 +470,7 @@ var init_package = __esm({
|
|
|
468
470
|
"@tailwindcss/postcss": "^4",
|
|
469
471
|
"@types/estree": "^1.0.9",
|
|
470
472
|
"@types/jest": "^30.0.0",
|
|
473
|
+
"@types/jszip": "^3.4.1",
|
|
471
474
|
"@types/katex": "^0.16.8",
|
|
472
475
|
"@types/node": "^20",
|
|
473
476
|
"@types/pdf-parse": "^1.1.5",
|
|
@@ -2744,6 +2747,7 @@ __export(server_exports, {
|
|
|
2744
2747
|
PineconeProvider: () => PineconeProvider,
|
|
2745
2748
|
Pipeline: () => Pipeline,
|
|
2746
2749
|
PostgreSQLProvider: () => PostgreSQLProvider,
|
|
2750
|
+
PromptOptimizer: () => PromptOptimizer,
|
|
2747
2751
|
ProviderHealthCheck: () => ProviderHealthCheck,
|
|
2748
2752
|
ProviderNotFoundException: () => ProviderNotFoundException,
|
|
2749
2753
|
ProviderRegistry: () => ProviderRegistry,
|
|
@@ -2898,7 +2902,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2898
2902
|
*/
|
|
2899
2903
|
static async generateMachineFingerprint() {
|
|
2900
2904
|
if (typeof window !== "undefined" && typeof navigator !== "undefined") {
|
|
2901
|
-
return
|
|
2905
|
+
return "browser-client";
|
|
2902
2906
|
}
|
|
2903
2907
|
try {
|
|
2904
2908
|
const os2 = await import("os");
|
|
@@ -2970,6 +2974,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2970
2974
|
try {
|
|
2971
2975
|
response = await fetch(endpoint, {
|
|
2972
2976
|
method: "POST",
|
|
2977
|
+
signal: AbortSignal.timeout(6e3),
|
|
2973
2978
|
headers: {
|
|
2974
2979
|
"Content-Type": "application/json",
|
|
2975
2980
|
"x-license-key": licenseKey
|
|
@@ -5695,15 +5700,29 @@ Text to extract from:
|
|
|
5695
5700
|
const response = await this.llm.chat([
|
|
5696
5701
|
{ role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
|
|
5697
5702
|
{ role: "user", content: prompt }
|
|
5698
|
-
], "", { maxTokens:
|
|
5703
|
+
], "", { maxTokens: 800, temperature: 0.1 });
|
|
5699
5704
|
try {
|
|
5700
|
-
const
|
|
5701
|
-
const
|
|
5702
|
-
|
|
5705
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5706
|
+
const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
|
5707
|
+
const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
|
|
5708
|
+
const jsonMatch = candidate.match(/\{[\s\S]*\}/);
|
|
5709
|
+
const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
|
|
5710
|
+
const parsed = JSON.parse(cleanJson);
|
|
5711
|
+
return {
|
|
5712
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5713
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5714
|
+
};
|
|
5703
5715
|
} catch (e) {
|
|
5704
5716
|
try {
|
|
5705
|
-
const
|
|
5706
|
-
|
|
5717
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5718
|
+
const repaired = this.repairTruncatedJson(withoutThinking);
|
|
5719
|
+
if (repaired) {
|
|
5720
|
+
const parsed = JSON.parse(repaired);
|
|
5721
|
+
return {
|
|
5722
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5723
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5724
|
+
};
|
|
5725
|
+
}
|
|
5707
5726
|
} catch (e2) {
|
|
5708
5727
|
}
|
|
5709
5728
|
console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
|
|
@@ -5981,6 +6000,208 @@ ${error instanceof Error ? error.message : String(error)}`
|
|
|
5981
6000
|
}
|
|
5982
6001
|
};
|
|
5983
6002
|
|
|
6003
|
+
// src/rag/PromptOptimizer.ts
|
|
6004
|
+
var PromptOptimizer = class {
|
|
6005
|
+
/**
|
|
6006
|
+
* Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
|
|
6007
|
+
*/
|
|
6008
|
+
static optimize(rawPrompt, history = [], options = {}) {
|
|
6009
|
+
if (!rawPrompt || typeof rawPrompt !== "string") {
|
|
6010
|
+
return {
|
|
6011
|
+
optimizedPrompt: "",
|
|
6012
|
+
originalPrompt: rawPrompt || "",
|
|
6013
|
+
wasModified: false,
|
|
6014
|
+
transformations: []
|
|
6015
|
+
};
|
|
6016
|
+
}
|
|
6017
|
+
const { resolveHistory = true, expandKeywords = true } = options;
|
|
6018
|
+
const transformations = [];
|
|
6019
|
+
let current = rawPrompt;
|
|
6020
|
+
const cleaned = this.cleanText(current);
|
|
6021
|
+
if (cleaned !== current) {
|
|
6022
|
+
transformations.push("clean_whitespace_and_artifacts");
|
|
6023
|
+
current = cleaned;
|
|
6024
|
+
}
|
|
6025
|
+
const expandedAbbr = this.expandAbbreviations(current);
|
|
6026
|
+
if (expandedAbbr !== current) {
|
|
6027
|
+
transformations.push("expand_abbreviations");
|
|
6028
|
+
current = expandedAbbr;
|
|
6029
|
+
}
|
|
6030
|
+
if (expandKeywords) {
|
|
6031
|
+
const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
6032
|
+
if (this.TOPIC_EXPANSIONS[trimmedLower]) {
|
|
6033
|
+
current = this.TOPIC_EXPANSIONS[trimmedLower];
|
|
6034
|
+
transformations.push(`topic_expansion_${trimmedLower}`);
|
|
6035
|
+
}
|
|
6036
|
+
}
|
|
6037
|
+
if (resolveHistory && history && history.length > 0) {
|
|
6038
|
+
const resolved = this.resolveCoreferences(current, history);
|
|
6039
|
+
if (resolved !== current) {
|
|
6040
|
+
transformations.push("resolve_history_coreferences");
|
|
6041
|
+
current = resolved;
|
|
6042
|
+
}
|
|
6043
|
+
}
|
|
6044
|
+
if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
|
|
6045
|
+
if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
|
|
6046
|
+
current = `${current.trim()}?`;
|
|
6047
|
+
transformations.push("add_question_mark");
|
|
6048
|
+
}
|
|
6049
|
+
}
|
|
6050
|
+
return {
|
|
6051
|
+
optimizedPrompt: current,
|
|
6052
|
+
originalPrompt: rawPrompt,
|
|
6053
|
+
wasModified: current !== rawPrompt,
|
|
6054
|
+
transformations
|
|
6055
|
+
};
|
|
6056
|
+
}
|
|
6057
|
+
/**
|
|
6058
|
+
* Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
|
|
6059
|
+
*/
|
|
6060
|
+
static cleanText(text) {
|
|
6061
|
+
let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
|
|
6062
|
+
const doubleQuotesCount = (clean.match(/"/g) || []).length;
|
|
6063
|
+
if (doubleQuotesCount % 2 !== 0) {
|
|
6064
|
+
clean += '"';
|
|
6065
|
+
}
|
|
6066
|
+
const singleQuotesCount = (clean.match(/'/g) || []).length;
|
|
6067
|
+
if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
|
|
6068
|
+
clean += "'";
|
|
6069
|
+
}
|
|
6070
|
+
const openParen = (clean.match(/\(/g) || []).length;
|
|
6071
|
+
const closeParen = (clean.match(/\)/g) || []).length;
|
|
6072
|
+
if (openParen > closeParen) {
|
|
6073
|
+
clean += ")".repeat(openParen - closeParen);
|
|
6074
|
+
}
|
|
6075
|
+
return clean;
|
|
6076
|
+
}
|
|
6077
|
+
/**
|
|
6078
|
+
* Replaces shorthand chat abbreviations with clear natural language.
|
|
6079
|
+
*/
|
|
6080
|
+
static expandAbbreviations(text) {
|
|
6081
|
+
let expanded = text;
|
|
6082
|
+
for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
|
|
6083
|
+
expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
|
|
6084
|
+
}
|
|
6085
|
+
return expanded;
|
|
6086
|
+
}
|
|
6087
|
+
/**
|
|
6088
|
+
* Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
|
|
6089
|
+
*/
|
|
6090
|
+
static resolveCoreferences(prompt, history) {
|
|
6091
|
+
const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
|
|
6092
|
+
if (!pronounPattern.test(prompt)) {
|
|
6093
|
+
return prompt;
|
|
6094
|
+
}
|
|
6095
|
+
const recentMessages = [...history].slice(-4).reverse();
|
|
6096
|
+
let detectedEntity = null;
|
|
6097
|
+
for (const msg of recentMessages) {
|
|
6098
|
+
if (!msg.content) continue;
|
|
6099
|
+
const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
|
|
6100
|
+
if (quotedMatch) {
|
|
6101
|
+
detectedEntity = quotedMatch[1];
|
|
6102
|
+
break;
|
|
6103
|
+
}
|
|
6104
|
+
const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
|
|
6105
|
+
if (headingMatch) {
|
|
6106
|
+
detectedEntity = headingMatch[1].trim();
|
|
6107
|
+
break;
|
|
6108
|
+
}
|
|
6109
|
+
const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
|
|
6110
|
+
if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
|
|
6111
|
+
detectedEntity = entityMatches[1];
|
|
6112
|
+
break;
|
|
6113
|
+
}
|
|
6114
|
+
}
|
|
6115
|
+
if (detectedEntity) {
|
|
6116
|
+
let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
|
|
6117
|
+
return resolved;
|
|
6118
|
+
}
|
|
6119
|
+
return prompt;
|
|
6120
|
+
}
|
|
6121
|
+
/**
|
|
6122
|
+
* Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
|
|
6123
|
+
*/
|
|
6124
|
+
static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
|
|
6125
|
+
const fastResult = this.optimize(rawPrompt, history);
|
|
6126
|
+
if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
|
|
6127
|
+
return fastResult;
|
|
6128
|
+
}
|
|
6129
|
+
try {
|
|
6130
|
+
const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
|
|
6131
|
+
const prompt = `You are a query optimizer in an enterprise RAG system.
|
|
6132
|
+
Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
|
|
6133
|
+
- Do NOT answer the question.
|
|
6134
|
+
- Resolve any pronouns like "it", "they", "that" using the context.
|
|
6135
|
+
- Keep the user's intent 100% intact.
|
|
6136
|
+
- If the question is already clear, return it as is.
|
|
6137
|
+
- Return ONLY the optimized query string. No quotes or explanations.
|
|
6138
|
+
|
|
6139
|
+
Context:
|
|
6140
|
+
${historyContext || "No previous context."}
|
|
6141
|
+
|
|
6142
|
+
User Question: "${rawPrompt}"
|
|
6143
|
+
|
|
6144
|
+
Optimized Search Prompt:`;
|
|
6145
|
+
const response = await llmProvider.chat(
|
|
6146
|
+
[{ role: "user", content: prompt }],
|
|
6147
|
+
"",
|
|
6148
|
+
{
|
|
6149
|
+
systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
|
|
6150
|
+
temperature: 0,
|
|
6151
|
+
maxTokens: 100
|
|
6152
|
+
}
|
|
6153
|
+
);
|
|
6154
|
+
const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
|
|
6155
|
+
if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
|
|
6156
|
+
return {
|
|
6157
|
+
optimizedPrompt: cleaned,
|
|
6158
|
+
originalPrompt: rawPrompt,
|
|
6159
|
+
wasModified: cleaned !== rawPrompt,
|
|
6160
|
+
transformations: [...fastResult.transformations, "llm_deep_reformulation"]
|
|
6161
|
+
};
|
|
6162
|
+
}
|
|
6163
|
+
} catch (err) {
|
|
6164
|
+
console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
|
|
6165
|
+
}
|
|
6166
|
+
return fastResult;
|
|
6167
|
+
}
|
|
6168
|
+
};
|
|
6169
|
+
PromptOptimizer.COMMON_ABBREVIATIONS = {
|
|
6170
|
+
"\\bpls\\b": "please",
|
|
6171
|
+
"\\bplz\\b": "please",
|
|
6172
|
+
"\\bthx\\b": "thanks",
|
|
6173
|
+
"\\bty\\b": "thank you",
|
|
6174
|
+
"\\bapprox\\b": "approximately",
|
|
6175
|
+
"\\bw/o\\b": "without",
|
|
6176
|
+
"\\bw/\\b": "with",
|
|
6177
|
+
"\\bdiff\\b": "difference",
|
|
6178
|
+
"\\binfo\\b": "information",
|
|
6179
|
+
"\\bconfig\\b": "configuration",
|
|
6180
|
+
"\\bdoc\\b": "document",
|
|
6181
|
+
"\\bdocs\\b": "documentation",
|
|
6182
|
+
"\\bfeat\\b": "feature",
|
|
6183
|
+
"\\bfeats\\b": "features",
|
|
6184
|
+
"\\bvs\\b": "versus",
|
|
6185
|
+
"\\bu\\b": "you",
|
|
6186
|
+
"\\bur\\b": "your",
|
|
6187
|
+
"\\br\\b": "are"
|
|
6188
|
+
};
|
|
6189
|
+
PromptOptimizer.TOPIC_EXPANSIONS = {
|
|
6190
|
+
"pricing": "What are the available pricing plans, tiers, and cost structures?",
|
|
6191
|
+
"price": "What is the pricing and subscription cost?",
|
|
6192
|
+
"features": "What are the core features, capabilities, and tools provided?",
|
|
6193
|
+
"security": "What are the security, privacy, and compliance standards?",
|
|
6194
|
+
"privacy": "What is the privacy policy and how is data protected?",
|
|
6195
|
+
"refund": "What is the refund and cancellation policy?",
|
|
6196
|
+
"refunds": "What is the refund and cancellation policy?",
|
|
6197
|
+
"contact": "How can I contact customer support or sales?",
|
|
6198
|
+
"support": "What support channels and assistance are available?",
|
|
6199
|
+
"docs": "Where can I find the official documentation and setup guides?",
|
|
6200
|
+
"installation": "How do I install and configure the SDK or service?",
|
|
6201
|
+
"install": "What are the step-by-step installation instructions?",
|
|
6202
|
+
"faq": "What are the frequently asked questions and key answers?"
|
|
6203
|
+
};
|
|
6204
|
+
|
|
5984
6205
|
// src/core/mcp.ts
|
|
5985
6206
|
var import_child_process = require("child_process");
|
|
5986
6207
|
var MCPClient = class {
|
|
@@ -7204,6 +7425,26 @@ var IntentClassifier = class {
|
|
|
7204
7425
|
static extractDataSignals(context) {
|
|
7205
7426
|
const docs = context.retrievedDocuments || [];
|
|
7206
7427
|
const rowCount = docs.length;
|
|
7428
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
7429
|
+
"filetype",
|
|
7430
|
+
"mimetype",
|
|
7431
|
+
"filename",
|
|
7432
|
+
"docid",
|
|
7433
|
+
"uploadedat",
|
|
7434
|
+
"dimension",
|
|
7435
|
+
"chunkindex",
|
|
7436
|
+
"totalchunks",
|
|
7437
|
+
"charactercount",
|
|
7438
|
+
"workspaceid",
|
|
7439
|
+
"pineconenamespace",
|
|
7440
|
+
"tier",
|
|
7441
|
+
"custommetadata",
|
|
7442
|
+
"csvheaders",
|
|
7443
|
+
"headers",
|
|
7444
|
+
"filesize",
|
|
7445
|
+
"content",
|
|
7446
|
+
"id"
|
|
7447
|
+
]);
|
|
7207
7448
|
let hasNumericFields = false;
|
|
7208
7449
|
let hasDateFields = false;
|
|
7209
7450
|
let hasCategoricalFields = false;
|
|
@@ -7219,6 +7460,7 @@ var IntentClassifier = class {
|
|
|
7219
7460
|
docs.forEach((doc) => {
|
|
7220
7461
|
const meta = doc.metadata || {};
|
|
7221
7462
|
Object.entries(meta).forEach(([key, val]) => {
|
|
7463
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
7222
7464
|
keys.add(key);
|
|
7223
7465
|
const kLower = key.toLowerCase();
|
|
7224
7466
|
if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
|
|
@@ -7227,7 +7469,7 @@ var IntentClassifier = class {
|
|
|
7227
7469
|
if (typeof val === "number") {
|
|
7228
7470
|
numericKeys.add(key);
|
|
7229
7471
|
} else if (typeof val === "string") {
|
|
7230
|
-
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
|
|
7472
|
+
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
|
|
7231
7473
|
dateKeys.add(key);
|
|
7232
7474
|
} else if (val.length < 50 && !val.includes("\n")) {
|
|
7233
7475
|
catKeys.add(key);
|
|
@@ -7240,10 +7482,7 @@ var IntentClassifier = class {
|
|
|
7240
7482
|
hasCategoricalFields = catKeys.size > 0;
|
|
7241
7483
|
numericFieldCount = numericKeys.size;
|
|
7242
7484
|
categoricalFieldCount = catKeys.size;
|
|
7243
|
-
if (productKeyMatches >= 2
|
|
7244
|
-
var _a2, _b;
|
|
7245
|
-
return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
|
|
7246
|
-
})) {
|
|
7485
|
+
if (productKeyMatches >= 2) {
|
|
7247
7486
|
isProductLike = true;
|
|
7248
7487
|
}
|
|
7249
7488
|
}
|
|
@@ -7259,12 +7498,13 @@ var IntentClassifier = class {
|
|
|
7259
7498
|
}
|
|
7260
7499
|
// ─── Intent Heuristic Checkers ─────────────────────────────────────────────
|
|
7261
7500
|
static isInformationLookup(query) {
|
|
7262
|
-
|
|
7263
|
-
|
|
7501
|
+
const q = query.toLowerCase().trim();
|
|
7502
|
+
if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
|
|
7503
|
+
if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
|
|
7264
7504
|
return true;
|
|
7265
7505
|
}
|
|
7266
7506
|
}
|
|
7267
|
-
if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(
|
|
7507
|
+
if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
|
|
7268
7508
|
return true;
|
|
7269
7509
|
}
|
|
7270
7510
|
return false;
|
|
@@ -7279,13 +7519,13 @@ var IntentClassifier = class {
|
|
|
7279
7519
|
return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
|
|
7280
7520
|
}
|
|
7281
7521
|
static isTrendQuery(query) {
|
|
7282
|
-
return /\b(trend|trends|growth
|
|
7522
|
+
return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
|
|
7283
7523
|
}
|
|
7284
7524
|
static isRankingQuery(query) {
|
|
7285
7525
|
return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
|
|
7286
7526
|
}
|
|
7287
7527
|
static isAnalyticsQuery(query) {
|
|
7288
|
-
return /\b(
|
|
7528
|
+
return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
|
|
7289
7529
|
}
|
|
7290
7530
|
};
|
|
7291
7531
|
|
|
@@ -7297,8 +7537,8 @@ var Rule1SpecificInfoRule = class {
|
|
|
7297
7537
|
}
|
|
7298
7538
|
evaluate(context, intent) {
|
|
7299
7539
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7300
|
-
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
|
|
7301
|
-
const isNonVisual = !/\b(
|
|
7540
|
+
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
|
|
7541
|
+
const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
|
|
7302
7542
|
if (isFactQuery && isNonVisual) {
|
|
7303
7543
|
return {
|
|
7304
7544
|
renderType: "text",
|
|
@@ -7363,11 +7603,11 @@ var Rule4AnalyticalRule = class {
|
|
|
7363
7603
|
const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
|
|
7364
7604
|
if (!isAnalytical) return null;
|
|
7365
7605
|
let chartType = "bar";
|
|
7366
|
-
if (intent === "trend_analysis" || /\b(monthly|yearly|
|
|
7606
|
+
if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
|
|
7367
7607
|
chartType = "line";
|
|
7368
|
-
} else if (/\b(share|percentage|
|
|
7608
|
+
} else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
|
|
7369
7609
|
chartType = "pie";
|
|
7370
|
-
} else if (/\b(correlation|scatter|relationship
|
|
7610
|
+
} else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
|
|
7371
7611
|
chartType = "scatter";
|
|
7372
7612
|
} else {
|
|
7373
7613
|
chartType = "bar";
|
|
@@ -8138,12 +8378,20 @@ ${schemaProfileText}` : ""}`;
|
|
|
8138
8378
|
var _a3;
|
|
8139
8379
|
return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
|
|
8140
8380
|
}).filter(Boolean))) : this.detectCategories(data);
|
|
8141
|
-
if (categories.length
|
|
8381
|
+
if (categories.length <= 1) return null;
|
|
8142
8382
|
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
8383
|
+
const hasInventoryData = data.some((item) => {
|
|
8384
|
+
const meta = item.metadata || {};
|
|
8385
|
+
return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
|
|
8386
|
+
});
|
|
8143
8387
|
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
8388
|
+
if (!hasInventoryData) {
|
|
8389
|
+
return { label, value: count };
|
|
8390
|
+
}
|
|
8144
8391
|
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
8145
8392
|
return { label, value: count, inStockCount, outOfStockCount };
|
|
8146
8393
|
});
|
|
8394
|
+
if (pieData.length <= 1) return null;
|
|
8147
8395
|
return {
|
|
8148
8396
|
type: "pie_chart",
|
|
8149
8397
|
title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
|
|
@@ -8450,41 +8698,34 @@ ${schemaProfileText}` : ""}`;
|
|
|
8450
8698
|
}
|
|
8451
8699
|
static _productQueryTest(query) {
|
|
8452
8700
|
const q = query.toLowerCase();
|
|
8453
|
-
const
|
|
8701
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
|
|
8702
|
+
if (financialTerms.test(q)) {
|
|
8703
|
+
return false;
|
|
8704
|
+
}
|
|
8705
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
8454
8706
|
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
8455
8707
|
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
8456
8708
|
return productTerms && productAction && !nonProductEntityTerms;
|
|
8457
8709
|
}
|
|
8458
8710
|
static isProductData(item) {
|
|
8459
8711
|
if (!item) return false;
|
|
8460
|
-
const content = (item.content || "").toLowerCase();
|
|
8461
|
-
const productKeywords = [
|
|
8462
|
-
"product",
|
|
8463
|
-
"price",
|
|
8464
|
-
"stock",
|
|
8465
|
-
"sku",
|
|
8466
|
-
"brand",
|
|
8467
|
-
"msrp",
|
|
8468
|
-
"inventory",
|
|
8469
|
-
"buy",
|
|
8470
|
-
"shop",
|
|
8471
|
-
"beauty",
|
|
8472
|
-
"cosmetic",
|
|
8473
|
-
"facial",
|
|
8474
|
-
"cream",
|
|
8475
|
-
"serum",
|
|
8476
|
-
"mask",
|
|
8477
|
-
"makeup",
|
|
8478
|
-
"fragrance"
|
|
8479
|
-
];
|
|
8480
|
-
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
8481
8712
|
const metadata = item.metadata || {};
|
|
8482
|
-
const
|
|
8483
|
-
|
|
8484
|
-
return
|
|
8485
|
-
}
|
|
8486
|
-
|
|
8487
|
-
|
|
8713
|
+
const domain = String(metadata.domain || "").toLowerCase();
|
|
8714
|
+
if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
|
|
8715
|
+
return false;
|
|
8716
|
+
}
|
|
8717
|
+
if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
|
|
8718
|
+
return true;
|
|
8719
|
+
}
|
|
8720
|
+
const content = (item.content || "").toLowerCase();
|
|
8721
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
|
|
8722
|
+
if (financialTerms.test(content)) {
|
|
8723
|
+
return false;
|
|
8724
|
+
}
|
|
8725
|
+
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
|
|
8726
|
+
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
8727
|
+
const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
8728
|
+
return hasSku || hasProductIdentity && hasPrice;
|
|
8488
8729
|
}
|
|
8489
8730
|
static isTimeSeriesData(item) {
|
|
8490
8731
|
const metadata = item.metadata || {};
|
|
@@ -8503,10 +8744,31 @@ ${schemaProfileText}` : ""}`;
|
|
|
8503
8744
|
return fieldCount.size > 2;
|
|
8504
8745
|
}
|
|
8505
8746
|
static profileData(data) {
|
|
8747
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
8748
|
+
"filetype",
|
|
8749
|
+
"mimetype",
|
|
8750
|
+
"filename",
|
|
8751
|
+
"docid",
|
|
8752
|
+
"uploadedat",
|
|
8753
|
+
"dimension",
|
|
8754
|
+
"chunkindex",
|
|
8755
|
+
"totalchunks",
|
|
8756
|
+
"charactercount",
|
|
8757
|
+
"workspaceid",
|
|
8758
|
+
"pineconenamespace",
|
|
8759
|
+
"tier",
|
|
8760
|
+
"custommetadata",
|
|
8761
|
+
"csvheaders",
|
|
8762
|
+
"headers",
|
|
8763
|
+
"filesize",
|
|
8764
|
+
"content",
|
|
8765
|
+
"id"
|
|
8766
|
+
]);
|
|
8506
8767
|
const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
|
|
8507
8768
|
var _a2, _b;
|
|
8508
8769
|
const fields2 = {};
|
|
8509
8770
|
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
8771
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
8510
8772
|
const primitive = this.toPrimitive(value);
|
|
8511
8773
|
if (primitive !== null) fields2[key] = primitive;
|
|
8512
8774
|
});
|
|
@@ -9460,27 +9722,74 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9460
9722
|
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
9461
9723
|
*/
|
|
9462
9724
|
async ingest(documents, namespace) {
|
|
9725
|
+
var _a2;
|
|
9463
9726
|
await this.initialize();
|
|
9464
9727
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9465
9728
|
const results = [];
|
|
9729
|
+
if (!documents || documents.length === 0) return results;
|
|
9730
|
+
const docChunksMap = [];
|
|
9731
|
+
const allChunks = [];
|
|
9466
9732
|
for (const doc of documents) {
|
|
9467
9733
|
try {
|
|
9468
9734
|
const chunks = await this.prepareChunks(doc);
|
|
9469
|
-
|
|
9735
|
+
docChunksMap.push({ doc, chunks });
|
|
9736
|
+
allChunks.push(...chunks);
|
|
9737
|
+
} catch (error) {
|
|
9738
|
+
Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
|
|
9739
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9740
|
+
}
|
|
9741
|
+
}
|
|
9742
|
+
if (allChunks.length === 0) {
|
|
9743
|
+
return results;
|
|
9744
|
+
}
|
|
9745
|
+
try {
|
|
9746
|
+
const allVectors = await this.processEmbeddings(allChunks);
|
|
9747
|
+
let vectorOffset = 0;
|
|
9748
|
+
const allUpsertDocs = [];
|
|
9749
|
+
const docSuccessCount = /* @__PURE__ */ new Map();
|
|
9750
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9751
|
+
const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
|
|
9752
|
+
vectorOffset += chunks.length;
|
|
9470
9753
|
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9471
9754
|
id: chunk.id,
|
|
9472
|
-
vector:
|
|
9755
|
+
vector: docVectors[i],
|
|
9473
9756
|
content: chunk.content,
|
|
9474
9757
|
metadata: chunk.metadata
|
|
9475
9758
|
}));
|
|
9476
|
-
|
|
9477
|
-
|
|
9759
|
+
allUpsertDocs.push(...upsertDocs);
|
|
9760
|
+
docSuccessCount.set(doc.docId, chunks.length);
|
|
9761
|
+
}
|
|
9762
|
+
const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
|
|
9763
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9764
|
+
results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
|
|
9478
9765
|
if (this.graphDB && this.entityExtractor) {
|
|
9479
|
-
|
|
9766
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9767
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9768
|
+
});
|
|
9769
|
+
}
|
|
9770
|
+
}
|
|
9771
|
+
} catch (batchErr) {
|
|
9772
|
+
Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
|
|
9773
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9774
|
+
try {
|
|
9775
|
+
const vectors = await this.processEmbeddings(chunks);
|
|
9776
|
+
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9777
|
+
id: chunk.id,
|
|
9778
|
+
vector: vectors[i],
|
|
9779
|
+
content: chunk.content,
|
|
9780
|
+
metadata: chunk.metadata
|
|
9781
|
+
}));
|
|
9782
|
+
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
9783
|
+
results.push({ docId: doc.docId, chunksIngested: totalProcessed });
|
|
9784
|
+
if (this.graphDB && this.entityExtractor) {
|
|
9785
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9786
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9787
|
+
});
|
|
9788
|
+
}
|
|
9789
|
+
} catch (singleErr) {
|
|
9790
|
+
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
|
|
9791
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9480
9792
|
}
|
|
9481
|
-
} catch (error) {
|
|
9482
|
-
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
|
|
9483
|
-
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9484
9793
|
}
|
|
9485
9794
|
}
|
|
9486
9795
|
return results;
|
|
@@ -9500,19 +9809,40 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9500
9809
|
}
|
|
9501
9810
|
/**
|
|
9502
9811
|
* Step 2: Generate embeddings for chunks with retry logic.
|
|
9503
|
-
* Uses batchEmbed when available for efficiency; falls back to
|
|
9812
|
+
* Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
|
|
9504
9813
|
*/
|
|
9505
9814
|
async processEmbeddings(chunks) {
|
|
9506
|
-
|
|
9507
|
-
|
|
9508
|
-
|
|
9509
|
-
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9815
|
+
if (chunks.length === 0) return [];
|
|
9816
|
+
const texts = chunks.map((c) => c.content);
|
|
9817
|
+
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9818
|
+
try {
|
|
9819
|
+
const batchSize = 50;
|
|
9820
|
+
const allVectors = [];
|
|
9821
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
9822
|
+
const slice = texts.slice(i, i + batchSize);
|
|
9823
|
+
const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
|
|
9824
|
+
allVectors.push(...batchVectors);
|
|
9825
|
+
}
|
|
9826
|
+
if (allVectors.length === chunks.length) {
|
|
9827
|
+
return allVectors;
|
|
9828
|
+
}
|
|
9829
|
+
} catch (err) {
|
|
9830
|
+
Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
|
|
9831
|
+
error: err instanceof Error ? err.message : String(err)
|
|
9832
|
+
});
|
|
9833
|
+
}
|
|
9834
|
+
}
|
|
9835
|
+
const concurrentResult = await BatchProcessor.processConcurrent(
|
|
9836
|
+
texts,
|
|
9513
9837
|
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
9514
|
-
|
|
9838
|
+
10,
|
|
9839
|
+
{
|
|
9840
|
+
maxRetries: 3,
|
|
9841
|
+
initialDelayMs: 100,
|
|
9842
|
+
throwOnPartialFailure: false
|
|
9843
|
+
}
|
|
9515
9844
|
);
|
|
9845
|
+
const vectors = concurrentResult.results;
|
|
9516
9846
|
if (vectors.length !== chunks.length) {
|
|
9517
9847
|
throw new EmbeddingFailedException(
|
|
9518
9848
|
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
|
|
@@ -9539,25 +9869,24 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9539
9869
|
}
|
|
9540
9870
|
/** Step 4: Optional graph-based entity extraction and ingestion. */
|
|
9541
9871
|
async processGraphIngestion(docId, chunks) {
|
|
9542
|
-
const
|
|
9543
|
-
|
|
9544
|
-
maxRetries: 1,
|
|
9545
|
-
initialDelayMs: 500
|
|
9546
|
-
};
|
|
9547
|
-
await BatchProcessor.processBatch(
|
|
9872
|
+
const maxConcurrency = Math.min(3, chunks.length);
|
|
9873
|
+
await BatchProcessor.processConcurrent(
|
|
9548
9874
|
chunks,
|
|
9549
|
-
async (
|
|
9550
|
-
|
|
9551
|
-
|
|
9552
|
-
|
|
9553
|
-
|
|
9554
|
-
|
|
9555
|
-
}
|
|
9556
|
-
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9557
|
-
}
|
|
9875
|
+
async (chunk) => {
|
|
9876
|
+
try {
|
|
9877
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
9878
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
9879
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
9880
|
+
} catch (err) {
|
|
9881
|
+
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9558
9882
|
}
|
|
9559
9883
|
},
|
|
9560
|
-
|
|
9884
|
+
maxConcurrency,
|
|
9885
|
+
{
|
|
9886
|
+
maxRetries: 1,
|
|
9887
|
+
initialDelayMs: 200,
|
|
9888
|
+
throwOnPartialFailure: false
|
|
9889
|
+
}
|
|
9561
9890
|
);
|
|
9562
9891
|
}
|
|
9563
9892
|
async runNormalQuery(question, history = [], namespace) {
|
|
@@ -9682,7 +10011,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9682
10011
|
*/
|
|
9683
10012
|
askStreamInternal(_0) {
|
|
9684
10013
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
9685
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
|
|
10014
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
|
|
9686
10015
|
yield new __await(this.initialize());
|
|
9687
10016
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9688
10017
|
const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
|
|
@@ -9690,29 +10019,35 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9690
10019
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
9691
10020
|
const requestStart = performance.now();
|
|
9692
10021
|
try {
|
|
9693
|
-
let
|
|
10022
|
+
let effectiveQuestion = question;
|
|
10023
|
+
let promptOptResult;
|
|
10024
|
+
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
10025
|
+
promptOptResult = PromptOptimizer.optimize(question, history);
|
|
10026
|
+
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
10027
|
+
}
|
|
10028
|
+
let searchQuery = effectiveQuestion;
|
|
9694
10029
|
let rewrittenQuery;
|
|
9695
|
-
if ((
|
|
9696
|
-
searchQuery = yield new __await(this.rewriteQuery(
|
|
9697
|
-
rewrittenQuery = searchQuery !==
|
|
10030
|
+
if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
|
|
10031
|
+
searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
|
|
10032
|
+
rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
|
|
9698
10033
|
}
|
|
9699
|
-
const hints = QueryProcessor.extractQueryFieldHints(
|
|
9700
|
-
const filter = QueryProcessor.buildQueryFilter(
|
|
9701
|
-
const numericPredicates = QueryProcessor.extractNumericPredicates(
|
|
10034
|
+
const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
|
|
10035
|
+
const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
|
|
10036
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
|
|
9702
10037
|
if (numericPredicates.length > 0) {
|
|
9703
10038
|
filter.__numericPredicates = numericPredicates;
|
|
9704
10039
|
}
|
|
9705
10040
|
const embedStart = performance.now();
|
|
9706
10041
|
const cacheKey = `${ns}::${searchQuery}`;
|
|
9707
10042
|
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
9708
|
-
const arch = (
|
|
9709
|
-
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((
|
|
10043
|
+
const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
|
|
10044
|
+
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
|
|
9710
10045
|
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
9711
10046
|
QueryProcessor.determineRetrievalStrategy(
|
|
9712
10047
|
searchQuery,
|
|
9713
10048
|
useGraph ? this.llmRouter.get("fast") : void 0,
|
|
9714
|
-
(
|
|
9715
|
-
(
|
|
10049
|
+
(_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
|
|
10050
|
+
(_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
|
|
9716
10051
|
),
|
|
9717
10052
|
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
9718
10053
|
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
@@ -9722,7 +10057,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9722
10057
|
if (!cachedVector && queryVector.length > 0) {
|
|
9723
10058
|
this.embeddingCache.set(cacheKey, queryVector);
|
|
9724
10059
|
}
|
|
9725
|
-
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((
|
|
10060
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
9726
10061
|
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
9727
10062
|
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
9728
10063
|
const wantsExhaustiveList = true;
|
|
@@ -9739,7 +10074,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9739
10074
|
return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
|
|
9740
10075
|
});
|
|
9741
10076
|
const rerankLimit = Math.max(retrievalLimit, fullSources.length);
|
|
9742
|
-
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((
|
|
10077
|
+
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
|
|
9743
10078
|
if (!hasNumericPredicates && useReranking) {
|
|
9744
10079
|
fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
|
|
9745
10080
|
}
|
|
@@ -9767,7 +10102,7 @@ ${graphContext}
|
|
|
9767
10102
|
VECTOR CONTEXT:
|
|
9768
10103
|
${context}`;
|
|
9769
10104
|
}
|
|
9770
|
-
if ((
|
|
10105
|
+
if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
|
|
9771
10106
|
if (!this.coldContexts.has(ns)) {
|
|
9772
10107
|
yield new __await(this.loadColdContext(ns));
|
|
9773
10108
|
}
|
|
@@ -9799,31 +10134,31 @@ ${context}`;
|
|
|
9799
10134
|
SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
|
|
9800
10135
|
});
|
|
9801
10136
|
}
|
|
9802
|
-
const isProductQ = UITransformer.isProductQueryPublic(
|
|
10137
|
+
const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
|
|
9803
10138
|
const uiTransformationPromise = isProductQ ? Promise.resolve(
|
|
9804
10139
|
UITransformer.transform(
|
|
9805
|
-
|
|
10140
|
+
effectiveQuestion,
|
|
9806
10141
|
sources,
|
|
9807
10142
|
this.config,
|
|
9808
10143
|
cachedSchema,
|
|
9809
10144
|
{ visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
|
|
9810
10145
|
)
|
|
9811
10146
|
) : this.generateUiTransformation(
|
|
9812
|
-
|
|
10147
|
+
effectiveQuestion,
|
|
9813
10148
|
sources,
|
|
9814
10149
|
cachedSchema,
|
|
9815
10150
|
hasNumericPredicates
|
|
9816
10151
|
).catch((uiError) => {
|
|
9817
10152
|
Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
|
|
9818
|
-
return UITransformer.transform(
|
|
10153
|
+
return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
|
|
9819
10154
|
});
|
|
9820
10155
|
let hallucinationScoringPromise = Promise.resolve(void 0);
|
|
9821
10156
|
const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
9822
10157
|
const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
|
|
9823
|
-
const isNativeThinking = isClaude37 && (((
|
|
10158
|
+
const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
|
|
9824
10159
|
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
9825
10160
|
const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
|
|
9826
|
-
const isSimulatedThinking = !isNativeThinking && (((
|
|
10161
|
+
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
|
|
9827
10162
|
let finalRestrictionSuffix = restrictionSuffix;
|
|
9828
10163
|
if (isSimulatedThinking) {
|
|
9829
10164
|
finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
|
|
@@ -9832,9 +10167,9 @@ ${context}`;
|
|
|
9832
10167
|
role: m.role || "user",
|
|
9833
10168
|
content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
|
|
9834
10169
|
}));
|
|
9835
|
-
const userQuestion = { role: "user", content:
|
|
10170
|
+
const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
|
|
9836
10171
|
const messages = [...hardenedHistory, userQuestion];
|
|
9837
|
-
const systemPrompt = (
|
|
10172
|
+
const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
|
|
9838
10173
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
9839
10174
|
let fullReply = "";
|
|
9840
10175
|
let thinkingText = "";
|
|
@@ -9945,7 +10280,7 @@ ${context}`;
|
|
|
9945
10280
|
}
|
|
9946
10281
|
yield fullReply;
|
|
9947
10282
|
}
|
|
9948
|
-
const runHallucination = ((
|
|
10283
|
+
const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
|
|
9949
10284
|
if (runHallucination) {
|
|
9950
10285
|
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
|
|
9951
10286
|
}
|
|
@@ -9967,7 +10302,7 @@ ${context}`;
|
|
|
9967
10302
|
totalTokens: promptTokens + completionTokens,
|
|
9968
10303
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
9969
10304
|
};
|
|
9970
|
-
const awaitHallucination = ((
|
|
10305
|
+
const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
|
|
9971
10306
|
const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
|
|
9972
10307
|
uiTransformationPromise,
|
|
9973
10308
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
@@ -9975,9 +10310,9 @@ ${context}`;
|
|
|
9975
10310
|
const buildTrace = (hScore) => __spreadValues({
|
|
9976
10311
|
requestId,
|
|
9977
10312
|
query: question,
|
|
9978
|
-
rewrittenQuery,
|
|
10313
|
+
rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
|
|
9979
10314
|
systemPrompt,
|
|
9980
|
-
userPrompt:
|
|
10315
|
+
userPrompt: effectiveQuestion + finalRestrictionSuffix,
|
|
9981
10316
|
chunks: (sources || []).filter(Boolean).map((s) => {
|
|
9982
10317
|
var _a3, _b2;
|
|
9983
10318
|
return {
|
|
@@ -9996,10 +10331,10 @@ ${context}`;
|
|
|
9996
10331
|
hallucinationReason: hScore.reason
|
|
9997
10332
|
} : {});
|
|
9998
10333
|
const trace = buildTrace(hallucinationResult);
|
|
9999
|
-
const isTelemetryActive = (
|
|
10334
|
+
const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
|
|
10000
10335
|
if (isTelemetryActive) {
|
|
10001
10336
|
const defaultUrl = "https://www.retrivora.com/api/telemetry";
|
|
10002
|
-
const telemetryUrl = ((
|
|
10337
|
+
const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
10003
10338
|
const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
|
|
10004
10339
|
(async () => {
|
|
10005
10340
|
var _a3, _b2, _c2, _d2, _e2;
|
|
@@ -10862,13 +11197,81 @@ function createFromPreset(presetName) {
|
|
|
10862
11197
|
|
|
10863
11198
|
// src/utils/DocumentParser.ts
|
|
10864
11199
|
var DocumentParser = class {
|
|
11200
|
+
/**
|
|
11201
|
+
* Fast Magic-Bytes sniffer for instant file format identification (<1ms).
|
|
11202
|
+
*/
|
|
11203
|
+
static sniffMagicBytes(buffer) {
|
|
11204
|
+
if (!buffer || buffer.length < 4) return null;
|
|
11205
|
+
if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
|
|
11206
|
+
return "pdf";
|
|
11207
|
+
}
|
|
11208
|
+
if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
|
|
11209
|
+
return "zip";
|
|
11210
|
+
}
|
|
11211
|
+
if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
|
|
11212
|
+
return "ole2";
|
|
11213
|
+
}
|
|
11214
|
+
const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
|
|
11215
|
+
if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
|
|
11216
|
+
return "json";
|
|
11217
|
+
}
|
|
11218
|
+
return "text";
|
|
11219
|
+
}
|
|
11220
|
+
/**
|
|
11221
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
11222
|
+
* Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
|
|
11223
|
+
*/
|
|
11224
|
+
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
11225
|
+
const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
|
|
11226
|
+
if (!jsonRows || jsonRows.length === 0) {
|
|
11227
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11228
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11229
|
+
${csv.trim()}` : "";
|
|
11230
|
+
}
|
|
11231
|
+
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
11232
|
+
let currentHeader = [];
|
|
11233
|
+
for (let r = 0; r < jsonRows.length; r++) {
|
|
11234
|
+
const row = jsonRows[r];
|
|
11235
|
+
if (!row || row.length === 0) continue;
|
|
11236
|
+
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
11237
|
+
if (cells.length === 0) continue;
|
|
11238
|
+
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
11239
|
+
lines.push(`
|
|
11240
|
+
## ${cells.join(" ")}`);
|
|
11241
|
+
continue;
|
|
11242
|
+
}
|
|
11243
|
+
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
11244
|
+
if (isHeaderCandidate && cells.length >= 2) {
|
|
11245
|
+
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
11246
|
+
lines.push(`| ${cells.join(" | ")} |`);
|
|
11247
|
+
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
11248
|
+
continue;
|
|
11249
|
+
}
|
|
11250
|
+
if (currentHeader.length > 0 && cells.length > 1) {
|
|
11251
|
+
const fieldPairs = [];
|
|
11252
|
+
for (let c = 0; c < row.length; c++) {
|
|
11253
|
+
const h = currentHeader[c];
|
|
11254
|
+
const val = row[c];
|
|
11255
|
+
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
11256
|
+
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
11257
|
+
}
|
|
11258
|
+
}
|
|
11259
|
+
if (fieldPairs.length > 0) {
|
|
11260
|
+
lines.push(fieldPairs.join(" | "));
|
|
11261
|
+
continue;
|
|
11262
|
+
}
|
|
11263
|
+
}
|
|
11264
|
+
lines.push(cells.join(", "));
|
|
11265
|
+
}
|
|
11266
|
+
return lines.join("\n");
|
|
11267
|
+
}
|
|
10865
11268
|
/**
|
|
10866
11269
|
* Extract text from a File or Buffer based on its type.
|
|
10867
11270
|
*/
|
|
10868
|
-
static async parse(file, fileName, mimeType) {
|
|
11271
|
+
static async parse(file, fileName, mimeType = "") {
|
|
10869
11272
|
var _a2;
|
|
10870
11273
|
const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
10871
|
-
if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
11274
|
+
if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
10872
11275
|
return this.readAsText(file);
|
|
10873
11276
|
}
|
|
10874
11277
|
if (extension === "json" || mimeType === "application/json") {
|
|
@@ -10880,39 +11283,116 @@ var DocumentParser = class {
|
|
|
10880
11283
|
return text;
|
|
10881
11284
|
}
|
|
10882
11285
|
}
|
|
10883
|
-
if (extension === "
|
|
11286
|
+
if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
|
|
10884
11287
|
return this.readAsText(file);
|
|
10885
11288
|
}
|
|
10886
|
-
if (extension === "
|
|
11289
|
+
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
11290
|
+
return this.readAsText(file);
|
|
11291
|
+
}
|
|
11292
|
+
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
10887
11293
|
try {
|
|
10888
|
-
const
|
|
11294
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
10889
11295
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10890
|
-
const
|
|
10891
|
-
|
|
10892
|
-
|
|
10893
|
-
|
|
10894
|
-
const
|
|
10895
|
-
|
|
10896
|
-
|
|
10897
|
-
|
|
10898
|
-
|
|
11296
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
11297
|
+
const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
|
|
11298
|
+
var _a3, _b;
|
|
11299
|
+
const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
|
|
11300
|
+
const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
|
|
11301
|
+
return numA - numB;
|
|
11302
|
+
});
|
|
11303
|
+
if (slideFiles.length > 0) {
|
|
11304
|
+
const slidesText = [];
|
|
11305
|
+
for (let i = 0; i < slideFiles.length; i++) {
|
|
11306
|
+
const slideXml = await zip.files[slideFiles[i]].async("text");
|
|
11307
|
+
const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
|
|
11308
|
+
if (textMatches && textMatches.length > 0) {
|
|
11309
|
+
const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
|
|
11310
|
+
if (slideContent.trim()) {
|
|
11311
|
+
slidesText.push(`--- Slide ${i + 1} ---
|
|
11312
|
+
${slideContent.trim()}`);
|
|
11313
|
+
}
|
|
10899
11314
|
}
|
|
10900
|
-
});
|
|
10901
|
-
const csv = csvLines.join("\n");
|
|
10902
|
-
if (csv && csv.trim()) {
|
|
10903
|
-
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
10904
|
-
${csv.trim()}`);
|
|
10905
11315
|
}
|
|
10906
|
-
|
|
10907
|
-
|
|
10908
|
-
|
|
11316
|
+
if (slidesText.length > 0) {
|
|
11317
|
+
return slidesText.join("\n\n");
|
|
11318
|
+
}
|
|
11319
|
+
}
|
|
11320
|
+
if (zip.files["content.xml"]) {
|
|
11321
|
+
const contentXml = await zip.files["content.xml"].async("text");
|
|
11322
|
+
const cleanText = this.stripXmlTags(contentXml);
|
|
11323
|
+
if (cleanText.trim()) {
|
|
11324
|
+
return cleanText.trim();
|
|
11325
|
+
}
|
|
11326
|
+
}
|
|
11327
|
+
const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
|
|
11328
|
+
const extractedSegments = [];
|
|
11329
|
+
for (const xf of allXmlFiles) {
|
|
11330
|
+
const xml = await zip.files[xf].async("text");
|
|
11331
|
+
const stripped = this.stripXmlTags(xml);
|
|
11332
|
+
if (stripped.length > 10) extractedSegments.push(stripped);
|
|
11333
|
+
}
|
|
11334
|
+
if (extractedSegments.length > 0) {
|
|
11335
|
+
return extractedSegments.join("\n\n");
|
|
11336
|
+
}
|
|
11337
|
+
return `[PowerPoint Presentation: ${fileName}]`;
|
|
11338
|
+
} catch (err) {
|
|
11339
|
+
console.warn("[DocumentParser] PowerPoint parsing failed:", err);
|
|
10909
11340
|
try {
|
|
10910
11341
|
return await this.readAsText(file);
|
|
10911
|
-
} catch (
|
|
10912
|
-
return `[
|
|
11342
|
+
} catch (e) {
|
|
11343
|
+
return `[PowerPoint Presentation File: ${fileName}]`;
|
|
10913
11344
|
}
|
|
10914
11345
|
}
|
|
10915
11346
|
}
|
|
11347
|
+
if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
|
|
11348
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11349
|
+
try {
|
|
11350
|
+
const XLSX = await import("xlsx");
|
|
11351
|
+
const workbook = XLSX.read(buffer, { type: "buffer" });
|
|
11352
|
+
const sheetsText = [];
|
|
11353
|
+
for (const sheetName of workbook.SheetNames) {
|
|
11354
|
+
const sheet = workbook.Sheets[sheetName];
|
|
11355
|
+
const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
|
|
11356
|
+
if (formatted && formatted.trim()) {
|
|
11357
|
+
sheetsText.push(formatted.trim());
|
|
11358
|
+
}
|
|
11359
|
+
}
|
|
11360
|
+
if (sheetsText.length > 0) {
|
|
11361
|
+
return sheetsText.join("\n\n");
|
|
11362
|
+
}
|
|
11363
|
+
} catch (xlsxErr) {
|
|
11364
|
+
console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
|
|
11365
|
+
}
|
|
11366
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
11367
|
+
if (isZip) {
|
|
11368
|
+
try {
|
|
11369
|
+
const ExcelJS = await import("exceljs");
|
|
11370
|
+
const workbook = new ExcelJS.Workbook();
|
|
11371
|
+
await workbook.xlsx.load(buffer);
|
|
11372
|
+
const sheetsText = [];
|
|
11373
|
+
workbook.eachSheet((sheet) => {
|
|
11374
|
+
const csvLines = [];
|
|
11375
|
+
sheet.eachRow((row) => {
|
|
11376
|
+
const values = row.values;
|
|
11377
|
+
if (Array.isArray(values)) {
|
|
11378
|
+
csvLines.push(values.slice(1).join(","));
|
|
11379
|
+
}
|
|
11380
|
+
});
|
|
11381
|
+
const csv = csvLines.join("\n");
|
|
11382
|
+
if (csv && csv.trim()) {
|
|
11383
|
+
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
11384
|
+
${csv.trim()}`);
|
|
11385
|
+
}
|
|
11386
|
+
});
|
|
11387
|
+
if (sheetsText.length > 0) {
|
|
11388
|
+
return sheetsText.join("\n\n");
|
|
11389
|
+
}
|
|
11390
|
+
} catch (err) {
|
|
11391
|
+
console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
|
|
11392
|
+
}
|
|
11393
|
+
}
|
|
11394
|
+
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
11395
|
+
}
|
|
10916
11396
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
10917
11397
|
try {
|
|
10918
11398
|
const pdfModule = await import("pdf-parse");
|
|
@@ -10925,15 +11405,48 @@ ${csv.trim()}`);
|
|
|
10925
11405
|
return `[PDF Parsing Error for ${fileName}]`;
|
|
10926
11406
|
}
|
|
10927
11407
|
}
|
|
10928
|
-
if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
|
|
11408
|
+
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
10929
11409
|
try {
|
|
10930
11410
|
const mammoth = await import("mammoth");
|
|
10931
11411
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10932
11412
|
const result = await mammoth.extractRawText({ buffer });
|
|
10933
|
-
|
|
11413
|
+
if (result.value && result.value.trim()) {
|
|
11414
|
+
return result.value;
|
|
11415
|
+
}
|
|
11416
|
+
} catch (err) {
|
|
11417
|
+
console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
|
|
11418
|
+
}
|
|
11419
|
+
try {
|
|
11420
|
+
const raw = await this.readAsText(file);
|
|
11421
|
+
if (extension === "rtf" || raw.startsWith("{\\rtf")) {
|
|
11422
|
+
return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
|
|
11423
|
+
}
|
|
11424
|
+
return raw;
|
|
11425
|
+
} catch (e) {
|
|
11426
|
+
return `[Word Document File: ${fileName}]`;
|
|
11427
|
+
}
|
|
11428
|
+
}
|
|
11429
|
+
if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
|
|
11430
|
+
const raw = await this.readAsText(file);
|
|
11431
|
+
return this.stripHtmlAndXml(raw);
|
|
11432
|
+
}
|
|
11433
|
+
if (extension === "epub" || mimeType.includes("epub")) {
|
|
11434
|
+
try {
|
|
11435
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
11436
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11437
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
11438
|
+
const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
|
|
11439
|
+
const chapters = [];
|
|
11440
|
+
for (const hf of htmlFiles) {
|
|
11441
|
+
const content = await zip.files[hf].async("text");
|
|
11442
|
+
const clean = this.stripHtmlAndXml(content);
|
|
11443
|
+
if (clean.length > 50) chapters.push(clean);
|
|
11444
|
+
}
|
|
11445
|
+
if (chapters.length > 0) {
|
|
11446
|
+
return chapters.join("\n\n---\n\n");
|
|
11447
|
+
}
|
|
10934
11448
|
} catch (err) {
|
|
10935
|
-
console.warn(
|
|
10936
|
-
return `[Word Parsing Error for ${fileName}]`;
|
|
11449
|
+
console.warn("[DocumentParser] EPUB parsing failed:", err);
|
|
10937
11450
|
}
|
|
10938
11451
|
}
|
|
10939
11452
|
try {
|
|
@@ -10942,6 +11455,237 @@ ${csv.trim()}`);
|
|
|
10942
11455
|
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
10943
11456
|
}
|
|
10944
11457
|
}
|
|
11458
|
+
/**
|
|
11459
|
+
* Helper to strip XML tags and normalize inner text content.
|
|
11460
|
+
*/
|
|
11461
|
+
static stripXmlTags(xml) {
|
|
11462
|
+
return xml.replace(/<[^>]+>/g, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/\s+/g, " ").trim();
|
|
11463
|
+
}
|
|
11464
|
+
/**
|
|
11465
|
+
* Helper to strip HTML / XML tags while removing script and style blocks completely.
|
|
11466
|
+
*/
|
|
11467
|
+
static stripHtmlAndXml(html) {
|
|
11468
|
+
return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/ /gi, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
|
|
11469
|
+
}
|
|
11470
|
+
/**
|
|
11471
|
+
* Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
|
|
11472
|
+
*/
|
|
11473
|
+
static parseLegacyExcelOrFallback(buffer, fileName) {
|
|
11474
|
+
const rawStr = buffer.toString("utf-8");
|
|
11475
|
+
if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
|
|
11476
|
+
const rows = [];
|
|
11477
|
+
const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
|
|
11478
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11479
|
+
for (const rowHtml of rowMatches) {
|
|
11480
|
+
const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
|
|
11481
|
+
if (cells && cells.length > 0) {
|
|
11482
|
+
const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
|
|
11483
|
+
if (rowText) rows.push(rowText);
|
|
11484
|
+
}
|
|
11485
|
+
}
|
|
11486
|
+
}
|
|
11487
|
+
if (rows.length > 0) {
|
|
11488
|
+
return rows.join("\n");
|
|
11489
|
+
}
|
|
11490
|
+
const clean = this.stripHtmlAndXml(rawStr);
|
|
11491
|
+
if (clean.length > 10) return clean;
|
|
11492
|
+
}
|
|
11493
|
+
if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
|
|
11494
|
+
const rows = [];
|
|
11495
|
+
const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
|
|
11496
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11497
|
+
for (const rowXml of rowMatches) {
|
|
11498
|
+
const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
|
|
11499
|
+
if (dataMatches && dataMatches.length > 0) {
|
|
11500
|
+
const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
|
|
11501
|
+
if (rowText) rows.push(rowText);
|
|
11502
|
+
}
|
|
11503
|
+
}
|
|
11504
|
+
}
|
|
11505
|
+
if (rows.length > 0) {
|
|
11506
|
+
return rows.join("\n");
|
|
11507
|
+
}
|
|
11508
|
+
}
|
|
11509
|
+
const extractedStrings = [];
|
|
11510
|
+
const ignoredOleMarkers = /* @__PURE__ */ new Set([
|
|
11511
|
+
"root entry",
|
|
11512
|
+
"workbook",
|
|
11513
|
+
"compobj",
|
|
11514
|
+
"summaryinformation",
|
|
11515
|
+
"documentsummaryinformation",
|
|
11516
|
+
"microsoft excel",
|
|
11517
|
+
"worksheet",
|
|
11518
|
+
"biff8",
|
|
11519
|
+
"calcpr"
|
|
11520
|
+
]);
|
|
11521
|
+
let currentUtf16 = "";
|
|
11522
|
+
for (let i = 0; i < buffer.length - 1; i += 2) {
|
|
11523
|
+
const charCode = buffer[i];
|
|
11524
|
+
const zeroByte = buffer[i + 1];
|
|
11525
|
+
if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
|
|
11526
|
+
currentUtf16 += String.fromCharCode(charCode);
|
|
11527
|
+
} else {
|
|
11528
|
+
if (currentUtf16.length >= 2) {
|
|
11529
|
+
const trimmed = currentUtf16.trim();
|
|
11530
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11531
|
+
extractedStrings.push(trimmed);
|
|
11532
|
+
}
|
|
11533
|
+
}
|
|
11534
|
+
currentUtf16 = "";
|
|
11535
|
+
}
|
|
11536
|
+
}
|
|
11537
|
+
let currentAscii = "";
|
|
11538
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
11539
|
+
const byte = buffer[i];
|
|
11540
|
+
if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
|
|
11541
|
+
currentAscii += String.fromCharCode(byte);
|
|
11542
|
+
} else {
|
|
11543
|
+
if (currentAscii.length >= 2) {
|
|
11544
|
+
const trimmed = currentAscii.trim();
|
|
11545
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11546
|
+
extractedStrings.push(trimmed);
|
|
11547
|
+
}
|
|
11548
|
+
}
|
|
11549
|
+
currentAscii = "";
|
|
11550
|
+
}
|
|
11551
|
+
}
|
|
11552
|
+
const uniqueStrings = [];
|
|
11553
|
+
for (const str of extractedStrings) {
|
|
11554
|
+
if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
|
|
11555
|
+
uniqueStrings.push(str);
|
|
11556
|
+
}
|
|
11557
|
+
}
|
|
11558
|
+
if (uniqueStrings.length > 0) {
|
|
11559
|
+
return uniqueStrings.join(" | ");
|
|
11560
|
+
}
|
|
11561
|
+
const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
|
|
11562
|
+
return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
|
|
11563
|
+
}
|
|
11564
|
+
/**
|
|
11565
|
+
* Dynamically infer document domain and schema from extracted text, filename, and format.
|
|
11566
|
+
*/
|
|
11567
|
+
static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
|
|
11568
|
+
const lowerText = text.toLowerCase();
|
|
11569
|
+
const lowerName = fileName.toLowerCase();
|
|
11570
|
+
const financialTerms = [
|
|
11571
|
+
"price per share",
|
|
11572
|
+
"share price",
|
|
11573
|
+
"total cost of the deal",
|
|
11574
|
+
"cost of the deal",
|
|
11575
|
+
"lbo",
|
|
11576
|
+
"merger",
|
|
11577
|
+
"valuation",
|
|
11578
|
+
"d/e ratio",
|
|
11579
|
+
"ebit",
|
|
11580
|
+
"ebitda",
|
|
11581
|
+
"fcfe",
|
|
11582
|
+
"fcff",
|
|
11583
|
+
"wacc",
|
|
11584
|
+
"beta",
|
|
11585
|
+
"working capital",
|
|
11586
|
+
"cogs",
|
|
11587
|
+
"cap spending",
|
|
11588
|
+
"depreciation",
|
|
11589
|
+
"cashflow",
|
|
11590
|
+
"cash flow",
|
|
11591
|
+
"net income",
|
|
11592
|
+
"balance sheet",
|
|
11593
|
+
"income statement",
|
|
11594
|
+
"interest rate",
|
|
11595
|
+
"terminal year",
|
|
11596
|
+
"equity",
|
|
11597
|
+
"debt outstanding",
|
|
11598
|
+
"gross margin",
|
|
11599
|
+
"fiscal year",
|
|
11600
|
+
"fy20",
|
|
11601
|
+
"fy21",
|
|
11602
|
+
"fy22",
|
|
11603
|
+
"fy23",
|
|
11604
|
+
"fy24",
|
|
11605
|
+
"fy25",
|
|
11606
|
+
"q1",
|
|
11607
|
+
"q2",
|
|
11608
|
+
"q3",
|
|
11609
|
+
"q4"
|
|
11610
|
+
];
|
|
11611
|
+
let financialScore = 0;
|
|
11612
|
+
for (const term of financialTerms) {
|
|
11613
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11614
|
+
financialScore += 1;
|
|
11615
|
+
}
|
|
11616
|
+
}
|
|
11617
|
+
const ecommerceTerms = [
|
|
11618
|
+
"sku",
|
|
11619
|
+
"product_id",
|
|
11620
|
+
"product id",
|
|
11621
|
+
"item_id",
|
|
11622
|
+
"add to cart",
|
|
11623
|
+
"buy now",
|
|
11624
|
+
"in stock",
|
|
11625
|
+
"out of stock",
|
|
11626
|
+
"inventory count",
|
|
11627
|
+
"retail price",
|
|
11628
|
+
"unit price",
|
|
11629
|
+
"msrp",
|
|
11630
|
+
"cart",
|
|
11631
|
+
"shop",
|
|
11632
|
+
"catalog",
|
|
11633
|
+
"product_name",
|
|
11634
|
+
"product name",
|
|
11635
|
+
"brand name"
|
|
11636
|
+
];
|
|
11637
|
+
let ecommerceScore = 0;
|
|
11638
|
+
for (const term of ecommerceTerms) {
|
|
11639
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11640
|
+
ecommerceScore += 1;
|
|
11641
|
+
}
|
|
11642
|
+
}
|
|
11643
|
+
const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
|
|
11644
|
+
let legalScore = 0;
|
|
11645
|
+
for (const term of legalTerms) {
|
|
11646
|
+
if (lowerText.includes(term)) legalScore += 1;
|
|
11647
|
+
}
|
|
11648
|
+
const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
|
|
11649
|
+
let healthScore = 0;
|
|
11650
|
+
for (const term of healthTerms) {
|
|
11651
|
+
if (lowerText.includes(term)) healthScore += 1;
|
|
11652
|
+
}
|
|
11653
|
+
let schema;
|
|
11654
|
+
const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
11655
|
+
if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
|
|
11656
|
+
const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
|
|
11657
|
+
const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
|
|
11658
|
+
if (headers.length > 1 && lines.length > 1) {
|
|
11659
|
+
const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
|
|
11660
|
+
schema = {};
|
|
11661
|
+
headers.forEach((h, idx) => {
|
|
11662
|
+
const val = sampleRow[idx] || "";
|
|
11663
|
+
if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
|
|
11664
|
+
schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
|
|
11665
|
+
} else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
|
|
11666
|
+
schema[h] = "date";
|
|
11667
|
+
} else {
|
|
11668
|
+
schema[h] = "string";
|
|
11669
|
+
}
|
|
11670
|
+
});
|
|
11671
|
+
}
|
|
11672
|
+
}
|
|
11673
|
+
const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
|
|
11674
|
+
const isCatalog = ecommerceScore >= 2 && !isFinancial;
|
|
11675
|
+
let domain = "general";
|
|
11676
|
+
if (isFinancial) {
|
|
11677
|
+
domain = "finance";
|
|
11678
|
+
} else if (isCatalog) {
|
|
11679
|
+
domain = "ecommerce";
|
|
11680
|
+
} else if (legalScore >= 2) {
|
|
11681
|
+
domain = "legal";
|
|
11682
|
+
} else if (healthScore >= 2) {
|
|
11683
|
+
domain = "healthcare";
|
|
11684
|
+
} else if (schema && Object.keys(schema).length > 0) {
|
|
11685
|
+
domain = "tabular";
|
|
11686
|
+
}
|
|
11687
|
+
return { domain, schema, isCatalog, isFinancial };
|
|
11688
|
+
}
|
|
10945
11689
|
static async readAsText(file) {
|
|
10946
11690
|
if (Buffer.isBuffer(file)) {
|
|
10947
11691
|
return file.toString("utf-8");
|
|
@@ -11604,18 +12348,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
|
|
|
11604
12348
|
}
|
|
11605
12349
|
}
|
|
11606
12350
|
function getAllowedNamespaces(projectId) {
|
|
11607
|
-
const trimmed = projectId.trim();
|
|
12351
|
+
const trimmed = (projectId || "default").trim();
|
|
11608
12352
|
const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
|
|
11609
12353
|
return new Set([
|
|
11610
12354
|
trimmed,
|
|
11611
12355
|
withoutPrefix,
|
|
11612
|
-
`retrivora-${withoutPrefix}
|
|
12356
|
+
`retrivora-${withoutPrefix}`,
|
|
12357
|
+
"default",
|
|
12358
|
+
"retrivora-default"
|
|
11613
12359
|
].filter(Boolean));
|
|
11614
12360
|
}
|
|
11615
12361
|
function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
12362
|
+
var _a2;
|
|
11616
12363
|
const allowed = getAllowedNamespaces(payload.projectId);
|
|
11617
12364
|
const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
|
|
11618
|
-
|
|
12365
|
+
const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
|
|
12366
|
+
if (requested && !isDevOrFree && !allowed.has(requested)) {
|
|
11619
12367
|
return {
|
|
11620
12368
|
ok: false,
|
|
11621
12369
|
response: new Response(JSON.stringify({
|
|
@@ -11629,7 +12377,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
|
11629
12377
|
})
|
|
11630
12378
|
};
|
|
11631
12379
|
}
|
|
11632
|
-
const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
|
|
12380
|
+
const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
|
|
11633
12381
|
return { ok: true, namespace: requested || defaultNamespace };
|
|
11634
12382
|
}
|
|
11635
12383
|
var MAX_MESSAGE_LENGTH = 8e3;
|
|
@@ -12073,7 +12821,7 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
12073
12821
|
function createHealthHandler(configOrPlugin, options) {
|
|
12074
12822
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
12075
12823
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
12076
|
-
return async function GET(req) {
|
|
12824
|
+
return async function GET(req, context) {
|
|
12077
12825
|
if (req) {
|
|
12078
12826
|
const authResult = await checkAuth(req, onAuthorize);
|
|
12079
12827
|
if (authResult) return authResult;
|
|
@@ -12184,9 +12932,10 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12184
12932
|
}
|
|
12185
12933
|
const documents = [];
|
|
12186
12934
|
for (const file of files) {
|
|
12935
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
12187
12936
|
const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
|
|
12188
12937
|
if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
|
|
12189
|
-
const text =
|
|
12938
|
+
const text = buffer.toString("utf-8");
|
|
12190
12939
|
const Papa = await import("papaparse");
|
|
12191
12940
|
const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
|
|
12192
12941
|
if (parsed.data && parsed.data.length > 0) {
|
|
@@ -12226,49 +12975,70 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12226
12975
|
}
|
|
12227
12976
|
}
|
|
12228
12977
|
} else if (isExcel) {
|
|
12229
|
-
|
|
12230
|
-
|
|
12231
|
-
|
|
12232
|
-
|
|
12233
|
-
|
|
12234
|
-
|
|
12235
|
-
|
|
12236
|
-
|
|
12237
|
-
|
|
12238
|
-
|
|
12239
|
-
|
|
12240
|
-
|
|
12241
|
-
|
|
12242
|
-
|
|
12243
|
-
|
|
12244
|
-
|
|
12245
|
-
|
|
12246
|
-
|
|
12247
|
-
|
|
12248
|
-
|
|
12249
|
-
|
|
12250
|
-
|
|
12251
|
-
|
|
12252
|
-
|
|
12253
|
-
|
|
12254
|
-
|
|
12255
|
-
|
|
12256
|
-
|
|
12257
|
-
|
|
12258
|
-
|
|
12259
|
-
|
|
12260
|
-
|
|
12261
|
-
|
|
12262
|
-
|
|
12263
|
-
|
|
12264
|
-
|
|
12265
|
-
|
|
12978
|
+
let excelLoaded = false;
|
|
12979
|
+
try {
|
|
12980
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
12981
|
+
if (isZip) {
|
|
12982
|
+
const ExcelJS = await import("exceljs");
|
|
12983
|
+
const workbook = new ExcelJS.Workbook();
|
|
12984
|
+
await workbook.xlsx.load(buffer);
|
|
12985
|
+
workbook.eachSheet((sheet, sheetId) => {
|
|
12986
|
+
const sheetName = sheet.name;
|
|
12987
|
+
let headers = [];
|
|
12988
|
+
let i = 0;
|
|
12989
|
+
sheet.eachRow((row, rowNumber) => {
|
|
12990
|
+
const values = Array.isArray(row.values) ? row.values.slice(1) : [];
|
|
12991
|
+
if (rowNumber === 1) {
|
|
12992
|
+
headers = values.map((v) => String(v));
|
|
12993
|
+
return;
|
|
12994
|
+
}
|
|
12995
|
+
i++;
|
|
12996
|
+
const contentParts = [];
|
|
12997
|
+
const rowData = {};
|
|
12998
|
+
values.forEach((val, index) => {
|
|
12999
|
+
const key = headers[index] || `Column${index + 1}`;
|
|
13000
|
+
rowData[key] = val;
|
|
13001
|
+
if (key && val !== void 0 && val !== null && String(val).trim()) {
|
|
13002
|
+
contentParts.push(`${key}: ${val}`);
|
|
13003
|
+
}
|
|
13004
|
+
});
|
|
13005
|
+
if (contentParts.length > 0) {
|
|
13006
|
+
documents.push({
|
|
13007
|
+
docId: `${file.name}-${sheetName}-row-${i}`,
|
|
13008
|
+
content: contentParts.join(", "),
|
|
13009
|
+
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
13010
|
+
fileName: file.name,
|
|
13011
|
+
sheetName,
|
|
13012
|
+
fileSize: file.size,
|
|
13013
|
+
fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
13014
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13015
|
+
}, dimension ? { dimension } : {}), {
|
|
13016
|
+
headers
|
|
13017
|
+
}), rowData)
|
|
13018
|
+
});
|
|
13019
|
+
}
|
|
12266
13020
|
});
|
|
12267
|
-
}
|
|
13021
|
+
});
|
|
13022
|
+
excelLoaded = documents.length > 0;
|
|
13023
|
+
}
|
|
13024
|
+
} catch (excelErr) {
|
|
13025
|
+
excelLoaded = false;
|
|
13026
|
+
}
|
|
13027
|
+
if (!excelLoaded) {
|
|
13028
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
13029
|
+
documents.push({
|
|
13030
|
+
docId: file.name,
|
|
13031
|
+
content,
|
|
13032
|
+
metadata: __spreadValues({
|
|
13033
|
+
fileName: file.name,
|
|
13034
|
+
fileSize: file.size,
|
|
13035
|
+
fileType: file.type || "application/vnd.ms-excel",
|
|
13036
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
13037
|
+
}, dimension ? { dimension } : {})
|
|
12268
13038
|
});
|
|
12269
|
-
}
|
|
13039
|
+
}
|
|
12270
13040
|
} else {
|
|
12271
|
-
const content = await DocumentParser.parse(
|
|
13041
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
12272
13042
|
documents.push({
|
|
12273
13043
|
docId: file.name,
|
|
12274
13044
|
content,
|
|
@@ -12427,7 +13197,7 @@ function createSessionsHandler(configOrPlugin, options) {
|
|
|
12427
13197
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
12428
13198
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
12429
13199
|
const onResolveScope = options == null ? void 0 : options.onResolveScope;
|
|
12430
|
-
return async function GET(req) {
|
|
13200
|
+
return async function GET(req, context) {
|
|
12431
13201
|
if (req) {
|
|
12432
13202
|
const authResult = await checkAuth(req, onAuthorize);
|
|
12433
13203
|
if (authResult) return authResult;
|
|
@@ -13359,6 +14129,7 @@ async function createRetrivora(options) {
|
|
|
13359
14129
|
PineconeProvider,
|
|
13360
14130
|
Pipeline,
|
|
13361
14131
|
PostgreSQLProvider,
|
|
14132
|
+
PromptOptimizer,
|
|
13362
14133
|
ProviderHealthCheck,
|
|
13363
14134
|
ProviderNotFoundException,
|
|
13364
14135
|
ProviderRegistry,
|