@retrivora-ai/rag-engine 2.5.7 → 2.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-DwFj0h2u.d.mts → ILLMProvider-B2N3lMBc.d.mts} +10 -1
- package/dist/{ILLMProvider-DwFj0h2u.d.ts → ILLMProvider-B2N3lMBc.d.ts} +10 -1
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +961 -192
- package/dist/handlers/index.mjs +12821 -24
- package/dist/{index-Cb0GsT16.d.ts → index-37V810a-.d.mts} +3 -3
- package/dist/{index-D1jN8KSF.d.mts → index-BxzDv2jd.d.ts} +3 -3
- package/dist/{index-9h_ENWQn.d.mts → index-CAGGPyI7.d.mts} +6 -2
- package/dist/{index-Xci6GfpI.d.ts → index-DdQxoIpu.d.ts} +6 -2
- package/dist/index.css +76 -4
- package/dist/index.d.mts +6 -6
- package/dist/index.d.ts +6 -6
- package/dist/index.js +136 -94
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +1553 -129
- package/dist/server.d.mts +100 -9
- package/dist/server.d.ts +100 -9
- package/dist/server.js +963 -192
- package/dist/server.mjs +13378 -372
- package/package.json +5 -2
- package/dist/ChromaDBProvider-4RPJN7Y3.mjs +0 -8
- package/dist/MilvusProvider-WJLLZ6TW.mjs +0 -8
- package/dist/MongoDBProvider-QSERELZX.mjs +0 -8
- package/dist/MultiTablePostgresProvider-JF2HEUDZ.mjs +0 -9
- package/dist/PineconeProvider-HLZL7HKU.mjs +0 -10
- package/dist/PostgreSQLProvider-JQMCNHRO.mjs +0 -8
- package/dist/QdrantProvider-6MNFPEVC.mjs +0 -8
- package/dist/RedisProvider-ERQL6AEB.mjs +0 -8
- package/dist/SimpleGraphProvider-OP3HAJJM.mjs +0 -70
- package/dist/UniversalVectorProvider-INM3KH5F.mjs +0 -9
- package/dist/WeaviateProvider-VHWWICCC.mjs +0 -8
- package/dist/chunk-366EFBZC.mjs +0 -165
- package/dist/chunk-6MGVQTU4.mjs +0 -8883
- package/dist/chunk-B4AXIUM7.mjs +0 -49
- package/dist/chunk-BEPLYQBS.mjs +0 -89
- package/dist/chunk-BNOPQCZW.mjs +0 -238
- package/dist/chunk-FJ43J4FW.mjs +0 -654
- package/dist/chunk-GP2AJQRQ.mjs +0 -107
- package/dist/chunk-IMP6FUCY.mjs +0 -30
- package/dist/chunk-JRQWSJNF.mjs +0 -202
- package/dist/chunk-LYHCKUNF.mjs +0 -112
- package/dist/chunk-MFIWWUC7.mjs +0 -152
- package/dist/chunk-MWICNCB2.mjs +0 -157
- package/dist/chunk-N54YTFBN.mjs +0 -76
- package/dist/chunk-POOWU2HR.mjs +0 -211
- package/dist/chunk-QGDGPOZH.mjs +0 -217
- package/dist/chunk-QI3FTN6G.mjs +0 -281
- package/dist/chunk-QJIDGVG6.mjs +0 -334
- package/dist/chunk-W2DKVD3G.mjs +0 -81
- package/dist/chunk-XHSDHPKI.mjs +0 -336
package/dist/handlers/index.js
CHANGED
|
@@ -94,8 +94,8 @@ function buildPayload(template, vars) {
|
|
|
94
94
|
let raw = template;
|
|
95
95
|
for (const [key, val] of Object.entries(vars)) {
|
|
96
96
|
const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
|
|
97
|
-
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
|
|
98
|
-
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
|
|
97
|
+
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
|
|
98
|
+
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
|
|
99
99
|
}
|
|
100
100
|
try {
|
|
101
101
|
return JSON.parse(raw);
|
|
@@ -318,7 +318,7 @@ var init_package = __esm({
|
|
|
318
318
|
"package.json"() {
|
|
319
319
|
package_default = {
|
|
320
320
|
name: "@retrivora-ai/rag-engine",
|
|
321
|
-
version: "2.5.
|
|
321
|
+
version: "2.5.8",
|
|
322
322
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
323
323
|
author: "Abhinav Alkuchi",
|
|
324
324
|
license: "UNLICENSED",
|
|
@@ -421,9 +421,11 @@ var init_package = __esm({
|
|
|
421
421
|
"@google/genai": "^0.8.0",
|
|
422
422
|
"@google/generative-ai": "^0.24.1",
|
|
423
423
|
"@pinecone-database/pinecone": "^7.2.0",
|
|
424
|
-
"@retrivora-ai/billing-guards": "
|
|
424
|
+
"@retrivora-ai/billing-guards": "file:../billing-guards",
|
|
425
425
|
"@types/papaparse": "^5.5.2",
|
|
426
|
+
exceljs: "^4.4.0",
|
|
426
427
|
"framer-motion": "^13.1.0",
|
|
428
|
+
jszip: "^3.10.1",
|
|
427
429
|
katex: "^0.18.4",
|
|
428
430
|
"lucide-react": "^1.8.0",
|
|
429
431
|
"next-themes": "^0.4.6",
|
|
@@ -436,7 +438,7 @@ var init_package = __esm({
|
|
|
436
438
|
"rehype-katex": "^7.0.1",
|
|
437
439
|
"remark-gfm": "^4.0.1",
|
|
438
440
|
"remark-math": "^6.0.0",
|
|
439
|
-
|
|
441
|
+
xlsx: "^0.18.5",
|
|
440
442
|
zod: "^4.4.3"
|
|
441
443
|
},
|
|
442
444
|
optionalDependencies: {
|
|
@@ -454,6 +456,7 @@ var init_package = __esm({
|
|
|
454
456
|
"@tailwindcss/postcss": "^4",
|
|
455
457
|
"@types/estree": "^1.0.9",
|
|
456
458
|
"@types/jest": "^30.0.0",
|
|
459
|
+
"@types/jszip": "^3.4.1",
|
|
457
460
|
"@types/katex": "^0.16.8",
|
|
458
461
|
"@types/node": "^20",
|
|
459
462
|
"@types/pdf-parse": "^1.1.5",
|
|
@@ -2870,6 +2873,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2870
2873
|
try {
|
|
2871
2874
|
response = await fetch(endpoint, {
|
|
2872
2875
|
method: "POST",
|
|
2876
|
+
signal: AbortSignal.timeout(3e3),
|
|
2873
2877
|
headers: {
|
|
2874
2878
|
"Content-Type": "application/json",
|
|
2875
2879
|
"x-license-key": licenseKey
|
|
@@ -5547,15 +5551,29 @@ Text to extract from:
|
|
|
5547
5551
|
const response = await this.llm.chat([
|
|
5548
5552
|
{ role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
|
|
5549
5553
|
{ role: "user", content: prompt }
|
|
5550
|
-
], "", { maxTokens:
|
|
5554
|
+
], "", { maxTokens: 800, temperature: 0.1 });
|
|
5551
5555
|
try {
|
|
5552
|
-
const
|
|
5553
|
-
const
|
|
5554
|
-
|
|
5556
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5557
|
+
const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
|
5558
|
+
const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
|
|
5559
|
+
const jsonMatch = candidate.match(/\{[\s\S]*\}/);
|
|
5560
|
+
const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
|
|
5561
|
+
const parsed = JSON.parse(cleanJson);
|
|
5562
|
+
return {
|
|
5563
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5564
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5565
|
+
};
|
|
5555
5566
|
} catch (e) {
|
|
5556
5567
|
try {
|
|
5557
|
-
const
|
|
5558
|
-
|
|
5568
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5569
|
+
const repaired = this.repairTruncatedJson(withoutThinking);
|
|
5570
|
+
if (repaired) {
|
|
5571
|
+
const parsed = JSON.parse(repaired);
|
|
5572
|
+
return {
|
|
5573
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5574
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5575
|
+
};
|
|
5576
|
+
}
|
|
5559
5577
|
} catch (e2) {
|
|
5560
5578
|
}
|
|
5561
5579
|
console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
|
|
@@ -5833,6 +5851,208 @@ ${error instanceof Error ? error.message : String(error)}`
|
|
|
5833
5851
|
}
|
|
5834
5852
|
};
|
|
5835
5853
|
|
|
5854
|
+
// src/rag/PromptOptimizer.ts
|
|
5855
|
+
var PromptOptimizer = class {
|
|
5856
|
+
/**
|
|
5857
|
+
* Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
|
|
5858
|
+
*/
|
|
5859
|
+
static optimize(rawPrompt, history = [], options = {}) {
|
|
5860
|
+
if (!rawPrompt || typeof rawPrompt !== "string") {
|
|
5861
|
+
return {
|
|
5862
|
+
optimizedPrompt: "",
|
|
5863
|
+
originalPrompt: rawPrompt || "",
|
|
5864
|
+
wasModified: false,
|
|
5865
|
+
transformations: []
|
|
5866
|
+
};
|
|
5867
|
+
}
|
|
5868
|
+
const { resolveHistory = true, expandKeywords = true } = options;
|
|
5869
|
+
const transformations = [];
|
|
5870
|
+
let current = rawPrompt;
|
|
5871
|
+
const cleaned = this.cleanText(current);
|
|
5872
|
+
if (cleaned !== current) {
|
|
5873
|
+
transformations.push("clean_whitespace_and_artifacts");
|
|
5874
|
+
current = cleaned;
|
|
5875
|
+
}
|
|
5876
|
+
const expandedAbbr = this.expandAbbreviations(current);
|
|
5877
|
+
if (expandedAbbr !== current) {
|
|
5878
|
+
transformations.push("expand_abbreviations");
|
|
5879
|
+
current = expandedAbbr;
|
|
5880
|
+
}
|
|
5881
|
+
if (expandKeywords) {
|
|
5882
|
+
const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
5883
|
+
if (this.TOPIC_EXPANSIONS[trimmedLower]) {
|
|
5884
|
+
current = this.TOPIC_EXPANSIONS[trimmedLower];
|
|
5885
|
+
transformations.push(`topic_expansion_${trimmedLower}`);
|
|
5886
|
+
}
|
|
5887
|
+
}
|
|
5888
|
+
if (resolveHistory && history && history.length > 0) {
|
|
5889
|
+
const resolved = this.resolveCoreferences(current, history);
|
|
5890
|
+
if (resolved !== current) {
|
|
5891
|
+
transformations.push("resolve_history_coreferences");
|
|
5892
|
+
current = resolved;
|
|
5893
|
+
}
|
|
5894
|
+
}
|
|
5895
|
+
if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
|
|
5896
|
+
if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
|
|
5897
|
+
current = `${current.trim()}?`;
|
|
5898
|
+
transformations.push("add_question_mark");
|
|
5899
|
+
}
|
|
5900
|
+
}
|
|
5901
|
+
return {
|
|
5902
|
+
optimizedPrompt: current,
|
|
5903
|
+
originalPrompt: rawPrompt,
|
|
5904
|
+
wasModified: current !== rawPrompt,
|
|
5905
|
+
transformations
|
|
5906
|
+
};
|
|
5907
|
+
}
|
|
5908
|
+
/**
|
|
5909
|
+
* Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
|
|
5910
|
+
*/
|
|
5911
|
+
static cleanText(text) {
|
|
5912
|
+
let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
|
|
5913
|
+
const doubleQuotesCount = (clean.match(/"/g) || []).length;
|
|
5914
|
+
if (doubleQuotesCount % 2 !== 0) {
|
|
5915
|
+
clean += '"';
|
|
5916
|
+
}
|
|
5917
|
+
const singleQuotesCount = (clean.match(/'/g) || []).length;
|
|
5918
|
+
if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
|
|
5919
|
+
clean += "'";
|
|
5920
|
+
}
|
|
5921
|
+
const openParen = (clean.match(/\(/g) || []).length;
|
|
5922
|
+
const closeParen = (clean.match(/\)/g) || []).length;
|
|
5923
|
+
if (openParen > closeParen) {
|
|
5924
|
+
clean += ")".repeat(openParen - closeParen);
|
|
5925
|
+
}
|
|
5926
|
+
return clean;
|
|
5927
|
+
}
|
|
5928
|
+
/**
|
|
5929
|
+
* Replaces shorthand chat abbreviations with clear natural language.
|
|
5930
|
+
*/
|
|
5931
|
+
static expandAbbreviations(text) {
|
|
5932
|
+
let expanded = text;
|
|
5933
|
+
for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
|
|
5934
|
+
expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
|
|
5935
|
+
}
|
|
5936
|
+
return expanded;
|
|
5937
|
+
}
|
|
5938
|
+
/**
|
|
5939
|
+
* Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
|
|
5940
|
+
*/
|
|
5941
|
+
static resolveCoreferences(prompt, history) {
|
|
5942
|
+
const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
|
|
5943
|
+
if (!pronounPattern.test(prompt)) {
|
|
5944
|
+
return prompt;
|
|
5945
|
+
}
|
|
5946
|
+
const recentMessages = [...history].slice(-4).reverse();
|
|
5947
|
+
let detectedEntity = null;
|
|
5948
|
+
for (const msg of recentMessages) {
|
|
5949
|
+
if (!msg.content) continue;
|
|
5950
|
+
const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
|
|
5951
|
+
if (quotedMatch) {
|
|
5952
|
+
detectedEntity = quotedMatch[1];
|
|
5953
|
+
break;
|
|
5954
|
+
}
|
|
5955
|
+
const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
|
|
5956
|
+
if (headingMatch) {
|
|
5957
|
+
detectedEntity = headingMatch[1].trim();
|
|
5958
|
+
break;
|
|
5959
|
+
}
|
|
5960
|
+
const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
|
|
5961
|
+
if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
|
|
5962
|
+
detectedEntity = entityMatches[1];
|
|
5963
|
+
break;
|
|
5964
|
+
}
|
|
5965
|
+
}
|
|
5966
|
+
if (detectedEntity) {
|
|
5967
|
+
let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
|
|
5968
|
+
return resolved;
|
|
5969
|
+
}
|
|
5970
|
+
return prompt;
|
|
5971
|
+
}
|
|
5972
|
+
/**
|
|
5973
|
+
* Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
|
|
5974
|
+
*/
|
|
5975
|
+
static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
|
|
5976
|
+
const fastResult = this.optimize(rawPrompt, history);
|
|
5977
|
+
if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
|
|
5978
|
+
return fastResult;
|
|
5979
|
+
}
|
|
5980
|
+
try {
|
|
5981
|
+
const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
|
|
5982
|
+
const prompt = `You are a query optimizer in an enterprise RAG system.
|
|
5983
|
+
Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
|
|
5984
|
+
- Do NOT answer the question.
|
|
5985
|
+
- Resolve any pronouns like "it", "they", "that" using the context.
|
|
5986
|
+
- Keep the user's intent 100% intact.
|
|
5987
|
+
- If the question is already clear, return it as is.
|
|
5988
|
+
- Return ONLY the optimized query string. No quotes or explanations.
|
|
5989
|
+
|
|
5990
|
+
Context:
|
|
5991
|
+
${historyContext || "No previous context."}
|
|
5992
|
+
|
|
5993
|
+
User Question: "${rawPrompt}"
|
|
5994
|
+
|
|
5995
|
+
Optimized Search Prompt:`;
|
|
5996
|
+
const response = await llmProvider.chat(
|
|
5997
|
+
[{ role: "user", content: prompt }],
|
|
5998
|
+
"",
|
|
5999
|
+
{
|
|
6000
|
+
systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
|
|
6001
|
+
temperature: 0,
|
|
6002
|
+
maxTokens: 100
|
|
6003
|
+
}
|
|
6004
|
+
);
|
|
6005
|
+
const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
|
|
6006
|
+
if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
|
|
6007
|
+
return {
|
|
6008
|
+
optimizedPrompt: cleaned,
|
|
6009
|
+
originalPrompt: rawPrompt,
|
|
6010
|
+
wasModified: cleaned !== rawPrompt,
|
|
6011
|
+
transformations: [...fastResult.transformations, "llm_deep_reformulation"]
|
|
6012
|
+
};
|
|
6013
|
+
}
|
|
6014
|
+
} catch (err) {
|
|
6015
|
+
console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
|
|
6016
|
+
}
|
|
6017
|
+
return fastResult;
|
|
6018
|
+
}
|
|
6019
|
+
};
|
|
6020
|
+
PromptOptimizer.COMMON_ABBREVIATIONS = {
|
|
6021
|
+
"\\bpls\\b": "please",
|
|
6022
|
+
"\\bplz\\b": "please",
|
|
6023
|
+
"\\bthx\\b": "thanks",
|
|
6024
|
+
"\\bty\\b": "thank you",
|
|
6025
|
+
"\\bapprox\\b": "approximately",
|
|
6026
|
+
"\\bw/o\\b": "without",
|
|
6027
|
+
"\\bw/\\b": "with",
|
|
6028
|
+
"\\bdiff\\b": "difference",
|
|
6029
|
+
"\\binfo\\b": "information",
|
|
6030
|
+
"\\bconfig\\b": "configuration",
|
|
6031
|
+
"\\bdoc\\b": "document",
|
|
6032
|
+
"\\bdocs\\b": "documentation",
|
|
6033
|
+
"\\bfeat\\b": "feature",
|
|
6034
|
+
"\\bfeats\\b": "features",
|
|
6035
|
+
"\\bvs\\b": "versus",
|
|
6036
|
+
"\\bu\\b": "you",
|
|
6037
|
+
"\\bur\\b": "your",
|
|
6038
|
+
"\\br\\b": "are"
|
|
6039
|
+
};
|
|
6040
|
+
PromptOptimizer.TOPIC_EXPANSIONS = {
|
|
6041
|
+
"pricing": "What are the available pricing plans, tiers, and cost structures?",
|
|
6042
|
+
"price": "What is the pricing and subscription cost?",
|
|
6043
|
+
"features": "What are the core features, capabilities, and tools provided?",
|
|
6044
|
+
"security": "What are the security, privacy, and compliance standards?",
|
|
6045
|
+
"privacy": "What is the privacy policy and how is data protected?",
|
|
6046
|
+
"refund": "What is the refund and cancellation policy?",
|
|
6047
|
+
"refunds": "What is the refund and cancellation policy?",
|
|
6048
|
+
"contact": "How can I contact customer support or sales?",
|
|
6049
|
+
"support": "What support channels and assistance are available?",
|
|
6050
|
+
"docs": "Where can I find the official documentation and setup guides?",
|
|
6051
|
+
"installation": "How do I install and configure the SDK or service?",
|
|
6052
|
+
"install": "What are the step-by-step installation instructions?",
|
|
6053
|
+
"faq": "What are the frequently asked questions and key answers?"
|
|
6054
|
+
};
|
|
6055
|
+
|
|
5836
6056
|
// src/core/mcp.ts
|
|
5837
6057
|
var import_child_process = require("child_process");
|
|
5838
6058
|
var MCPClient = class {
|
|
@@ -7050,6 +7270,26 @@ var IntentClassifier = class {
|
|
|
7050
7270
|
static extractDataSignals(context) {
|
|
7051
7271
|
const docs = context.retrievedDocuments || [];
|
|
7052
7272
|
const rowCount = docs.length;
|
|
7273
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
7274
|
+
"filetype",
|
|
7275
|
+
"mimetype",
|
|
7276
|
+
"filename",
|
|
7277
|
+
"docid",
|
|
7278
|
+
"uploadedat",
|
|
7279
|
+
"dimension",
|
|
7280
|
+
"chunkindex",
|
|
7281
|
+
"totalchunks",
|
|
7282
|
+
"charactercount",
|
|
7283
|
+
"workspaceid",
|
|
7284
|
+
"pineconenamespace",
|
|
7285
|
+
"tier",
|
|
7286
|
+
"custommetadata",
|
|
7287
|
+
"csvheaders",
|
|
7288
|
+
"headers",
|
|
7289
|
+
"filesize",
|
|
7290
|
+
"content",
|
|
7291
|
+
"id"
|
|
7292
|
+
]);
|
|
7053
7293
|
let hasNumericFields = false;
|
|
7054
7294
|
let hasDateFields = false;
|
|
7055
7295
|
let hasCategoricalFields = false;
|
|
@@ -7065,6 +7305,7 @@ var IntentClassifier = class {
|
|
|
7065
7305
|
docs.forEach((doc) => {
|
|
7066
7306
|
const meta = doc.metadata || {};
|
|
7067
7307
|
Object.entries(meta).forEach(([key, val]) => {
|
|
7308
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
7068
7309
|
keys.add(key);
|
|
7069
7310
|
const kLower = key.toLowerCase();
|
|
7070
7311
|
if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
|
|
@@ -7073,7 +7314,7 @@ var IntentClassifier = class {
|
|
|
7073
7314
|
if (typeof val === "number") {
|
|
7074
7315
|
numericKeys.add(key);
|
|
7075
7316
|
} else if (typeof val === "string") {
|
|
7076
|
-
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
|
|
7317
|
+
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
|
|
7077
7318
|
dateKeys.add(key);
|
|
7078
7319
|
} else if (val.length < 50 && !val.includes("\n")) {
|
|
7079
7320
|
catKeys.add(key);
|
|
@@ -7086,10 +7327,7 @@ var IntentClassifier = class {
|
|
|
7086
7327
|
hasCategoricalFields = catKeys.size > 0;
|
|
7087
7328
|
numericFieldCount = numericKeys.size;
|
|
7088
7329
|
categoricalFieldCount = catKeys.size;
|
|
7089
|
-
if (productKeyMatches >= 2
|
|
7090
|
-
var _a2, _b;
|
|
7091
|
-
return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
|
|
7092
|
-
})) {
|
|
7330
|
+
if (productKeyMatches >= 2) {
|
|
7093
7331
|
isProductLike = true;
|
|
7094
7332
|
}
|
|
7095
7333
|
}
|
|
@@ -7105,12 +7343,13 @@ var IntentClassifier = class {
|
|
|
7105
7343
|
}
|
|
7106
7344
|
// ─── Intent Heuristic Checkers ─────────────────────────────────────────────
|
|
7107
7345
|
static isInformationLookup(query) {
|
|
7108
|
-
|
|
7109
|
-
|
|
7346
|
+
const q = query.toLowerCase().trim();
|
|
7347
|
+
if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
|
|
7348
|
+
if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
|
|
7110
7349
|
return true;
|
|
7111
7350
|
}
|
|
7112
7351
|
}
|
|
7113
|
-
if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(
|
|
7352
|
+
if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
|
|
7114
7353
|
return true;
|
|
7115
7354
|
}
|
|
7116
7355
|
return false;
|
|
@@ -7125,13 +7364,13 @@ var IntentClassifier = class {
|
|
|
7125
7364
|
return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
|
|
7126
7365
|
}
|
|
7127
7366
|
static isTrendQuery(query) {
|
|
7128
|
-
return /\b(trend|trends|growth
|
|
7367
|
+
return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
|
|
7129
7368
|
}
|
|
7130
7369
|
static isRankingQuery(query) {
|
|
7131
7370
|
return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
|
|
7132
7371
|
}
|
|
7133
7372
|
static isAnalyticsQuery(query) {
|
|
7134
|
-
return /\b(
|
|
7373
|
+
return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
|
|
7135
7374
|
}
|
|
7136
7375
|
};
|
|
7137
7376
|
|
|
@@ -7143,8 +7382,8 @@ var Rule1SpecificInfoRule = class {
|
|
|
7143
7382
|
}
|
|
7144
7383
|
evaluate(context, intent) {
|
|
7145
7384
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7146
|
-
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
|
|
7147
|
-
const isNonVisual = !/\b(
|
|
7385
|
+
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
|
|
7386
|
+
const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
|
|
7148
7387
|
if (isFactQuery && isNonVisual) {
|
|
7149
7388
|
return {
|
|
7150
7389
|
renderType: "text",
|
|
@@ -7209,11 +7448,11 @@ var Rule4AnalyticalRule = class {
|
|
|
7209
7448
|
const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
|
|
7210
7449
|
if (!isAnalytical) return null;
|
|
7211
7450
|
let chartType = "bar";
|
|
7212
|
-
if (intent === "trend_analysis" || /\b(monthly|yearly|
|
|
7451
|
+
if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
|
|
7213
7452
|
chartType = "line";
|
|
7214
|
-
} else if (/\b(share|percentage|
|
|
7453
|
+
} else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
|
|
7215
7454
|
chartType = "pie";
|
|
7216
|
-
} else if (/\b(correlation|scatter|relationship
|
|
7455
|
+
} else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
|
|
7217
7456
|
chartType = "scatter";
|
|
7218
7457
|
} else {
|
|
7219
7458
|
chartType = "bar";
|
|
@@ -7976,12 +8215,20 @@ ${schemaProfileText}` : ""}`;
|
|
|
7976
8215
|
var _a3;
|
|
7977
8216
|
return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
|
|
7978
8217
|
}).filter(Boolean))) : this.detectCategories(data);
|
|
7979
|
-
if (categories.length
|
|
8218
|
+
if (categories.length <= 1) return null;
|
|
7980
8219
|
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
8220
|
+
const hasInventoryData = data.some((item) => {
|
|
8221
|
+
const meta = item.metadata || {};
|
|
8222
|
+
return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
|
|
8223
|
+
});
|
|
7981
8224
|
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
8225
|
+
if (!hasInventoryData) {
|
|
8226
|
+
return { label, value: count };
|
|
8227
|
+
}
|
|
7982
8228
|
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
7983
8229
|
return { label, value: count, inStockCount, outOfStockCount };
|
|
7984
8230
|
});
|
|
8231
|
+
if (pieData.length <= 1) return null;
|
|
7985
8232
|
return {
|
|
7986
8233
|
type: "pie_chart",
|
|
7987
8234
|
title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
|
|
@@ -8288,41 +8535,34 @@ ${schemaProfileText}` : ""}`;
|
|
|
8288
8535
|
}
|
|
8289
8536
|
static _productQueryTest(query) {
|
|
8290
8537
|
const q = query.toLowerCase();
|
|
8291
|
-
const
|
|
8538
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
|
|
8539
|
+
if (financialTerms.test(q)) {
|
|
8540
|
+
return false;
|
|
8541
|
+
}
|
|
8542
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
8292
8543
|
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
8293
8544
|
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
8294
8545
|
return productTerms && productAction && !nonProductEntityTerms;
|
|
8295
8546
|
}
|
|
8296
8547
|
static isProductData(item) {
|
|
8297
8548
|
if (!item) return false;
|
|
8298
|
-
const content = (item.content || "").toLowerCase();
|
|
8299
|
-
const productKeywords = [
|
|
8300
|
-
"product",
|
|
8301
|
-
"price",
|
|
8302
|
-
"stock",
|
|
8303
|
-
"sku",
|
|
8304
|
-
"brand",
|
|
8305
|
-
"msrp",
|
|
8306
|
-
"inventory",
|
|
8307
|
-
"buy",
|
|
8308
|
-
"shop",
|
|
8309
|
-
"beauty",
|
|
8310
|
-
"cosmetic",
|
|
8311
|
-
"facial",
|
|
8312
|
-
"cream",
|
|
8313
|
-
"serum",
|
|
8314
|
-
"mask",
|
|
8315
|
-
"makeup",
|
|
8316
|
-
"fragrance"
|
|
8317
|
-
];
|
|
8318
|
-
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
8319
8549
|
const metadata = item.metadata || {};
|
|
8320
|
-
const
|
|
8321
|
-
|
|
8322
|
-
return
|
|
8323
|
-
}
|
|
8324
|
-
|
|
8325
|
-
|
|
8550
|
+
const domain = String(metadata.domain || "").toLowerCase();
|
|
8551
|
+
if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
|
|
8552
|
+
return false;
|
|
8553
|
+
}
|
|
8554
|
+
if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
|
|
8555
|
+
return true;
|
|
8556
|
+
}
|
|
8557
|
+
const content = (item.content || "").toLowerCase();
|
|
8558
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
|
|
8559
|
+
if (financialTerms.test(content)) {
|
|
8560
|
+
return false;
|
|
8561
|
+
}
|
|
8562
|
+
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
|
|
8563
|
+
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
8564
|
+
const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
8565
|
+
return hasSku || hasProductIdentity && hasPrice;
|
|
8326
8566
|
}
|
|
8327
8567
|
static isTimeSeriesData(item) {
|
|
8328
8568
|
const metadata = item.metadata || {};
|
|
@@ -8341,10 +8581,31 @@ ${schemaProfileText}` : ""}`;
|
|
|
8341
8581
|
return fieldCount.size > 2;
|
|
8342
8582
|
}
|
|
8343
8583
|
static profileData(data) {
|
|
8584
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
8585
|
+
"filetype",
|
|
8586
|
+
"mimetype",
|
|
8587
|
+
"filename",
|
|
8588
|
+
"docid",
|
|
8589
|
+
"uploadedat",
|
|
8590
|
+
"dimension",
|
|
8591
|
+
"chunkindex",
|
|
8592
|
+
"totalchunks",
|
|
8593
|
+
"charactercount",
|
|
8594
|
+
"workspaceid",
|
|
8595
|
+
"pineconenamespace",
|
|
8596
|
+
"tier",
|
|
8597
|
+
"custommetadata",
|
|
8598
|
+
"csvheaders",
|
|
8599
|
+
"headers",
|
|
8600
|
+
"filesize",
|
|
8601
|
+
"content",
|
|
8602
|
+
"id"
|
|
8603
|
+
]);
|
|
8344
8604
|
const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
|
|
8345
8605
|
var _a2, _b;
|
|
8346
8606
|
const fields2 = {};
|
|
8347
8607
|
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
8608
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
8348
8609
|
const primitive = this.toPrimitive(value);
|
|
8349
8610
|
if (primitive !== null) fields2[key] = primitive;
|
|
8350
8611
|
});
|
|
@@ -9298,27 +9559,74 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9298
9559
|
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
9299
9560
|
*/
|
|
9300
9561
|
async ingest(documents, namespace) {
|
|
9562
|
+
var _a2;
|
|
9301
9563
|
await this.initialize();
|
|
9302
9564
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9303
9565
|
const results = [];
|
|
9566
|
+
if (!documents || documents.length === 0) return results;
|
|
9567
|
+
const docChunksMap = [];
|
|
9568
|
+
const allChunks = [];
|
|
9304
9569
|
for (const doc of documents) {
|
|
9305
9570
|
try {
|
|
9306
9571
|
const chunks = await this.prepareChunks(doc);
|
|
9307
|
-
|
|
9572
|
+
docChunksMap.push({ doc, chunks });
|
|
9573
|
+
allChunks.push(...chunks);
|
|
9574
|
+
} catch (error) {
|
|
9575
|
+
Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
|
|
9576
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9577
|
+
}
|
|
9578
|
+
}
|
|
9579
|
+
if (allChunks.length === 0) {
|
|
9580
|
+
return results;
|
|
9581
|
+
}
|
|
9582
|
+
try {
|
|
9583
|
+
const allVectors = await this.processEmbeddings(allChunks);
|
|
9584
|
+
let vectorOffset = 0;
|
|
9585
|
+
const allUpsertDocs = [];
|
|
9586
|
+
const docSuccessCount = /* @__PURE__ */ new Map();
|
|
9587
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9588
|
+
const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
|
|
9589
|
+
vectorOffset += chunks.length;
|
|
9308
9590
|
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9309
9591
|
id: chunk.id,
|
|
9310
|
-
vector:
|
|
9592
|
+
vector: docVectors[i],
|
|
9311
9593
|
content: chunk.content,
|
|
9312
9594
|
metadata: chunk.metadata
|
|
9313
9595
|
}));
|
|
9314
|
-
|
|
9315
|
-
|
|
9596
|
+
allUpsertDocs.push(...upsertDocs);
|
|
9597
|
+
docSuccessCount.set(doc.docId, chunks.length);
|
|
9598
|
+
}
|
|
9599
|
+
const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
|
|
9600
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9601
|
+
results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
|
|
9316
9602
|
if (this.graphDB && this.entityExtractor) {
|
|
9317
|
-
|
|
9603
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9604
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9605
|
+
});
|
|
9606
|
+
}
|
|
9607
|
+
}
|
|
9608
|
+
} catch (batchErr) {
|
|
9609
|
+
Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
|
|
9610
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9611
|
+
try {
|
|
9612
|
+
const vectors = await this.processEmbeddings(chunks);
|
|
9613
|
+
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9614
|
+
id: chunk.id,
|
|
9615
|
+
vector: vectors[i],
|
|
9616
|
+
content: chunk.content,
|
|
9617
|
+
metadata: chunk.metadata
|
|
9618
|
+
}));
|
|
9619
|
+
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
9620
|
+
results.push({ docId: doc.docId, chunksIngested: totalProcessed });
|
|
9621
|
+
if (this.graphDB && this.entityExtractor) {
|
|
9622
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9623
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9624
|
+
});
|
|
9625
|
+
}
|
|
9626
|
+
} catch (singleErr) {
|
|
9627
|
+
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
|
|
9628
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9318
9629
|
}
|
|
9319
|
-
} catch (error) {
|
|
9320
|
-
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
|
|
9321
|
-
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9322
9630
|
}
|
|
9323
9631
|
}
|
|
9324
9632
|
return results;
|
|
@@ -9338,19 +9646,40 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9338
9646
|
}
|
|
9339
9647
|
/**
|
|
9340
9648
|
* Step 2: Generate embeddings for chunks with retry logic.
|
|
9341
|
-
* Uses batchEmbed when available for efficiency; falls back to
|
|
9649
|
+
* Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
|
|
9342
9650
|
*/
|
|
9343
9651
|
async processEmbeddings(chunks) {
|
|
9344
|
-
|
|
9345
|
-
|
|
9346
|
-
|
|
9347
|
-
|
|
9348
|
-
|
|
9349
|
-
|
|
9350
|
-
|
|
9652
|
+
if (chunks.length === 0) return [];
|
|
9653
|
+
const texts = chunks.map((c) => c.content);
|
|
9654
|
+
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9655
|
+
try {
|
|
9656
|
+
const batchSize = 50;
|
|
9657
|
+
const allVectors = [];
|
|
9658
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
9659
|
+
const slice = texts.slice(i, i + batchSize);
|
|
9660
|
+
const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
|
|
9661
|
+
allVectors.push(...batchVectors);
|
|
9662
|
+
}
|
|
9663
|
+
if (allVectors.length === chunks.length) {
|
|
9664
|
+
return allVectors;
|
|
9665
|
+
}
|
|
9666
|
+
} catch (err) {
|
|
9667
|
+
Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
|
|
9668
|
+
error: err instanceof Error ? err.message : String(err)
|
|
9669
|
+
});
|
|
9670
|
+
}
|
|
9671
|
+
}
|
|
9672
|
+
const concurrentResult = await BatchProcessor.processConcurrent(
|
|
9673
|
+
texts,
|
|
9351
9674
|
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
9352
|
-
|
|
9675
|
+
10,
|
|
9676
|
+
{
|
|
9677
|
+
maxRetries: 3,
|
|
9678
|
+
initialDelayMs: 100,
|
|
9679
|
+
throwOnPartialFailure: false
|
|
9680
|
+
}
|
|
9353
9681
|
);
|
|
9682
|
+
const vectors = concurrentResult.results;
|
|
9354
9683
|
if (vectors.length !== chunks.length) {
|
|
9355
9684
|
throw new EmbeddingFailedException(
|
|
9356
9685
|
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
|
|
@@ -9377,25 +9706,24 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9377
9706
|
}
|
|
9378
9707
|
/** Step 4: Optional graph-based entity extraction and ingestion. */
|
|
9379
9708
|
async processGraphIngestion(docId, chunks) {
|
|
9380
|
-
const
|
|
9381
|
-
|
|
9382
|
-
maxRetries: 1,
|
|
9383
|
-
initialDelayMs: 500
|
|
9384
|
-
};
|
|
9385
|
-
await BatchProcessor.processBatch(
|
|
9709
|
+
const maxConcurrency = Math.min(3, chunks.length);
|
|
9710
|
+
await BatchProcessor.processConcurrent(
|
|
9386
9711
|
chunks,
|
|
9387
|
-
async (
|
|
9388
|
-
|
|
9389
|
-
|
|
9390
|
-
|
|
9391
|
-
|
|
9392
|
-
|
|
9393
|
-
}
|
|
9394
|
-
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9395
|
-
}
|
|
9712
|
+
async (chunk) => {
|
|
9713
|
+
try {
|
|
9714
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
9715
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
9716
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
9717
|
+
} catch (err) {
|
|
9718
|
+
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9396
9719
|
}
|
|
9397
9720
|
},
|
|
9398
|
-
|
|
9721
|
+
maxConcurrency,
|
|
9722
|
+
{
|
|
9723
|
+
maxRetries: 1,
|
|
9724
|
+
initialDelayMs: 200,
|
|
9725
|
+
throwOnPartialFailure: false
|
|
9726
|
+
}
|
|
9399
9727
|
);
|
|
9400
9728
|
}
|
|
9401
9729
|
async runNormalQuery(question, history = [], namespace) {
|
|
@@ -9520,7 +9848,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9520
9848
|
*/
|
|
9521
9849
|
askStreamInternal(_0) {
|
|
9522
9850
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
9523
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
|
|
9851
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
|
|
9524
9852
|
yield new __await(this.initialize());
|
|
9525
9853
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9526
9854
|
const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
|
|
@@ -9528,29 +9856,35 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9528
9856
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
9529
9857
|
const requestStart = performance.now();
|
|
9530
9858
|
try {
|
|
9531
|
-
let
|
|
9859
|
+
let effectiveQuestion = question;
|
|
9860
|
+
let promptOptResult;
|
|
9861
|
+
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
9862
|
+
promptOptResult = PromptOptimizer.optimize(question, history);
|
|
9863
|
+
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
9864
|
+
}
|
|
9865
|
+
let searchQuery = effectiveQuestion;
|
|
9532
9866
|
let rewrittenQuery;
|
|
9533
|
-
if ((
|
|
9534
|
-
searchQuery = yield new __await(this.rewriteQuery(
|
|
9535
|
-
rewrittenQuery = searchQuery !==
|
|
9867
|
+
if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
|
|
9868
|
+
searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
|
|
9869
|
+
rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
|
|
9536
9870
|
}
|
|
9537
|
-
const hints = QueryProcessor.extractQueryFieldHints(
|
|
9538
|
-
const filter = QueryProcessor.buildQueryFilter(
|
|
9539
|
-
const numericPredicates = QueryProcessor.extractNumericPredicates(
|
|
9871
|
+
const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
|
|
9872
|
+
const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
|
|
9873
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
|
|
9540
9874
|
if (numericPredicates.length > 0) {
|
|
9541
9875
|
filter.__numericPredicates = numericPredicates;
|
|
9542
9876
|
}
|
|
9543
9877
|
const embedStart = performance.now();
|
|
9544
9878
|
const cacheKey = `${ns}::${searchQuery}`;
|
|
9545
9879
|
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
9546
|
-
const arch = (
|
|
9547
|
-
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((
|
|
9880
|
+
const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
|
|
9881
|
+
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
|
|
9548
9882
|
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
9549
9883
|
QueryProcessor.determineRetrievalStrategy(
|
|
9550
9884
|
searchQuery,
|
|
9551
9885
|
useGraph ? this.llmRouter.get("fast") : void 0,
|
|
9552
|
-
(
|
|
9553
|
-
(
|
|
9886
|
+
(_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
|
|
9887
|
+
(_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
|
|
9554
9888
|
),
|
|
9555
9889
|
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
9556
9890
|
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
@@ -9560,7 +9894,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9560
9894
|
if (!cachedVector && queryVector.length > 0) {
|
|
9561
9895
|
this.embeddingCache.set(cacheKey, queryVector);
|
|
9562
9896
|
}
|
|
9563
|
-
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((
|
|
9897
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
9564
9898
|
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
9565
9899
|
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
9566
9900
|
const wantsExhaustiveList = true;
|
|
@@ -9577,7 +9911,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9577
9911
|
return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
|
|
9578
9912
|
});
|
|
9579
9913
|
const rerankLimit = Math.max(retrievalLimit, fullSources.length);
|
|
9580
|
-
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((
|
|
9914
|
+
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
|
|
9581
9915
|
if (!hasNumericPredicates && useReranking) {
|
|
9582
9916
|
fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
|
|
9583
9917
|
}
|
|
@@ -9605,7 +9939,7 @@ ${graphContext}
|
|
|
9605
9939
|
VECTOR CONTEXT:
|
|
9606
9940
|
${context}`;
|
|
9607
9941
|
}
|
|
9608
|
-
if ((
|
|
9942
|
+
if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
|
|
9609
9943
|
if (!this.coldContexts.has(ns)) {
|
|
9610
9944
|
yield new __await(this.loadColdContext(ns));
|
|
9611
9945
|
}
|
|
@@ -9637,31 +9971,31 @@ ${context}`;
|
|
|
9637
9971
|
SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
|
|
9638
9972
|
});
|
|
9639
9973
|
}
|
|
9640
|
-
const isProductQ = UITransformer.isProductQueryPublic(
|
|
9974
|
+
const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
|
|
9641
9975
|
const uiTransformationPromise = isProductQ ? Promise.resolve(
|
|
9642
9976
|
UITransformer.transform(
|
|
9643
|
-
|
|
9977
|
+
effectiveQuestion,
|
|
9644
9978
|
sources,
|
|
9645
9979
|
this.config,
|
|
9646
9980
|
cachedSchema,
|
|
9647
9981
|
{ visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
|
|
9648
9982
|
)
|
|
9649
9983
|
) : this.generateUiTransformation(
|
|
9650
|
-
|
|
9984
|
+
effectiveQuestion,
|
|
9651
9985
|
sources,
|
|
9652
9986
|
cachedSchema,
|
|
9653
9987
|
hasNumericPredicates
|
|
9654
9988
|
).catch((uiError) => {
|
|
9655
9989
|
Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
|
|
9656
|
-
return UITransformer.transform(
|
|
9990
|
+
return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
|
|
9657
9991
|
});
|
|
9658
9992
|
let hallucinationScoringPromise = Promise.resolve(void 0);
|
|
9659
9993
|
const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
9660
9994
|
const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
|
|
9661
|
-
const isNativeThinking = isClaude37 && (((
|
|
9995
|
+
const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
|
|
9662
9996
|
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
9663
9997
|
const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
|
|
9664
|
-
const isSimulatedThinking = !isNativeThinking && (((
|
|
9998
|
+
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
|
|
9665
9999
|
let finalRestrictionSuffix = restrictionSuffix;
|
|
9666
10000
|
if (isSimulatedThinking) {
|
|
9667
10001
|
finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
|
|
@@ -9670,9 +10004,9 @@ ${context}`;
|
|
|
9670
10004
|
role: m.role || "user",
|
|
9671
10005
|
content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
|
|
9672
10006
|
}));
|
|
9673
|
-
const userQuestion = { role: "user", content:
|
|
10007
|
+
const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
|
|
9674
10008
|
const messages = [...hardenedHistory, userQuestion];
|
|
9675
|
-
const systemPrompt = (
|
|
10009
|
+
const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
|
|
9676
10010
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
9677
10011
|
let fullReply = "";
|
|
9678
10012
|
let thinkingText = "";
|
|
@@ -9783,7 +10117,7 @@ ${context}`;
|
|
|
9783
10117
|
}
|
|
9784
10118
|
yield fullReply;
|
|
9785
10119
|
}
|
|
9786
|
-
const runHallucination = ((
|
|
10120
|
+
const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
|
|
9787
10121
|
if (runHallucination) {
|
|
9788
10122
|
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
|
|
9789
10123
|
}
|
|
@@ -9805,7 +10139,7 @@ ${context}`;
|
|
|
9805
10139
|
totalTokens: promptTokens + completionTokens,
|
|
9806
10140
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
9807
10141
|
};
|
|
9808
|
-
const awaitHallucination = ((
|
|
10142
|
+
const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
|
|
9809
10143
|
const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
|
|
9810
10144
|
uiTransformationPromise,
|
|
9811
10145
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
@@ -9813,9 +10147,9 @@ ${context}`;
|
|
|
9813
10147
|
const buildTrace = (hScore) => __spreadValues({
|
|
9814
10148
|
requestId,
|
|
9815
10149
|
query: question,
|
|
9816
|
-
rewrittenQuery,
|
|
10150
|
+
rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
|
|
9817
10151
|
systemPrompt,
|
|
9818
|
-
userPrompt:
|
|
10152
|
+
userPrompt: effectiveQuestion + finalRestrictionSuffix,
|
|
9819
10153
|
chunks: (sources || []).filter(Boolean).map((s) => {
|
|
9820
10154
|
var _a3, _b2;
|
|
9821
10155
|
return {
|
|
@@ -9834,10 +10168,10 @@ ${context}`;
|
|
|
9834
10168
|
hallucinationReason: hScore.reason
|
|
9835
10169
|
} : {});
|
|
9836
10170
|
const trace = buildTrace(hallucinationResult);
|
|
9837
|
-
const isTelemetryActive = (
|
|
10171
|
+
const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
|
|
9838
10172
|
if (isTelemetryActive) {
|
|
9839
10173
|
const defaultUrl = "https://www.retrivora.com/api/telemetry";
|
|
9840
|
-
const telemetryUrl = ((
|
|
10174
|
+
const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
9841
10175
|
const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
|
|
9842
10176
|
(async () => {
|
|
9843
10177
|
var _a3, _b2, _c2, _d2, _e2;
|
|
@@ -10412,13 +10746,81 @@ var VectorPlugin = class {
|
|
|
10412
10746
|
|
|
10413
10747
|
// src/utils/DocumentParser.ts
|
|
10414
10748
|
var DocumentParser = class {
|
|
10749
|
+
/**
|
|
10750
|
+
* Fast Magic-Bytes sniffer for instant file format identification (<1ms).
|
|
10751
|
+
*/
|
|
10752
|
+
static sniffMagicBytes(buffer) {
|
|
10753
|
+
if (!buffer || buffer.length < 4) return null;
|
|
10754
|
+
if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
|
|
10755
|
+
return "pdf";
|
|
10756
|
+
}
|
|
10757
|
+
if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
|
|
10758
|
+
return "zip";
|
|
10759
|
+
}
|
|
10760
|
+
if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
|
|
10761
|
+
return "ole2";
|
|
10762
|
+
}
|
|
10763
|
+
const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
|
|
10764
|
+
if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
|
|
10765
|
+
return "json";
|
|
10766
|
+
}
|
|
10767
|
+
return "text";
|
|
10768
|
+
}
|
|
10769
|
+
/**
|
|
10770
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
10771
|
+
* Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
|
|
10772
|
+
*/
|
|
10773
|
+
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
10774
|
+
const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
|
|
10775
|
+
if (!jsonRows || jsonRows.length === 0) {
|
|
10776
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
10777
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
10778
|
+
${csv.trim()}` : "";
|
|
10779
|
+
}
|
|
10780
|
+
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
10781
|
+
let currentHeader = [];
|
|
10782
|
+
for (let r = 0; r < jsonRows.length; r++) {
|
|
10783
|
+
const row = jsonRows[r];
|
|
10784
|
+
if (!row || row.length === 0) continue;
|
|
10785
|
+
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
10786
|
+
if (cells.length === 0) continue;
|
|
10787
|
+
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
10788
|
+
lines.push(`
|
|
10789
|
+
## ${cells.join(" ")}`);
|
|
10790
|
+
continue;
|
|
10791
|
+
}
|
|
10792
|
+
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
10793
|
+
if (isHeaderCandidate && cells.length >= 2) {
|
|
10794
|
+
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
10795
|
+
lines.push(`| ${cells.join(" | ")} |`);
|
|
10796
|
+
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
10797
|
+
continue;
|
|
10798
|
+
}
|
|
10799
|
+
if (currentHeader.length > 0 && cells.length > 1) {
|
|
10800
|
+
const fieldPairs = [];
|
|
10801
|
+
for (let c = 0; c < row.length; c++) {
|
|
10802
|
+
const h = currentHeader[c];
|
|
10803
|
+
const val = row[c];
|
|
10804
|
+
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
10805
|
+
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
10806
|
+
}
|
|
10807
|
+
}
|
|
10808
|
+
if (fieldPairs.length > 0) {
|
|
10809
|
+
lines.push(fieldPairs.join(" | "));
|
|
10810
|
+
continue;
|
|
10811
|
+
}
|
|
10812
|
+
}
|
|
10813
|
+
lines.push(cells.join(", "));
|
|
10814
|
+
}
|
|
10815
|
+
return lines.join("\n");
|
|
10816
|
+
}
|
|
10415
10817
|
/**
|
|
10416
10818
|
* Extract text from a File or Buffer based on its type.
|
|
10417
10819
|
*/
|
|
10418
|
-
static async parse(file, fileName, mimeType) {
|
|
10820
|
+
static async parse(file, fileName, mimeType = "") {
|
|
10419
10821
|
var _a2;
|
|
10420
10822
|
const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
10421
|
-
if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
10823
|
+
if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
10422
10824
|
return this.readAsText(file);
|
|
10423
10825
|
}
|
|
10424
10826
|
if (extension === "json" || mimeType === "application/json") {
|
|
@@ -10430,38 +10832,115 @@ var DocumentParser = class {
|
|
|
10430
10832
|
return text;
|
|
10431
10833
|
}
|
|
10432
10834
|
}
|
|
10433
|
-
if (extension === "
|
|
10835
|
+
if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
|
|
10836
|
+
return this.readAsText(file);
|
|
10837
|
+
}
|
|
10838
|
+
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
10434
10839
|
return this.readAsText(file);
|
|
10435
10840
|
}
|
|
10436
|
-
if (extension === "
|
|
10841
|
+
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
10437
10842
|
try {
|
|
10438
|
-
const
|
|
10843
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
10439
10844
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10440
|
-
const
|
|
10441
|
-
|
|
10442
|
-
|
|
10443
|
-
|
|
10444
|
-
const
|
|
10445
|
-
|
|
10446
|
-
|
|
10447
|
-
|
|
10448
|
-
|
|
10845
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
10846
|
+
const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
|
|
10847
|
+
var _a3, _b;
|
|
10848
|
+
const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
|
|
10849
|
+
const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
|
|
10850
|
+
return numA - numB;
|
|
10851
|
+
});
|
|
10852
|
+
if (slideFiles.length > 0) {
|
|
10853
|
+
const slidesText = [];
|
|
10854
|
+
for (let i = 0; i < slideFiles.length; i++) {
|
|
10855
|
+
const slideXml = await zip.files[slideFiles[i]].async("text");
|
|
10856
|
+
const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
|
|
10857
|
+
if (textMatches && textMatches.length > 0) {
|
|
10858
|
+
const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
|
|
10859
|
+
if (slideContent.trim()) {
|
|
10860
|
+
slidesText.push(`--- Slide ${i + 1} ---
|
|
10861
|
+
${slideContent.trim()}`);
|
|
10862
|
+
}
|
|
10449
10863
|
}
|
|
10450
|
-
});
|
|
10451
|
-
const csv = csvLines.join("\n");
|
|
10452
|
-
if (csv && csv.trim()) {
|
|
10453
|
-
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
10454
|
-
${csv.trim()}`);
|
|
10455
10864
|
}
|
|
10456
|
-
|
|
10457
|
-
|
|
10458
|
-
|
|
10865
|
+
if (slidesText.length > 0) {
|
|
10866
|
+
return slidesText.join("\n\n");
|
|
10867
|
+
}
|
|
10868
|
+
}
|
|
10869
|
+
if (zip.files["content.xml"]) {
|
|
10870
|
+
const contentXml = await zip.files["content.xml"].async("text");
|
|
10871
|
+
const cleanText = this.stripXmlTags(contentXml);
|
|
10872
|
+
if (cleanText.trim()) {
|
|
10873
|
+
return cleanText.trim();
|
|
10874
|
+
}
|
|
10875
|
+
}
|
|
10876
|
+
const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
|
|
10877
|
+
const extractedSegments = [];
|
|
10878
|
+
for (const xf of allXmlFiles) {
|
|
10879
|
+
const xml = await zip.files[xf].async("text");
|
|
10880
|
+
const stripped = this.stripXmlTags(xml);
|
|
10881
|
+
if (stripped.length > 10) extractedSegments.push(stripped);
|
|
10882
|
+
}
|
|
10883
|
+
if (extractedSegments.length > 0) {
|
|
10884
|
+
return extractedSegments.join("\n\n");
|
|
10885
|
+
}
|
|
10886
|
+
return `[PowerPoint Presentation: ${fileName}]`;
|
|
10887
|
+
} catch (err) {
|
|
10888
|
+
console.warn("[DocumentParser] PowerPoint parsing failed:", err);
|
|
10459
10889
|
try {
|
|
10460
10890
|
return await this.readAsText(file);
|
|
10461
|
-
} catch (
|
|
10462
|
-
return `[
|
|
10891
|
+
} catch (e) {
|
|
10892
|
+
return `[PowerPoint Presentation File: ${fileName}]`;
|
|
10893
|
+
}
|
|
10894
|
+
}
|
|
10895
|
+
}
|
|
10896
|
+
if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
|
|
10897
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10898
|
+
try {
|
|
10899
|
+
const XLSX = await import("xlsx");
|
|
10900
|
+
const workbook = XLSX.read(buffer, { type: "buffer" });
|
|
10901
|
+
const sheetsText = [];
|
|
10902
|
+
for (const sheetName of workbook.SheetNames) {
|
|
10903
|
+
const sheet = workbook.Sheets[sheetName];
|
|
10904
|
+
const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
|
|
10905
|
+
if (formatted && formatted.trim()) {
|
|
10906
|
+
sheetsText.push(formatted.trim());
|
|
10907
|
+
}
|
|
10908
|
+
}
|
|
10909
|
+
if (sheetsText.length > 0) {
|
|
10910
|
+
return sheetsText.join("\n\n");
|
|
10463
10911
|
}
|
|
10912
|
+
} catch (xlsxErr) {
|
|
10913
|
+
console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
|
|
10464
10914
|
}
|
|
10915
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
10916
|
+
if (isZip) {
|
|
10917
|
+
try {
|
|
10918
|
+
const ExcelJS = await import("exceljs");
|
|
10919
|
+
const workbook = new ExcelJS.Workbook();
|
|
10920
|
+
await workbook.xlsx.load(buffer);
|
|
10921
|
+
const sheetsText = [];
|
|
10922
|
+
workbook.eachSheet((sheet) => {
|
|
10923
|
+
const csvLines = [];
|
|
10924
|
+
sheet.eachRow((row) => {
|
|
10925
|
+
const values = row.values;
|
|
10926
|
+
if (Array.isArray(values)) {
|
|
10927
|
+
csvLines.push(values.slice(1).join(","));
|
|
10928
|
+
}
|
|
10929
|
+
});
|
|
10930
|
+
const csv = csvLines.join("\n");
|
|
10931
|
+
if (csv && csv.trim()) {
|
|
10932
|
+
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
10933
|
+
${csv.trim()}`);
|
|
10934
|
+
}
|
|
10935
|
+
});
|
|
10936
|
+
if (sheetsText.length > 0) {
|
|
10937
|
+
return sheetsText.join("\n\n");
|
|
10938
|
+
}
|
|
10939
|
+
} catch (err) {
|
|
10940
|
+
console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
|
|
10941
|
+
}
|
|
10942
|
+
}
|
|
10943
|
+
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
10465
10944
|
}
|
|
10466
10945
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
10467
10946
|
try {
|
|
@@ -10475,15 +10954,48 @@ ${csv.trim()}`);
|
|
|
10475
10954
|
return `[PDF Parsing Error for ${fileName}]`;
|
|
10476
10955
|
}
|
|
10477
10956
|
}
|
|
10478
|
-
if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
|
|
10957
|
+
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
10479
10958
|
try {
|
|
10480
10959
|
const mammoth = await import("mammoth");
|
|
10481
10960
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10482
10961
|
const result = await mammoth.extractRawText({ buffer });
|
|
10483
|
-
|
|
10962
|
+
if (result.value && result.value.trim()) {
|
|
10963
|
+
return result.value;
|
|
10964
|
+
}
|
|
10965
|
+
} catch (err) {
|
|
10966
|
+
console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
|
|
10967
|
+
}
|
|
10968
|
+
try {
|
|
10969
|
+
const raw = await this.readAsText(file);
|
|
10970
|
+
if (extension === "rtf" || raw.startsWith("{\\rtf")) {
|
|
10971
|
+
return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
|
|
10972
|
+
}
|
|
10973
|
+
return raw;
|
|
10974
|
+
} catch (e) {
|
|
10975
|
+
return `[Word Document File: ${fileName}]`;
|
|
10976
|
+
}
|
|
10977
|
+
}
|
|
10978
|
+
if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
|
|
10979
|
+
const raw = await this.readAsText(file);
|
|
10980
|
+
return this.stripHtmlAndXml(raw);
|
|
10981
|
+
}
|
|
10982
|
+
if (extension === "epub" || mimeType.includes("epub")) {
|
|
10983
|
+
try {
|
|
10984
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
10985
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10986
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
10987
|
+
const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
|
|
10988
|
+
const chapters = [];
|
|
10989
|
+
for (const hf of htmlFiles) {
|
|
10990
|
+
const content = await zip.files[hf].async("text");
|
|
10991
|
+
const clean = this.stripHtmlAndXml(content);
|
|
10992
|
+
if (clean.length > 50) chapters.push(clean);
|
|
10993
|
+
}
|
|
10994
|
+
if (chapters.length > 0) {
|
|
10995
|
+
return chapters.join("\n\n---\n\n");
|
|
10996
|
+
}
|
|
10484
10997
|
} catch (err) {
|
|
10485
|
-
console.warn(
|
|
10486
|
-
return `[Word Parsing Error for ${fileName}]`;
|
|
10998
|
+
console.warn("[DocumentParser] EPUB parsing failed:", err);
|
|
10487
10999
|
}
|
|
10488
11000
|
}
|
|
10489
11001
|
try {
|
|
@@ -10492,6 +11004,237 @@ ${csv.trim()}`);
|
|
|
10492
11004
|
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
10493
11005
|
}
|
|
10494
11006
|
}
|
|
11007
|
+
/**
|
|
11008
|
+
* Helper to strip XML tags and normalize inner text content.
|
|
11009
|
+
*/
|
|
11010
|
+
static stripXmlTags(xml) {
|
|
11011
|
+
return xml.replace(/<[^>]+>/g, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/\s+/g, " ").trim();
|
|
11012
|
+
}
|
|
11013
|
+
/**
|
|
11014
|
+
* Helper to strip HTML / XML tags while removing script and style blocks completely.
|
|
11015
|
+
*/
|
|
11016
|
+
static stripHtmlAndXml(html) {
|
|
11017
|
+
return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/ /gi, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
|
|
11018
|
+
}
|
|
11019
|
+
/**
|
|
11020
|
+
* Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
|
|
11021
|
+
*/
|
|
11022
|
+
static parseLegacyExcelOrFallback(buffer, fileName) {
|
|
11023
|
+
const rawStr = buffer.toString("utf-8");
|
|
11024
|
+
if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
|
|
11025
|
+
const rows = [];
|
|
11026
|
+
const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
|
|
11027
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11028
|
+
for (const rowHtml of rowMatches) {
|
|
11029
|
+
const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
|
|
11030
|
+
if (cells && cells.length > 0) {
|
|
11031
|
+
const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
|
|
11032
|
+
if (rowText) rows.push(rowText);
|
|
11033
|
+
}
|
|
11034
|
+
}
|
|
11035
|
+
}
|
|
11036
|
+
if (rows.length > 0) {
|
|
11037
|
+
return rows.join("\n");
|
|
11038
|
+
}
|
|
11039
|
+
const clean = this.stripHtmlAndXml(rawStr);
|
|
11040
|
+
if (clean.length > 10) return clean;
|
|
11041
|
+
}
|
|
11042
|
+
if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
|
|
11043
|
+
const rows = [];
|
|
11044
|
+
const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
|
|
11045
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11046
|
+
for (const rowXml of rowMatches) {
|
|
11047
|
+
const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
|
|
11048
|
+
if (dataMatches && dataMatches.length > 0) {
|
|
11049
|
+
const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
|
|
11050
|
+
if (rowText) rows.push(rowText);
|
|
11051
|
+
}
|
|
11052
|
+
}
|
|
11053
|
+
}
|
|
11054
|
+
if (rows.length > 0) {
|
|
11055
|
+
return rows.join("\n");
|
|
11056
|
+
}
|
|
11057
|
+
}
|
|
11058
|
+
const extractedStrings = [];
|
|
11059
|
+
const ignoredOleMarkers = /* @__PURE__ */ new Set([
|
|
11060
|
+
"root entry",
|
|
11061
|
+
"workbook",
|
|
11062
|
+
"compobj",
|
|
11063
|
+
"summaryinformation",
|
|
11064
|
+
"documentsummaryinformation",
|
|
11065
|
+
"microsoft excel",
|
|
11066
|
+
"worksheet",
|
|
11067
|
+
"biff8",
|
|
11068
|
+
"calcpr"
|
|
11069
|
+
]);
|
|
11070
|
+
let currentUtf16 = "";
|
|
11071
|
+
for (let i = 0; i < buffer.length - 1; i += 2) {
|
|
11072
|
+
const charCode = buffer[i];
|
|
11073
|
+
const zeroByte = buffer[i + 1];
|
|
11074
|
+
if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
|
|
11075
|
+
currentUtf16 += String.fromCharCode(charCode);
|
|
11076
|
+
} else {
|
|
11077
|
+
if (currentUtf16.length >= 2) {
|
|
11078
|
+
const trimmed = currentUtf16.trim();
|
|
11079
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11080
|
+
extractedStrings.push(trimmed);
|
|
11081
|
+
}
|
|
11082
|
+
}
|
|
11083
|
+
currentUtf16 = "";
|
|
11084
|
+
}
|
|
11085
|
+
}
|
|
11086
|
+
let currentAscii = "";
|
|
11087
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
11088
|
+
const byte = buffer[i];
|
|
11089
|
+
if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
|
|
11090
|
+
currentAscii += String.fromCharCode(byte);
|
|
11091
|
+
} else {
|
|
11092
|
+
if (currentAscii.length >= 2) {
|
|
11093
|
+
const trimmed = currentAscii.trim();
|
|
11094
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11095
|
+
extractedStrings.push(trimmed);
|
|
11096
|
+
}
|
|
11097
|
+
}
|
|
11098
|
+
currentAscii = "";
|
|
11099
|
+
}
|
|
11100
|
+
}
|
|
11101
|
+
const uniqueStrings = [];
|
|
11102
|
+
for (const str of extractedStrings) {
|
|
11103
|
+
if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
|
|
11104
|
+
uniqueStrings.push(str);
|
|
11105
|
+
}
|
|
11106
|
+
}
|
|
11107
|
+
if (uniqueStrings.length > 0) {
|
|
11108
|
+
return uniqueStrings.join(" | ");
|
|
11109
|
+
}
|
|
11110
|
+
const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
|
|
11111
|
+
return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
|
|
11112
|
+
}
|
|
11113
|
+
/**
|
|
11114
|
+
* Dynamically infer document domain and schema from extracted text, filename, and format.
|
|
11115
|
+
*/
|
|
11116
|
+
static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
|
|
11117
|
+
const lowerText = text.toLowerCase();
|
|
11118
|
+
const lowerName = fileName.toLowerCase();
|
|
11119
|
+
const financialTerms = [
|
|
11120
|
+
"price per share",
|
|
11121
|
+
"share price",
|
|
11122
|
+
"total cost of the deal",
|
|
11123
|
+
"cost of the deal",
|
|
11124
|
+
"lbo",
|
|
11125
|
+
"merger",
|
|
11126
|
+
"valuation",
|
|
11127
|
+
"d/e ratio",
|
|
11128
|
+
"ebit",
|
|
11129
|
+
"ebitda",
|
|
11130
|
+
"fcfe",
|
|
11131
|
+
"fcff",
|
|
11132
|
+
"wacc",
|
|
11133
|
+
"beta",
|
|
11134
|
+
"working capital",
|
|
11135
|
+
"cogs",
|
|
11136
|
+
"cap spending",
|
|
11137
|
+
"depreciation",
|
|
11138
|
+
"cashflow",
|
|
11139
|
+
"cash flow",
|
|
11140
|
+
"net income",
|
|
11141
|
+
"balance sheet",
|
|
11142
|
+
"income statement",
|
|
11143
|
+
"interest rate",
|
|
11144
|
+
"terminal year",
|
|
11145
|
+
"equity",
|
|
11146
|
+
"debt outstanding",
|
|
11147
|
+
"gross margin",
|
|
11148
|
+
"fiscal year",
|
|
11149
|
+
"fy20",
|
|
11150
|
+
"fy21",
|
|
11151
|
+
"fy22",
|
|
11152
|
+
"fy23",
|
|
11153
|
+
"fy24",
|
|
11154
|
+
"fy25",
|
|
11155
|
+
"q1",
|
|
11156
|
+
"q2",
|
|
11157
|
+
"q3",
|
|
11158
|
+
"q4"
|
|
11159
|
+
];
|
|
11160
|
+
let financialScore = 0;
|
|
11161
|
+
for (const term of financialTerms) {
|
|
11162
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11163
|
+
financialScore += 1;
|
|
11164
|
+
}
|
|
11165
|
+
}
|
|
11166
|
+
const ecommerceTerms = [
|
|
11167
|
+
"sku",
|
|
11168
|
+
"product_id",
|
|
11169
|
+
"product id",
|
|
11170
|
+
"item_id",
|
|
11171
|
+
"add to cart",
|
|
11172
|
+
"buy now",
|
|
11173
|
+
"in stock",
|
|
11174
|
+
"out of stock",
|
|
11175
|
+
"inventory count",
|
|
11176
|
+
"retail price",
|
|
11177
|
+
"unit price",
|
|
11178
|
+
"msrp",
|
|
11179
|
+
"cart",
|
|
11180
|
+
"shop",
|
|
11181
|
+
"catalog",
|
|
11182
|
+
"product_name",
|
|
11183
|
+
"product name",
|
|
11184
|
+
"brand name"
|
|
11185
|
+
];
|
|
11186
|
+
let ecommerceScore = 0;
|
|
11187
|
+
for (const term of ecommerceTerms) {
|
|
11188
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11189
|
+
ecommerceScore += 1;
|
|
11190
|
+
}
|
|
11191
|
+
}
|
|
11192
|
+
const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
|
|
11193
|
+
let legalScore = 0;
|
|
11194
|
+
for (const term of legalTerms) {
|
|
11195
|
+
if (lowerText.includes(term)) legalScore += 1;
|
|
11196
|
+
}
|
|
11197
|
+
const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
|
|
11198
|
+
let healthScore = 0;
|
|
11199
|
+
for (const term of healthTerms) {
|
|
11200
|
+
if (lowerText.includes(term)) healthScore += 1;
|
|
11201
|
+
}
|
|
11202
|
+
let schema;
|
|
11203
|
+
const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
11204
|
+
if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
|
|
11205
|
+
const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
|
|
11206
|
+
const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
|
|
11207
|
+
if (headers.length > 1 && lines.length > 1) {
|
|
11208
|
+
const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
|
|
11209
|
+
schema = {};
|
|
11210
|
+
headers.forEach((h, idx) => {
|
|
11211
|
+
const val = sampleRow[idx] || "";
|
|
11212
|
+
if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
|
|
11213
|
+
schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
|
|
11214
|
+
} else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
|
|
11215
|
+
schema[h] = "date";
|
|
11216
|
+
} else {
|
|
11217
|
+
schema[h] = "string";
|
|
11218
|
+
}
|
|
11219
|
+
});
|
|
11220
|
+
}
|
|
11221
|
+
}
|
|
11222
|
+
const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
|
|
11223
|
+
const isCatalog = ecommerceScore >= 2 && !isFinancial;
|
|
11224
|
+
let domain = "general";
|
|
11225
|
+
if (isFinancial) {
|
|
11226
|
+
domain = "finance";
|
|
11227
|
+
} else if (isCatalog) {
|
|
11228
|
+
domain = "ecommerce";
|
|
11229
|
+
} else if (legalScore >= 2) {
|
|
11230
|
+
domain = "legal";
|
|
11231
|
+
} else if (healthScore >= 2) {
|
|
11232
|
+
domain = "healthcare";
|
|
11233
|
+
} else if (schema && Object.keys(schema).length > 0) {
|
|
11234
|
+
domain = "tabular";
|
|
11235
|
+
}
|
|
11236
|
+
return { domain, schema, isCatalog, isFinancial };
|
|
11237
|
+
}
|
|
10495
11238
|
static async readAsText(file) {
|
|
10496
11239
|
if (Buffer.isBuffer(file)) {
|
|
10497
11240
|
return file.toString("utf-8");
|
|
@@ -11138,18 +11881,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
|
|
|
11138
11881
|
}
|
|
11139
11882
|
}
|
|
11140
11883
|
function getAllowedNamespaces(projectId) {
|
|
11141
|
-
const trimmed = projectId.trim();
|
|
11884
|
+
const trimmed = (projectId || "default").trim();
|
|
11142
11885
|
const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
|
|
11143
11886
|
return new Set([
|
|
11144
11887
|
trimmed,
|
|
11145
11888
|
withoutPrefix,
|
|
11146
|
-
`retrivora-${withoutPrefix}
|
|
11889
|
+
`retrivora-${withoutPrefix}`,
|
|
11890
|
+
"default",
|
|
11891
|
+
"retrivora-default"
|
|
11147
11892
|
].filter(Boolean));
|
|
11148
11893
|
}
|
|
11149
11894
|
function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
11895
|
+
var _a2;
|
|
11150
11896
|
const allowed = getAllowedNamespaces(payload.projectId);
|
|
11151
11897
|
const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
|
|
11152
|
-
|
|
11898
|
+
const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
|
|
11899
|
+
if (requested && !isDevOrFree && !allowed.has(requested)) {
|
|
11153
11900
|
return {
|
|
11154
11901
|
ok: false,
|
|
11155
11902
|
response: new Response(JSON.stringify({
|
|
@@ -11163,7 +11910,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
|
11163
11910
|
})
|
|
11164
11911
|
};
|
|
11165
11912
|
}
|
|
11166
|
-
const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
|
|
11913
|
+
const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
|
|
11167
11914
|
return { ok: true, namespace: requested || defaultNamespace };
|
|
11168
11915
|
}
|
|
11169
11916
|
var MAX_MESSAGE_LENGTH = 8e3;
|
|
@@ -11607,7 +12354,7 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
11607
12354
|
function createHealthHandler(configOrPlugin, options) {
|
|
11608
12355
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
11609
12356
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
11610
|
-
return async function GET(req) {
|
|
12357
|
+
return async function GET(req, context) {
|
|
11611
12358
|
if (req) {
|
|
11612
12359
|
const authResult = await checkAuth(req, onAuthorize);
|
|
11613
12360
|
if (authResult) return authResult;
|
|
@@ -11718,9 +12465,10 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
11718
12465
|
}
|
|
11719
12466
|
const documents = [];
|
|
11720
12467
|
for (const file of files) {
|
|
12468
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11721
12469
|
const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
|
|
11722
12470
|
if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
|
|
11723
|
-
const text =
|
|
12471
|
+
const text = buffer.toString("utf-8");
|
|
11724
12472
|
const Papa = await import("papaparse");
|
|
11725
12473
|
const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
|
|
11726
12474
|
if (parsed.data && parsed.data.length > 0) {
|
|
@@ -11760,49 +12508,70 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
11760
12508
|
}
|
|
11761
12509
|
}
|
|
11762
12510
|
} else if (isExcel) {
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
11774
|
-
|
|
11775
|
-
|
|
11776
|
-
|
|
11777
|
-
|
|
11778
|
-
|
|
11779
|
-
|
|
11780
|
-
|
|
11781
|
-
|
|
11782
|
-
|
|
11783
|
-
|
|
11784
|
-
|
|
11785
|
-
|
|
11786
|
-
|
|
11787
|
-
|
|
11788
|
-
|
|
11789
|
-
|
|
11790
|
-
|
|
11791
|
-
|
|
11792
|
-
|
|
11793
|
-
|
|
11794
|
-
|
|
11795
|
-
|
|
11796
|
-
|
|
11797
|
-
|
|
11798
|
-
|
|
11799
|
-
|
|
12511
|
+
let excelLoaded = false;
|
|
12512
|
+
try {
|
|
12513
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
12514
|
+
if (isZip) {
|
|
12515
|
+
const ExcelJS = await import("exceljs");
|
|
12516
|
+
const workbook = new ExcelJS.Workbook();
|
|
12517
|
+
await workbook.xlsx.load(buffer);
|
|
12518
|
+
workbook.eachSheet((sheet, sheetId) => {
|
|
12519
|
+
const sheetName = sheet.name;
|
|
12520
|
+
let headers = [];
|
|
12521
|
+
let i = 0;
|
|
12522
|
+
sheet.eachRow((row, rowNumber) => {
|
|
12523
|
+
const values = Array.isArray(row.values) ? row.values.slice(1) : [];
|
|
12524
|
+
if (rowNumber === 1) {
|
|
12525
|
+
headers = values.map((v) => String(v));
|
|
12526
|
+
return;
|
|
12527
|
+
}
|
|
12528
|
+
i++;
|
|
12529
|
+
const contentParts = [];
|
|
12530
|
+
const rowData = {};
|
|
12531
|
+
values.forEach((val, index) => {
|
|
12532
|
+
const key = headers[index] || `Column${index + 1}`;
|
|
12533
|
+
rowData[key] = val;
|
|
12534
|
+
if (key && val !== void 0 && val !== null && String(val).trim()) {
|
|
12535
|
+
contentParts.push(`${key}: ${val}`);
|
|
12536
|
+
}
|
|
12537
|
+
});
|
|
12538
|
+
if (contentParts.length > 0) {
|
|
12539
|
+
documents.push({
|
|
12540
|
+
docId: `${file.name}-${sheetName}-row-${i}`,
|
|
12541
|
+
content: contentParts.join(", "),
|
|
12542
|
+
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
12543
|
+
fileName: file.name,
|
|
12544
|
+
sheetName,
|
|
12545
|
+
fileSize: file.size,
|
|
12546
|
+
fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
12547
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12548
|
+
}, dimension ? { dimension } : {}), {
|
|
12549
|
+
headers
|
|
12550
|
+
}), rowData)
|
|
12551
|
+
});
|
|
12552
|
+
}
|
|
11800
12553
|
});
|
|
11801
|
-
}
|
|
12554
|
+
});
|
|
12555
|
+
excelLoaded = documents.length > 0;
|
|
12556
|
+
}
|
|
12557
|
+
} catch (excelErr) {
|
|
12558
|
+
excelLoaded = false;
|
|
12559
|
+
}
|
|
12560
|
+
if (!excelLoaded) {
|
|
12561
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
12562
|
+
documents.push({
|
|
12563
|
+
docId: file.name,
|
|
12564
|
+
content,
|
|
12565
|
+
metadata: __spreadValues({
|
|
12566
|
+
fileName: file.name,
|
|
12567
|
+
fileSize: file.size,
|
|
12568
|
+
fileType: file.type || "application/vnd.ms-excel",
|
|
12569
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12570
|
+
}, dimension ? { dimension } : {})
|
|
11802
12571
|
});
|
|
11803
|
-
}
|
|
12572
|
+
}
|
|
11804
12573
|
} else {
|
|
11805
|
-
const content = await DocumentParser.parse(
|
|
12574
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
11806
12575
|
documents.push({
|
|
11807
12576
|
docId: file.name,
|
|
11808
12577
|
content,
|
|
@@ -11961,7 +12730,7 @@ function createSessionsHandler(configOrPlugin, options) {
|
|
|
11961
12730
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
11962
12731
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
11963
12732
|
const onResolveScope = options == null ? void 0 : options.onResolveScope;
|
|
11964
|
-
return async function GET(req) {
|
|
12733
|
+
return async function GET(req, context) {
|
|
11965
12734
|
if (req) {
|
|
11966
12735
|
const authResult = await checkAuth(req, onAuthorize);
|
|
11967
12736
|
if (authResult) return authResult;
|