@retrivora-ai/rag-engine 2.5.6 → 2.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-DwFj0h2u.d.mts → ILLMProvider-B2N3lMBc.d.mts} +10 -1
- package/dist/{ILLMProvider-DwFj0h2u.d.ts → ILLMProvider-B2N3lMBc.d.ts} +10 -1
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +961 -192
- package/dist/handlers/index.mjs +961 -192
- package/dist/{index-Cb0GsT16.d.ts → index-37V810a-.d.mts} +3 -3
- package/dist/{index-D1jN8KSF.d.mts → index-BxzDv2jd.d.ts} +3 -3
- package/dist/{index-Byv5MaYi.d.mts → index-CAGGPyI7.d.mts} +22 -2
- package/dist/{index-DP3_OpuB.d.ts → index-DdQxoIpu.d.ts} +22 -2
- package/dist/index.css +188 -4
- package/dist/index.d.mts +178 -20
- package/dist/index.d.ts +178 -20
- package/dist/index.js +735 -195
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +731 -193
- package/dist/server.d.mts +100 -9
- package/dist/server.d.ts +100 -9
- package/dist/server.js +963 -192
- package/dist/server.mjs +962 -192
- package/package.json +6 -3
package/dist/server.mjs
CHANGED
|
@@ -88,8 +88,8 @@ function buildPayload(template, vars) {
|
|
|
88
88
|
let raw = template;
|
|
89
89
|
for (const [key, val] of Object.entries(vars)) {
|
|
90
90
|
const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
|
|
91
|
-
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
|
|
92
|
-
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
|
|
91
|
+
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
|
|
92
|
+
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
|
|
93
93
|
}
|
|
94
94
|
try {
|
|
95
95
|
return JSON.parse(raw);
|
|
@@ -326,7 +326,7 @@ var init_package = __esm({
|
|
|
326
326
|
"package.json"() {
|
|
327
327
|
package_default = {
|
|
328
328
|
name: "@retrivora-ai/rag-engine",
|
|
329
|
-
version: "2.5.
|
|
329
|
+
version: "2.5.8",
|
|
330
330
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
331
331
|
author: "Abhinav Alkuchi",
|
|
332
332
|
license: "UNLICENSED",
|
|
@@ -429,9 +429,11 @@ var init_package = __esm({
|
|
|
429
429
|
"@google/genai": "^0.8.0",
|
|
430
430
|
"@google/generative-ai": "^0.24.1",
|
|
431
431
|
"@pinecone-database/pinecone": "^7.2.0",
|
|
432
|
-
"@retrivora-ai/billing-guards": "
|
|
432
|
+
"@retrivora-ai/billing-guards": "file:../billing-guards",
|
|
433
433
|
"@types/papaparse": "^5.5.2",
|
|
434
|
+
exceljs: "^4.4.0",
|
|
434
435
|
"framer-motion": "^13.1.0",
|
|
436
|
+
jszip: "^3.10.1",
|
|
435
437
|
katex: "^0.18.4",
|
|
436
438
|
"lucide-react": "^1.8.0",
|
|
437
439
|
"next-themes": "^0.4.6",
|
|
@@ -444,7 +446,7 @@ var init_package = __esm({
|
|
|
444
446
|
"rehype-katex": "^7.0.1",
|
|
445
447
|
"remark-gfm": "^4.0.1",
|
|
446
448
|
"remark-math": "^6.0.0",
|
|
447
|
-
|
|
449
|
+
xlsx: "^0.18.5",
|
|
448
450
|
zod: "^4.4.3"
|
|
449
451
|
},
|
|
450
452
|
optionalDependencies: {
|
|
@@ -462,6 +464,7 @@ var init_package = __esm({
|
|
|
462
464
|
"@tailwindcss/postcss": "^4",
|
|
463
465
|
"@types/estree": "^1.0.9",
|
|
464
466
|
"@types/jest": "^30.0.0",
|
|
467
|
+
"@types/jszip": "^3.4.1",
|
|
465
468
|
"@types/katex": "^0.16.8",
|
|
466
469
|
"@types/node": "^20",
|
|
467
470
|
"@types/pdf-parse": "^1.1.5",
|
|
@@ -2864,6 +2867,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2864
2867
|
try {
|
|
2865
2868
|
response = await fetch(endpoint, {
|
|
2866
2869
|
method: "POST",
|
|
2870
|
+
signal: AbortSignal.timeout(3e3),
|
|
2867
2871
|
headers: {
|
|
2868
2872
|
"Content-Type": "application/json",
|
|
2869
2873
|
"x-license-key": licenseKey
|
|
@@ -5589,15 +5593,29 @@ Text to extract from:
|
|
|
5589
5593
|
const response = await this.llm.chat([
|
|
5590
5594
|
{ role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
|
|
5591
5595
|
{ role: "user", content: prompt }
|
|
5592
|
-
], "", { maxTokens:
|
|
5596
|
+
], "", { maxTokens: 800, temperature: 0.1 });
|
|
5593
5597
|
try {
|
|
5594
|
-
const
|
|
5595
|
-
const
|
|
5596
|
-
|
|
5598
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5599
|
+
const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
|
5600
|
+
const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
|
|
5601
|
+
const jsonMatch = candidate.match(/\{[\s\S]*\}/);
|
|
5602
|
+
const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
|
|
5603
|
+
const parsed = JSON.parse(cleanJson);
|
|
5604
|
+
return {
|
|
5605
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5606
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5607
|
+
};
|
|
5597
5608
|
} catch (e) {
|
|
5598
5609
|
try {
|
|
5599
|
-
const
|
|
5600
|
-
|
|
5610
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5611
|
+
const repaired = this.repairTruncatedJson(withoutThinking);
|
|
5612
|
+
if (repaired) {
|
|
5613
|
+
const parsed = JSON.parse(repaired);
|
|
5614
|
+
return {
|
|
5615
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5616
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5617
|
+
};
|
|
5618
|
+
}
|
|
5601
5619
|
} catch (e2) {
|
|
5602
5620
|
}
|
|
5603
5621
|
console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
|
|
@@ -5875,6 +5893,208 @@ ${error instanceof Error ? error.message : String(error)}`
|
|
|
5875
5893
|
}
|
|
5876
5894
|
};
|
|
5877
5895
|
|
|
5896
|
+
// src/rag/PromptOptimizer.ts
|
|
5897
|
+
var PromptOptimizer = class {
|
|
5898
|
+
/**
|
|
5899
|
+
* Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
|
|
5900
|
+
*/
|
|
5901
|
+
static optimize(rawPrompt, history = [], options = {}) {
|
|
5902
|
+
if (!rawPrompt || typeof rawPrompt !== "string") {
|
|
5903
|
+
return {
|
|
5904
|
+
optimizedPrompt: "",
|
|
5905
|
+
originalPrompt: rawPrompt || "",
|
|
5906
|
+
wasModified: false,
|
|
5907
|
+
transformations: []
|
|
5908
|
+
};
|
|
5909
|
+
}
|
|
5910
|
+
const { resolveHistory = true, expandKeywords = true } = options;
|
|
5911
|
+
const transformations = [];
|
|
5912
|
+
let current = rawPrompt;
|
|
5913
|
+
const cleaned = this.cleanText(current);
|
|
5914
|
+
if (cleaned !== current) {
|
|
5915
|
+
transformations.push("clean_whitespace_and_artifacts");
|
|
5916
|
+
current = cleaned;
|
|
5917
|
+
}
|
|
5918
|
+
const expandedAbbr = this.expandAbbreviations(current);
|
|
5919
|
+
if (expandedAbbr !== current) {
|
|
5920
|
+
transformations.push("expand_abbreviations");
|
|
5921
|
+
current = expandedAbbr;
|
|
5922
|
+
}
|
|
5923
|
+
if (expandKeywords) {
|
|
5924
|
+
const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
5925
|
+
if (this.TOPIC_EXPANSIONS[trimmedLower]) {
|
|
5926
|
+
current = this.TOPIC_EXPANSIONS[trimmedLower];
|
|
5927
|
+
transformations.push(`topic_expansion_${trimmedLower}`);
|
|
5928
|
+
}
|
|
5929
|
+
}
|
|
5930
|
+
if (resolveHistory && history && history.length > 0) {
|
|
5931
|
+
const resolved = this.resolveCoreferences(current, history);
|
|
5932
|
+
if (resolved !== current) {
|
|
5933
|
+
transformations.push("resolve_history_coreferences");
|
|
5934
|
+
current = resolved;
|
|
5935
|
+
}
|
|
5936
|
+
}
|
|
5937
|
+
if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
|
|
5938
|
+
if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
|
|
5939
|
+
current = `${current.trim()}?`;
|
|
5940
|
+
transformations.push("add_question_mark");
|
|
5941
|
+
}
|
|
5942
|
+
}
|
|
5943
|
+
return {
|
|
5944
|
+
optimizedPrompt: current,
|
|
5945
|
+
originalPrompt: rawPrompt,
|
|
5946
|
+
wasModified: current !== rawPrompt,
|
|
5947
|
+
transformations
|
|
5948
|
+
};
|
|
5949
|
+
}
|
|
5950
|
+
/**
|
|
5951
|
+
* Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
|
|
5952
|
+
*/
|
|
5953
|
+
static cleanText(text) {
|
|
5954
|
+
let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
|
|
5955
|
+
const doubleQuotesCount = (clean.match(/"/g) || []).length;
|
|
5956
|
+
if (doubleQuotesCount % 2 !== 0) {
|
|
5957
|
+
clean += '"';
|
|
5958
|
+
}
|
|
5959
|
+
const singleQuotesCount = (clean.match(/'/g) || []).length;
|
|
5960
|
+
if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
|
|
5961
|
+
clean += "'";
|
|
5962
|
+
}
|
|
5963
|
+
const openParen = (clean.match(/\(/g) || []).length;
|
|
5964
|
+
const closeParen = (clean.match(/\)/g) || []).length;
|
|
5965
|
+
if (openParen > closeParen) {
|
|
5966
|
+
clean += ")".repeat(openParen - closeParen);
|
|
5967
|
+
}
|
|
5968
|
+
return clean;
|
|
5969
|
+
}
|
|
5970
|
+
/**
|
|
5971
|
+
* Replaces shorthand chat abbreviations with clear natural language.
|
|
5972
|
+
*/
|
|
5973
|
+
static expandAbbreviations(text) {
|
|
5974
|
+
let expanded = text;
|
|
5975
|
+
for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
|
|
5976
|
+
expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
|
|
5977
|
+
}
|
|
5978
|
+
return expanded;
|
|
5979
|
+
}
|
|
5980
|
+
/**
|
|
5981
|
+
* Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
|
|
5982
|
+
*/
|
|
5983
|
+
static resolveCoreferences(prompt, history) {
|
|
5984
|
+
const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
|
|
5985
|
+
if (!pronounPattern.test(prompt)) {
|
|
5986
|
+
return prompt;
|
|
5987
|
+
}
|
|
5988
|
+
const recentMessages = [...history].slice(-4).reverse();
|
|
5989
|
+
let detectedEntity = null;
|
|
5990
|
+
for (const msg of recentMessages) {
|
|
5991
|
+
if (!msg.content) continue;
|
|
5992
|
+
const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
|
|
5993
|
+
if (quotedMatch) {
|
|
5994
|
+
detectedEntity = quotedMatch[1];
|
|
5995
|
+
break;
|
|
5996
|
+
}
|
|
5997
|
+
const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
|
|
5998
|
+
if (headingMatch) {
|
|
5999
|
+
detectedEntity = headingMatch[1].trim();
|
|
6000
|
+
break;
|
|
6001
|
+
}
|
|
6002
|
+
const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
|
|
6003
|
+
if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
|
|
6004
|
+
detectedEntity = entityMatches[1];
|
|
6005
|
+
break;
|
|
6006
|
+
}
|
|
6007
|
+
}
|
|
6008
|
+
if (detectedEntity) {
|
|
6009
|
+
let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
|
|
6010
|
+
return resolved;
|
|
6011
|
+
}
|
|
6012
|
+
return prompt;
|
|
6013
|
+
}
|
|
6014
|
+
/**
|
|
6015
|
+
* Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
|
|
6016
|
+
*/
|
|
6017
|
+
static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
|
|
6018
|
+
const fastResult = this.optimize(rawPrompt, history);
|
|
6019
|
+
if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
|
|
6020
|
+
return fastResult;
|
|
6021
|
+
}
|
|
6022
|
+
try {
|
|
6023
|
+
const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
|
|
6024
|
+
const prompt = `You are a query optimizer in an enterprise RAG system.
|
|
6025
|
+
Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
|
|
6026
|
+
- Do NOT answer the question.
|
|
6027
|
+
- Resolve any pronouns like "it", "they", "that" using the context.
|
|
6028
|
+
- Keep the user's intent 100% intact.
|
|
6029
|
+
- If the question is already clear, return it as is.
|
|
6030
|
+
- Return ONLY the optimized query string. No quotes or explanations.
|
|
6031
|
+
|
|
6032
|
+
Context:
|
|
6033
|
+
${historyContext || "No previous context."}
|
|
6034
|
+
|
|
6035
|
+
User Question: "${rawPrompt}"
|
|
6036
|
+
|
|
6037
|
+
Optimized Search Prompt:`;
|
|
6038
|
+
const response = await llmProvider.chat(
|
|
6039
|
+
[{ role: "user", content: prompt }],
|
|
6040
|
+
"",
|
|
6041
|
+
{
|
|
6042
|
+
systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
|
|
6043
|
+
temperature: 0,
|
|
6044
|
+
maxTokens: 100
|
|
6045
|
+
}
|
|
6046
|
+
);
|
|
6047
|
+
const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
|
|
6048
|
+
if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
|
|
6049
|
+
return {
|
|
6050
|
+
optimizedPrompt: cleaned,
|
|
6051
|
+
originalPrompt: rawPrompt,
|
|
6052
|
+
wasModified: cleaned !== rawPrompt,
|
|
6053
|
+
transformations: [...fastResult.transformations, "llm_deep_reformulation"]
|
|
6054
|
+
};
|
|
6055
|
+
}
|
|
6056
|
+
} catch (err) {
|
|
6057
|
+
console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
|
|
6058
|
+
}
|
|
6059
|
+
return fastResult;
|
|
6060
|
+
}
|
|
6061
|
+
};
|
|
6062
|
+
PromptOptimizer.COMMON_ABBREVIATIONS = {
|
|
6063
|
+
"\\bpls\\b": "please",
|
|
6064
|
+
"\\bplz\\b": "please",
|
|
6065
|
+
"\\bthx\\b": "thanks",
|
|
6066
|
+
"\\bty\\b": "thank you",
|
|
6067
|
+
"\\bapprox\\b": "approximately",
|
|
6068
|
+
"\\bw/o\\b": "without",
|
|
6069
|
+
"\\bw/\\b": "with",
|
|
6070
|
+
"\\bdiff\\b": "difference",
|
|
6071
|
+
"\\binfo\\b": "information",
|
|
6072
|
+
"\\bconfig\\b": "configuration",
|
|
6073
|
+
"\\bdoc\\b": "document",
|
|
6074
|
+
"\\bdocs\\b": "documentation",
|
|
6075
|
+
"\\bfeat\\b": "feature",
|
|
6076
|
+
"\\bfeats\\b": "features",
|
|
6077
|
+
"\\bvs\\b": "versus",
|
|
6078
|
+
"\\bu\\b": "you",
|
|
6079
|
+
"\\bur\\b": "your",
|
|
6080
|
+
"\\br\\b": "are"
|
|
6081
|
+
};
|
|
6082
|
+
PromptOptimizer.TOPIC_EXPANSIONS = {
|
|
6083
|
+
"pricing": "What are the available pricing plans, tiers, and cost structures?",
|
|
6084
|
+
"price": "What is the pricing and subscription cost?",
|
|
6085
|
+
"features": "What are the core features, capabilities, and tools provided?",
|
|
6086
|
+
"security": "What are the security, privacy, and compliance standards?",
|
|
6087
|
+
"privacy": "What is the privacy policy and how is data protected?",
|
|
6088
|
+
"refund": "What is the refund and cancellation policy?",
|
|
6089
|
+
"refunds": "What is the refund and cancellation policy?",
|
|
6090
|
+
"contact": "How can I contact customer support or sales?",
|
|
6091
|
+
"support": "What support channels and assistance are available?",
|
|
6092
|
+
"docs": "Where can I find the official documentation and setup guides?",
|
|
6093
|
+
"installation": "How do I install and configure the SDK or service?",
|
|
6094
|
+
"install": "What are the step-by-step installation instructions?",
|
|
6095
|
+
"faq": "What are the frequently asked questions and key answers?"
|
|
6096
|
+
};
|
|
6097
|
+
|
|
5878
6098
|
// src/core/mcp.ts
|
|
5879
6099
|
import { spawn } from "child_process";
|
|
5880
6100
|
var MCPClient = class {
|
|
@@ -7098,6 +7318,26 @@ var IntentClassifier = class {
|
|
|
7098
7318
|
static extractDataSignals(context) {
|
|
7099
7319
|
const docs = context.retrievedDocuments || [];
|
|
7100
7320
|
const rowCount = docs.length;
|
|
7321
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
7322
|
+
"filetype",
|
|
7323
|
+
"mimetype",
|
|
7324
|
+
"filename",
|
|
7325
|
+
"docid",
|
|
7326
|
+
"uploadedat",
|
|
7327
|
+
"dimension",
|
|
7328
|
+
"chunkindex",
|
|
7329
|
+
"totalchunks",
|
|
7330
|
+
"charactercount",
|
|
7331
|
+
"workspaceid",
|
|
7332
|
+
"pineconenamespace",
|
|
7333
|
+
"tier",
|
|
7334
|
+
"custommetadata",
|
|
7335
|
+
"csvheaders",
|
|
7336
|
+
"headers",
|
|
7337
|
+
"filesize",
|
|
7338
|
+
"content",
|
|
7339
|
+
"id"
|
|
7340
|
+
]);
|
|
7101
7341
|
let hasNumericFields = false;
|
|
7102
7342
|
let hasDateFields = false;
|
|
7103
7343
|
let hasCategoricalFields = false;
|
|
@@ -7113,6 +7353,7 @@ var IntentClassifier = class {
|
|
|
7113
7353
|
docs.forEach((doc) => {
|
|
7114
7354
|
const meta = doc.metadata || {};
|
|
7115
7355
|
Object.entries(meta).forEach(([key, val]) => {
|
|
7356
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
7116
7357
|
keys.add(key);
|
|
7117
7358
|
const kLower = key.toLowerCase();
|
|
7118
7359
|
if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
|
|
@@ -7121,7 +7362,7 @@ var IntentClassifier = class {
|
|
|
7121
7362
|
if (typeof val === "number") {
|
|
7122
7363
|
numericKeys.add(key);
|
|
7123
7364
|
} else if (typeof val === "string") {
|
|
7124
|
-
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
|
|
7365
|
+
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
|
|
7125
7366
|
dateKeys.add(key);
|
|
7126
7367
|
} else if (val.length < 50 && !val.includes("\n")) {
|
|
7127
7368
|
catKeys.add(key);
|
|
@@ -7134,10 +7375,7 @@ var IntentClassifier = class {
|
|
|
7134
7375
|
hasCategoricalFields = catKeys.size > 0;
|
|
7135
7376
|
numericFieldCount = numericKeys.size;
|
|
7136
7377
|
categoricalFieldCount = catKeys.size;
|
|
7137
|
-
if (productKeyMatches >= 2
|
|
7138
|
-
var _a2, _b;
|
|
7139
|
-
return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
|
|
7140
|
-
})) {
|
|
7378
|
+
if (productKeyMatches >= 2) {
|
|
7141
7379
|
isProductLike = true;
|
|
7142
7380
|
}
|
|
7143
7381
|
}
|
|
@@ -7153,12 +7391,13 @@ var IntentClassifier = class {
|
|
|
7153
7391
|
}
|
|
7154
7392
|
// ─── Intent Heuristic Checkers ─────────────────────────────────────────────
|
|
7155
7393
|
static isInformationLookup(query) {
|
|
7156
|
-
|
|
7157
|
-
|
|
7394
|
+
const q = query.toLowerCase().trim();
|
|
7395
|
+
if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
|
|
7396
|
+
if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
|
|
7158
7397
|
return true;
|
|
7159
7398
|
}
|
|
7160
7399
|
}
|
|
7161
|
-
if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(
|
|
7400
|
+
if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
|
|
7162
7401
|
return true;
|
|
7163
7402
|
}
|
|
7164
7403
|
return false;
|
|
@@ -7173,13 +7412,13 @@ var IntentClassifier = class {
|
|
|
7173
7412
|
return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
|
|
7174
7413
|
}
|
|
7175
7414
|
static isTrendQuery(query) {
|
|
7176
|
-
return /\b(trend|trends|growth
|
|
7415
|
+
return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
|
|
7177
7416
|
}
|
|
7178
7417
|
static isRankingQuery(query) {
|
|
7179
7418
|
return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
|
|
7180
7419
|
}
|
|
7181
7420
|
static isAnalyticsQuery(query) {
|
|
7182
|
-
return /\b(
|
|
7421
|
+
return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
|
|
7183
7422
|
}
|
|
7184
7423
|
};
|
|
7185
7424
|
|
|
@@ -7191,8 +7430,8 @@ var Rule1SpecificInfoRule = class {
|
|
|
7191
7430
|
}
|
|
7192
7431
|
evaluate(context, intent) {
|
|
7193
7432
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7194
|
-
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
|
|
7195
|
-
const isNonVisual = !/\b(
|
|
7433
|
+
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
|
|
7434
|
+
const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
|
|
7196
7435
|
if (isFactQuery && isNonVisual) {
|
|
7197
7436
|
return {
|
|
7198
7437
|
renderType: "text",
|
|
@@ -7257,11 +7496,11 @@ var Rule4AnalyticalRule = class {
|
|
|
7257
7496
|
const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
|
|
7258
7497
|
if (!isAnalytical) return null;
|
|
7259
7498
|
let chartType = "bar";
|
|
7260
|
-
if (intent === "trend_analysis" || /\b(monthly|yearly|
|
|
7499
|
+
if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
|
|
7261
7500
|
chartType = "line";
|
|
7262
|
-
} else if (/\b(share|percentage|
|
|
7501
|
+
} else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
|
|
7263
7502
|
chartType = "pie";
|
|
7264
|
-
} else if (/\b(correlation|scatter|relationship
|
|
7503
|
+
} else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
|
|
7265
7504
|
chartType = "scatter";
|
|
7266
7505
|
} else {
|
|
7267
7506
|
chartType = "bar";
|
|
@@ -8032,12 +8271,20 @@ ${schemaProfileText}` : ""}`;
|
|
|
8032
8271
|
var _a3;
|
|
8033
8272
|
return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
|
|
8034
8273
|
}).filter(Boolean))) : this.detectCategories(data);
|
|
8035
|
-
if (categories.length
|
|
8274
|
+
if (categories.length <= 1) return null;
|
|
8036
8275
|
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
8276
|
+
const hasInventoryData = data.some((item) => {
|
|
8277
|
+
const meta = item.metadata || {};
|
|
8278
|
+
return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
|
|
8279
|
+
});
|
|
8037
8280
|
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
8281
|
+
if (!hasInventoryData) {
|
|
8282
|
+
return { label, value: count };
|
|
8283
|
+
}
|
|
8038
8284
|
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
8039
8285
|
return { label, value: count, inStockCount, outOfStockCount };
|
|
8040
8286
|
});
|
|
8287
|
+
if (pieData.length <= 1) return null;
|
|
8041
8288
|
return {
|
|
8042
8289
|
type: "pie_chart",
|
|
8043
8290
|
title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
|
|
@@ -8344,41 +8591,34 @@ ${schemaProfileText}` : ""}`;
|
|
|
8344
8591
|
}
|
|
8345
8592
|
static _productQueryTest(query) {
|
|
8346
8593
|
const q = query.toLowerCase();
|
|
8347
|
-
const
|
|
8594
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
|
|
8595
|
+
if (financialTerms.test(q)) {
|
|
8596
|
+
return false;
|
|
8597
|
+
}
|
|
8598
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
8348
8599
|
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
8349
8600
|
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
8350
8601
|
return productTerms && productAction && !nonProductEntityTerms;
|
|
8351
8602
|
}
|
|
8352
8603
|
static isProductData(item) {
|
|
8353
8604
|
if (!item) return false;
|
|
8354
|
-
const content = (item.content || "").toLowerCase();
|
|
8355
|
-
const productKeywords = [
|
|
8356
|
-
"product",
|
|
8357
|
-
"price",
|
|
8358
|
-
"stock",
|
|
8359
|
-
"sku",
|
|
8360
|
-
"brand",
|
|
8361
|
-
"msrp",
|
|
8362
|
-
"inventory",
|
|
8363
|
-
"buy",
|
|
8364
|
-
"shop",
|
|
8365
|
-
"beauty",
|
|
8366
|
-
"cosmetic",
|
|
8367
|
-
"facial",
|
|
8368
|
-
"cream",
|
|
8369
|
-
"serum",
|
|
8370
|
-
"mask",
|
|
8371
|
-
"makeup",
|
|
8372
|
-
"fragrance"
|
|
8373
|
-
];
|
|
8374
|
-
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
8375
8605
|
const metadata = item.metadata || {};
|
|
8376
|
-
const
|
|
8377
|
-
|
|
8378
|
-
return
|
|
8379
|
-
}
|
|
8380
|
-
|
|
8381
|
-
|
|
8606
|
+
const domain = String(metadata.domain || "").toLowerCase();
|
|
8607
|
+
if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
|
|
8608
|
+
return false;
|
|
8609
|
+
}
|
|
8610
|
+
if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
|
|
8611
|
+
return true;
|
|
8612
|
+
}
|
|
8613
|
+
const content = (item.content || "").toLowerCase();
|
|
8614
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
|
|
8615
|
+
if (financialTerms.test(content)) {
|
|
8616
|
+
return false;
|
|
8617
|
+
}
|
|
8618
|
+
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
|
|
8619
|
+
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
8620
|
+
const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
8621
|
+
return hasSku || hasProductIdentity && hasPrice;
|
|
8382
8622
|
}
|
|
8383
8623
|
static isTimeSeriesData(item) {
|
|
8384
8624
|
const metadata = item.metadata || {};
|
|
@@ -8397,10 +8637,31 @@ ${schemaProfileText}` : ""}`;
|
|
|
8397
8637
|
return fieldCount.size > 2;
|
|
8398
8638
|
}
|
|
8399
8639
|
static profileData(data) {
|
|
8640
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
8641
|
+
"filetype",
|
|
8642
|
+
"mimetype",
|
|
8643
|
+
"filename",
|
|
8644
|
+
"docid",
|
|
8645
|
+
"uploadedat",
|
|
8646
|
+
"dimension",
|
|
8647
|
+
"chunkindex",
|
|
8648
|
+
"totalchunks",
|
|
8649
|
+
"charactercount",
|
|
8650
|
+
"workspaceid",
|
|
8651
|
+
"pineconenamespace",
|
|
8652
|
+
"tier",
|
|
8653
|
+
"custommetadata",
|
|
8654
|
+
"csvheaders",
|
|
8655
|
+
"headers",
|
|
8656
|
+
"filesize",
|
|
8657
|
+
"content",
|
|
8658
|
+
"id"
|
|
8659
|
+
]);
|
|
8400
8660
|
const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
|
|
8401
8661
|
var _a2, _b;
|
|
8402
8662
|
const fields2 = {};
|
|
8403
8663
|
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
8664
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
8404
8665
|
const primitive = this.toPrimitive(value);
|
|
8405
8666
|
if (primitive !== null) fields2[key] = primitive;
|
|
8406
8667
|
});
|
|
@@ -9354,27 +9615,74 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9354
9615
|
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
9355
9616
|
*/
|
|
9356
9617
|
async ingest(documents, namespace) {
|
|
9618
|
+
var _a2;
|
|
9357
9619
|
await this.initialize();
|
|
9358
9620
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9359
9621
|
const results = [];
|
|
9622
|
+
if (!documents || documents.length === 0) return results;
|
|
9623
|
+
const docChunksMap = [];
|
|
9624
|
+
const allChunks = [];
|
|
9360
9625
|
for (const doc of documents) {
|
|
9361
9626
|
try {
|
|
9362
9627
|
const chunks = await this.prepareChunks(doc);
|
|
9363
|
-
|
|
9628
|
+
docChunksMap.push({ doc, chunks });
|
|
9629
|
+
allChunks.push(...chunks);
|
|
9630
|
+
} catch (error) {
|
|
9631
|
+
Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
|
|
9632
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9633
|
+
}
|
|
9634
|
+
}
|
|
9635
|
+
if (allChunks.length === 0) {
|
|
9636
|
+
return results;
|
|
9637
|
+
}
|
|
9638
|
+
try {
|
|
9639
|
+
const allVectors = await this.processEmbeddings(allChunks);
|
|
9640
|
+
let vectorOffset = 0;
|
|
9641
|
+
const allUpsertDocs = [];
|
|
9642
|
+
const docSuccessCount = /* @__PURE__ */ new Map();
|
|
9643
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9644
|
+
const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
|
|
9645
|
+
vectorOffset += chunks.length;
|
|
9364
9646
|
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9365
9647
|
id: chunk.id,
|
|
9366
|
-
vector:
|
|
9648
|
+
vector: docVectors[i],
|
|
9367
9649
|
content: chunk.content,
|
|
9368
9650
|
metadata: chunk.metadata
|
|
9369
9651
|
}));
|
|
9370
|
-
|
|
9371
|
-
|
|
9652
|
+
allUpsertDocs.push(...upsertDocs);
|
|
9653
|
+
docSuccessCount.set(doc.docId, chunks.length);
|
|
9654
|
+
}
|
|
9655
|
+
const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
|
|
9656
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9657
|
+
results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
|
|
9372
9658
|
if (this.graphDB && this.entityExtractor) {
|
|
9373
|
-
|
|
9659
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9660
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9661
|
+
});
|
|
9662
|
+
}
|
|
9663
|
+
}
|
|
9664
|
+
} catch (batchErr) {
|
|
9665
|
+
Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
|
|
9666
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9667
|
+
try {
|
|
9668
|
+
const vectors = await this.processEmbeddings(chunks);
|
|
9669
|
+
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9670
|
+
id: chunk.id,
|
|
9671
|
+
vector: vectors[i],
|
|
9672
|
+
content: chunk.content,
|
|
9673
|
+
metadata: chunk.metadata
|
|
9674
|
+
}));
|
|
9675
|
+
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
9676
|
+
results.push({ docId: doc.docId, chunksIngested: totalProcessed });
|
|
9677
|
+
if (this.graphDB && this.entityExtractor) {
|
|
9678
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9679
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9680
|
+
});
|
|
9681
|
+
}
|
|
9682
|
+
} catch (singleErr) {
|
|
9683
|
+
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
|
|
9684
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9374
9685
|
}
|
|
9375
|
-
} catch (error) {
|
|
9376
|
-
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
|
|
9377
|
-
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9378
9686
|
}
|
|
9379
9687
|
}
|
|
9380
9688
|
return results;
|
|
@@ -9394,19 +9702,40 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9394
9702
|
}
|
|
9395
9703
|
/**
|
|
9396
9704
|
* Step 2: Generate embeddings for chunks with retry logic.
|
|
9397
|
-
* Uses batchEmbed when available for efficiency; falls back to
|
|
9705
|
+
* Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
|
|
9398
9706
|
*/
|
|
9399
9707
|
async processEmbeddings(chunks) {
|
|
9400
|
-
|
|
9401
|
-
|
|
9402
|
-
|
|
9403
|
-
|
|
9404
|
-
|
|
9405
|
-
|
|
9406
|
-
|
|
9708
|
+
if (chunks.length === 0) return [];
|
|
9709
|
+
const texts = chunks.map((c) => c.content);
|
|
9710
|
+
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9711
|
+
try {
|
|
9712
|
+
const batchSize = 50;
|
|
9713
|
+
const allVectors = [];
|
|
9714
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
9715
|
+
const slice = texts.slice(i, i + batchSize);
|
|
9716
|
+
const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
|
|
9717
|
+
allVectors.push(...batchVectors);
|
|
9718
|
+
}
|
|
9719
|
+
if (allVectors.length === chunks.length) {
|
|
9720
|
+
return allVectors;
|
|
9721
|
+
}
|
|
9722
|
+
} catch (err) {
|
|
9723
|
+
Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
|
|
9724
|
+
error: err instanceof Error ? err.message : String(err)
|
|
9725
|
+
});
|
|
9726
|
+
}
|
|
9727
|
+
}
|
|
9728
|
+
const concurrentResult = await BatchProcessor.processConcurrent(
|
|
9729
|
+
texts,
|
|
9407
9730
|
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
9408
|
-
|
|
9731
|
+
10,
|
|
9732
|
+
{
|
|
9733
|
+
maxRetries: 3,
|
|
9734
|
+
initialDelayMs: 100,
|
|
9735
|
+
throwOnPartialFailure: false
|
|
9736
|
+
}
|
|
9409
9737
|
);
|
|
9738
|
+
const vectors = concurrentResult.results;
|
|
9410
9739
|
if (vectors.length !== chunks.length) {
|
|
9411
9740
|
throw new EmbeddingFailedException(
|
|
9412
9741
|
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
|
|
@@ -9433,25 +9762,24 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9433
9762
|
}
|
|
9434
9763
|
/** Step 4: Optional graph-based entity extraction and ingestion. */
|
|
9435
9764
|
async processGraphIngestion(docId, chunks) {
|
|
9436
|
-
const
|
|
9437
|
-
|
|
9438
|
-
maxRetries: 1,
|
|
9439
|
-
initialDelayMs: 500
|
|
9440
|
-
};
|
|
9441
|
-
await BatchProcessor.processBatch(
|
|
9765
|
+
const maxConcurrency = Math.min(3, chunks.length);
|
|
9766
|
+
await BatchProcessor.processConcurrent(
|
|
9442
9767
|
chunks,
|
|
9443
|
-
async (
|
|
9444
|
-
|
|
9445
|
-
|
|
9446
|
-
|
|
9447
|
-
|
|
9448
|
-
|
|
9449
|
-
}
|
|
9450
|
-
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9451
|
-
}
|
|
9768
|
+
async (chunk) => {
|
|
9769
|
+
try {
|
|
9770
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
9771
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
9772
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
9773
|
+
} catch (err) {
|
|
9774
|
+
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9452
9775
|
}
|
|
9453
9776
|
},
|
|
9454
|
-
|
|
9777
|
+
maxConcurrency,
|
|
9778
|
+
{
|
|
9779
|
+
maxRetries: 1,
|
|
9780
|
+
initialDelayMs: 200,
|
|
9781
|
+
throwOnPartialFailure: false
|
|
9782
|
+
}
|
|
9455
9783
|
);
|
|
9456
9784
|
}
|
|
9457
9785
|
async runNormalQuery(question, history = [], namespace) {
|
|
@@ -9576,7 +9904,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9576
9904
|
*/
|
|
9577
9905
|
askStreamInternal(_0) {
|
|
9578
9906
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
9579
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
|
|
9907
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
|
|
9580
9908
|
yield new __await(this.initialize());
|
|
9581
9909
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9582
9910
|
const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
|
|
@@ -9584,29 +9912,35 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9584
9912
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
9585
9913
|
const requestStart = performance.now();
|
|
9586
9914
|
try {
|
|
9587
|
-
let
|
|
9915
|
+
let effectiveQuestion = question;
|
|
9916
|
+
let promptOptResult;
|
|
9917
|
+
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
9918
|
+
promptOptResult = PromptOptimizer.optimize(question, history);
|
|
9919
|
+
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
9920
|
+
}
|
|
9921
|
+
let searchQuery = effectiveQuestion;
|
|
9588
9922
|
let rewrittenQuery;
|
|
9589
|
-
if ((
|
|
9590
|
-
searchQuery = yield new __await(this.rewriteQuery(
|
|
9591
|
-
rewrittenQuery = searchQuery !==
|
|
9923
|
+
if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
|
|
9924
|
+
searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
|
|
9925
|
+
rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
|
|
9592
9926
|
}
|
|
9593
|
-
const hints = QueryProcessor.extractQueryFieldHints(
|
|
9594
|
-
const filter = QueryProcessor.buildQueryFilter(
|
|
9595
|
-
const numericPredicates = QueryProcessor.extractNumericPredicates(
|
|
9927
|
+
const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
|
|
9928
|
+
const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
|
|
9929
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
|
|
9596
9930
|
if (numericPredicates.length > 0) {
|
|
9597
9931
|
filter.__numericPredicates = numericPredicates;
|
|
9598
9932
|
}
|
|
9599
9933
|
const embedStart = performance.now();
|
|
9600
9934
|
const cacheKey = `${ns}::${searchQuery}`;
|
|
9601
9935
|
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
9602
|
-
const arch = (
|
|
9603
|
-
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((
|
|
9936
|
+
const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
|
|
9937
|
+
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
|
|
9604
9938
|
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
9605
9939
|
QueryProcessor.determineRetrievalStrategy(
|
|
9606
9940
|
searchQuery,
|
|
9607
9941
|
useGraph ? this.llmRouter.get("fast") : void 0,
|
|
9608
|
-
(
|
|
9609
|
-
(
|
|
9942
|
+
(_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
|
|
9943
|
+
(_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
|
|
9610
9944
|
),
|
|
9611
9945
|
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
9612
9946
|
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
@@ -9616,7 +9950,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9616
9950
|
if (!cachedVector && queryVector.length > 0) {
|
|
9617
9951
|
this.embeddingCache.set(cacheKey, queryVector);
|
|
9618
9952
|
}
|
|
9619
|
-
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((
|
|
9953
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
9620
9954
|
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
9621
9955
|
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
9622
9956
|
const wantsExhaustiveList = true;
|
|
@@ -9633,7 +9967,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9633
9967
|
return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
|
|
9634
9968
|
});
|
|
9635
9969
|
const rerankLimit = Math.max(retrievalLimit, fullSources.length);
|
|
9636
|
-
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((
|
|
9970
|
+
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
|
|
9637
9971
|
if (!hasNumericPredicates && useReranking) {
|
|
9638
9972
|
fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
|
|
9639
9973
|
}
|
|
@@ -9661,7 +9995,7 @@ ${graphContext}
|
|
|
9661
9995
|
VECTOR CONTEXT:
|
|
9662
9996
|
${context}`;
|
|
9663
9997
|
}
|
|
9664
|
-
if ((
|
|
9998
|
+
if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
|
|
9665
9999
|
if (!this.coldContexts.has(ns)) {
|
|
9666
10000
|
yield new __await(this.loadColdContext(ns));
|
|
9667
10001
|
}
|
|
@@ -9693,31 +10027,31 @@ ${context}`;
|
|
|
9693
10027
|
SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
|
|
9694
10028
|
});
|
|
9695
10029
|
}
|
|
9696
|
-
const isProductQ = UITransformer.isProductQueryPublic(
|
|
10030
|
+
const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
|
|
9697
10031
|
const uiTransformationPromise = isProductQ ? Promise.resolve(
|
|
9698
10032
|
UITransformer.transform(
|
|
9699
|
-
|
|
10033
|
+
effectiveQuestion,
|
|
9700
10034
|
sources,
|
|
9701
10035
|
this.config,
|
|
9702
10036
|
cachedSchema,
|
|
9703
10037
|
{ visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
|
|
9704
10038
|
)
|
|
9705
10039
|
) : this.generateUiTransformation(
|
|
9706
|
-
|
|
10040
|
+
effectiveQuestion,
|
|
9707
10041
|
sources,
|
|
9708
10042
|
cachedSchema,
|
|
9709
10043
|
hasNumericPredicates
|
|
9710
10044
|
).catch((uiError) => {
|
|
9711
10045
|
Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
|
|
9712
|
-
return UITransformer.transform(
|
|
10046
|
+
return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
|
|
9713
10047
|
});
|
|
9714
10048
|
let hallucinationScoringPromise = Promise.resolve(void 0);
|
|
9715
10049
|
const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
9716
10050
|
const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
|
|
9717
|
-
const isNativeThinking = isClaude37 && (((
|
|
10051
|
+
const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
|
|
9718
10052
|
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
9719
10053
|
const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
|
|
9720
|
-
const isSimulatedThinking = !isNativeThinking && (((
|
|
10054
|
+
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
|
|
9721
10055
|
let finalRestrictionSuffix = restrictionSuffix;
|
|
9722
10056
|
if (isSimulatedThinking) {
|
|
9723
10057
|
finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
|
|
@@ -9726,9 +10060,9 @@ ${context}`;
|
|
|
9726
10060
|
role: m.role || "user",
|
|
9727
10061
|
content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
|
|
9728
10062
|
}));
|
|
9729
|
-
const userQuestion = { role: "user", content:
|
|
10063
|
+
const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
|
|
9730
10064
|
const messages = [...hardenedHistory, userQuestion];
|
|
9731
|
-
const systemPrompt = (
|
|
10065
|
+
const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
|
|
9732
10066
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
9733
10067
|
let fullReply = "";
|
|
9734
10068
|
let thinkingText = "";
|
|
@@ -9839,7 +10173,7 @@ ${context}`;
|
|
|
9839
10173
|
}
|
|
9840
10174
|
yield fullReply;
|
|
9841
10175
|
}
|
|
9842
|
-
const runHallucination = ((
|
|
10176
|
+
const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
|
|
9843
10177
|
if (runHallucination) {
|
|
9844
10178
|
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
|
|
9845
10179
|
}
|
|
@@ -9861,7 +10195,7 @@ ${context}`;
|
|
|
9861
10195
|
totalTokens: promptTokens + completionTokens,
|
|
9862
10196
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
9863
10197
|
};
|
|
9864
|
-
const awaitHallucination = ((
|
|
10198
|
+
const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
|
|
9865
10199
|
const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
|
|
9866
10200
|
uiTransformationPromise,
|
|
9867
10201
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
@@ -9869,9 +10203,9 @@ ${context}`;
|
|
|
9869
10203
|
const buildTrace = (hScore) => __spreadValues({
|
|
9870
10204
|
requestId,
|
|
9871
10205
|
query: question,
|
|
9872
|
-
rewrittenQuery,
|
|
10206
|
+
rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
|
|
9873
10207
|
systemPrompt,
|
|
9874
|
-
userPrompt:
|
|
10208
|
+
userPrompt: effectiveQuestion + finalRestrictionSuffix,
|
|
9875
10209
|
chunks: (sources || []).filter(Boolean).map((s) => {
|
|
9876
10210
|
var _a3, _b2;
|
|
9877
10211
|
return {
|
|
@@ -9890,10 +10224,10 @@ ${context}`;
|
|
|
9890
10224
|
hallucinationReason: hScore.reason
|
|
9891
10225
|
} : {});
|
|
9892
10226
|
const trace = buildTrace(hallucinationResult);
|
|
9893
|
-
const isTelemetryActive = (
|
|
10227
|
+
const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
|
|
9894
10228
|
if (isTelemetryActive) {
|
|
9895
10229
|
const defaultUrl = "https://www.retrivora.com/api/telemetry";
|
|
9896
|
-
const telemetryUrl = ((
|
|
10230
|
+
const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
9897
10231
|
const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
|
|
9898
10232
|
(async () => {
|
|
9899
10233
|
var _a3, _b2, _c2, _d2, _e2;
|
|
@@ -10757,13 +11091,81 @@ function createFromPreset(presetName) {
|
|
|
10757
11091
|
|
|
10758
11092
|
// src/utils/DocumentParser.ts
|
|
10759
11093
|
var DocumentParser = class {
|
|
11094
|
+
/**
|
|
11095
|
+
* Fast Magic-Bytes sniffer for instant file format identification (<1ms).
|
|
11096
|
+
*/
|
|
11097
|
+
static sniffMagicBytes(buffer) {
|
|
11098
|
+
if (!buffer || buffer.length < 4) return null;
|
|
11099
|
+
if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
|
|
11100
|
+
return "pdf";
|
|
11101
|
+
}
|
|
11102
|
+
if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
|
|
11103
|
+
return "zip";
|
|
11104
|
+
}
|
|
11105
|
+
if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
|
|
11106
|
+
return "ole2";
|
|
11107
|
+
}
|
|
11108
|
+
const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
|
|
11109
|
+
if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
|
|
11110
|
+
return "json";
|
|
11111
|
+
}
|
|
11112
|
+
return "text";
|
|
11113
|
+
}
|
|
11114
|
+
/**
|
|
11115
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
11116
|
+
* Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
|
|
11117
|
+
*/
|
|
11118
|
+
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
11119
|
+
const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
|
|
11120
|
+
if (!jsonRows || jsonRows.length === 0) {
|
|
11121
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
11122
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
11123
|
+
${csv.trim()}` : "";
|
|
11124
|
+
}
|
|
11125
|
+
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
11126
|
+
let currentHeader = [];
|
|
11127
|
+
for (let r = 0; r < jsonRows.length; r++) {
|
|
11128
|
+
const row = jsonRows[r];
|
|
11129
|
+
if (!row || row.length === 0) continue;
|
|
11130
|
+
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
11131
|
+
if (cells.length === 0) continue;
|
|
11132
|
+
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
11133
|
+
lines.push(`
|
|
11134
|
+
## ${cells.join(" ")}`);
|
|
11135
|
+
continue;
|
|
11136
|
+
}
|
|
11137
|
+
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
11138
|
+
if (isHeaderCandidate && cells.length >= 2) {
|
|
11139
|
+
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
11140
|
+
lines.push(`| ${cells.join(" | ")} |`);
|
|
11141
|
+
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
11142
|
+
continue;
|
|
11143
|
+
}
|
|
11144
|
+
if (currentHeader.length > 0 && cells.length > 1) {
|
|
11145
|
+
const fieldPairs = [];
|
|
11146
|
+
for (let c = 0; c < row.length; c++) {
|
|
11147
|
+
const h = currentHeader[c];
|
|
11148
|
+
const val = row[c];
|
|
11149
|
+
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
11150
|
+
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
11151
|
+
}
|
|
11152
|
+
}
|
|
11153
|
+
if (fieldPairs.length > 0) {
|
|
11154
|
+
lines.push(fieldPairs.join(" | "));
|
|
11155
|
+
continue;
|
|
11156
|
+
}
|
|
11157
|
+
}
|
|
11158
|
+
lines.push(cells.join(", "));
|
|
11159
|
+
}
|
|
11160
|
+
return lines.join("\n");
|
|
11161
|
+
}
|
|
10760
11162
|
/**
|
|
10761
11163
|
* Extract text from a File or Buffer based on its type.
|
|
10762
11164
|
*/
|
|
10763
|
-
static async parse(file, fileName, mimeType) {
|
|
11165
|
+
static async parse(file, fileName, mimeType = "") {
|
|
10764
11166
|
var _a2;
|
|
10765
11167
|
const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
10766
|
-
if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
11168
|
+
if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
10767
11169
|
return this.readAsText(file);
|
|
10768
11170
|
}
|
|
10769
11171
|
if (extension === "json" || mimeType === "application/json") {
|
|
@@ -10775,39 +11177,116 @@ var DocumentParser = class {
|
|
|
10775
11177
|
return text;
|
|
10776
11178
|
}
|
|
10777
11179
|
}
|
|
10778
|
-
if (extension === "
|
|
11180
|
+
if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
|
|
10779
11181
|
return this.readAsText(file);
|
|
10780
11182
|
}
|
|
10781
|
-
if (extension === "
|
|
11183
|
+
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
11184
|
+
return this.readAsText(file);
|
|
11185
|
+
}
|
|
11186
|
+
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
10782
11187
|
try {
|
|
10783
|
-
const
|
|
11188
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
10784
11189
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10785
|
-
const
|
|
10786
|
-
|
|
10787
|
-
|
|
10788
|
-
|
|
10789
|
-
const
|
|
10790
|
-
|
|
10791
|
-
|
|
10792
|
-
|
|
10793
|
-
|
|
11190
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
11191
|
+
const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
|
|
11192
|
+
var _a3, _b;
|
|
11193
|
+
const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
|
|
11194
|
+
const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
|
|
11195
|
+
return numA - numB;
|
|
11196
|
+
});
|
|
11197
|
+
if (slideFiles.length > 0) {
|
|
11198
|
+
const slidesText = [];
|
|
11199
|
+
for (let i = 0; i < slideFiles.length; i++) {
|
|
11200
|
+
const slideXml = await zip.files[slideFiles[i]].async("text");
|
|
11201
|
+
const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
|
|
11202
|
+
if (textMatches && textMatches.length > 0) {
|
|
11203
|
+
const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
|
|
11204
|
+
if (slideContent.trim()) {
|
|
11205
|
+
slidesText.push(`--- Slide ${i + 1} ---
|
|
11206
|
+
${slideContent.trim()}`);
|
|
11207
|
+
}
|
|
10794
11208
|
}
|
|
10795
|
-
});
|
|
10796
|
-
const csv = csvLines.join("\n");
|
|
10797
|
-
if (csv && csv.trim()) {
|
|
10798
|
-
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
10799
|
-
${csv.trim()}`);
|
|
10800
11209
|
}
|
|
10801
|
-
|
|
10802
|
-
|
|
10803
|
-
|
|
11210
|
+
if (slidesText.length > 0) {
|
|
11211
|
+
return slidesText.join("\n\n");
|
|
11212
|
+
}
|
|
11213
|
+
}
|
|
11214
|
+
if (zip.files["content.xml"]) {
|
|
11215
|
+
const contentXml = await zip.files["content.xml"].async("text");
|
|
11216
|
+
const cleanText = this.stripXmlTags(contentXml);
|
|
11217
|
+
if (cleanText.trim()) {
|
|
11218
|
+
return cleanText.trim();
|
|
11219
|
+
}
|
|
11220
|
+
}
|
|
11221
|
+
const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
|
|
11222
|
+
const extractedSegments = [];
|
|
11223
|
+
for (const xf of allXmlFiles) {
|
|
11224
|
+
const xml = await zip.files[xf].async("text");
|
|
11225
|
+
const stripped = this.stripXmlTags(xml);
|
|
11226
|
+
if (stripped.length > 10) extractedSegments.push(stripped);
|
|
11227
|
+
}
|
|
11228
|
+
if (extractedSegments.length > 0) {
|
|
11229
|
+
return extractedSegments.join("\n\n");
|
|
11230
|
+
}
|
|
11231
|
+
return `[PowerPoint Presentation: ${fileName}]`;
|
|
11232
|
+
} catch (err) {
|
|
11233
|
+
console.warn("[DocumentParser] PowerPoint parsing failed:", err);
|
|
10804
11234
|
try {
|
|
10805
11235
|
return await this.readAsText(file);
|
|
10806
|
-
} catch (
|
|
10807
|
-
return `[
|
|
11236
|
+
} catch (e) {
|
|
11237
|
+
return `[PowerPoint Presentation File: ${fileName}]`;
|
|
10808
11238
|
}
|
|
10809
11239
|
}
|
|
10810
11240
|
}
|
|
11241
|
+
if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
|
|
11242
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11243
|
+
try {
|
|
11244
|
+
const XLSX = await import("xlsx");
|
|
11245
|
+
const workbook = XLSX.read(buffer, { type: "buffer" });
|
|
11246
|
+
const sheetsText = [];
|
|
11247
|
+
for (const sheetName of workbook.SheetNames) {
|
|
11248
|
+
const sheet = workbook.Sheets[sheetName];
|
|
11249
|
+
const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
|
|
11250
|
+
if (formatted && formatted.trim()) {
|
|
11251
|
+
sheetsText.push(formatted.trim());
|
|
11252
|
+
}
|
|
11253
|
+
}
|
|
11254
|
+
if (sheetsText.length > 0) {
|
|
11255
|
+
return sheetsText.join("\n\n");
|
|
11256
|
+
}
|
|
11257
|
+
} catch (xlsxErr) {
|
|
11258
|
+
console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
|
|
11259
|
+
}
|
|
11260
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
11261
|
+
if (isZip) {
|
|
11262
|
+
try {
|
|
11263
|
+
const ExcelJS = await import("exceljs");
|
|
11264
|
+
const workbook = new ExcelJS.Workbook();
|
|
11265
|
+
await workbook.xlsx.load(buffer);
|
|
11266
|
+
const sheetsText = [];
|
|
11267
|
+
workbook.eachSheet((sheet) => {
|
|
11268
|
+
const csvLines = [];
|
|
11269
|
+
sheet.eachRow((row) => {
|
|
11270
|
+
const values = row.values;
|
|
11271
|
+
if (Array.isArray(values)) {
|
|
11272
|
+
csvLines.push(values.slice(1).join(","));
|
|
11273
|
+
}
|
|
11274
|
+
});
|
|
11275
|
+
const csv = csvLines.join("\n");
|
|
11276
|
+
if (csv && csv.trim()) {
|
|
11277
|
+
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
11278
|
+
${csv.trim()}`);
|
|
11279
|
+
}
|
|
11280
|
+
});
|
|
11281
|
+
if (sheetsText.length > 0) {
|
|
11282
|
+
return sheetsText.join("\n\n");
|
|
11283
|
+
}
|
|
11284
|
+
} catch (err) {
|
|
11285
|
+
console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
|
|
11286
|
+
}
|
|
11287
|
+
}
|
|
11288
|
+
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
11289
|
+
}
|
|
10811
11290
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
10812
11291
|
try {
|
|
10813
11292
|
const pdfModule = await import("pdf-parse");
|
|
@@ -10820,15 +11299,48 @@ ${csv.trim()}`);
|
|
|
10820
11299
|
return `[PDF Parsing Error for ${fileName}]`;
|
|
10821
11300
|
}
|
|
10822
11301
|
}
|
|
10823
|
-
if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
|
|
11302
|
+
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
10824
11303
|
try {
|
|
10825
11304
|
const mammoth = await import("mammoth");
|
|
10826
11305
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10827
11306
|
const result = await mammoth.extractRawText({ buffer });
|
|
10828
|
-
|
|
11307
|
+
if (result.value && result.value.trim()) {
|
|
11308
|
+
return result.value;
|
|
11309
|
+
}
|
|
11310
|
+
} catch (err) {
|
|
11311
|
+
console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
|
|
11312
|
+
}
|
|
11313
|
+
try {
|
|
11314
|
+
const raw = await this.readAsText(file);
|
|
11315
|
+
if (extension === "rtf" || raw.startsWith("{\\rtf")) {
|
|
11316
|
+
return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
|
|
11317
|
+
}
|
|
11318
|
+
return raw;
|
|
11319
|
+
} catch (e) {
|
|
11320
|
+
return `[Word Document File: ${fileName}]`;
|
|
11321
|
+
}
|
|
11322
|
+
}
|
|
11323
|
+
if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
|
|
11324
|
+
const raw = await this.readAsText(file);
|
|
11325
|
+
return this.stripHtmlAndXml(raw);
|
|
11326
|
+
}
|
|
11327
|
+
if (extension === "epub" || mimeType.includes("epub")) {
|
|
11328
|
+
try {
|
|
11329
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
11330
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11331
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
11332
|
+
const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
|
|
11333
|
+
const chapters = [];
|
|
11334
|
+
for (const hf of htmlFiles) {
|
|
11335
|
+
const content = await zip.files[hf].async("text");
|
|
11336
|
+
const clean = this.stripHtmlAndXml(content);
|
|
11337
|
+
if (clean.length > 50) chapters.push(clean);
|
|
11338
|
+
}
|
|
11339
|
+
if (chapters.length > 0) {
|
|
11340
|
+
return chapters.join("\n\n---\n\n");
|
|
11341
|
+
}
|
|
10829
11342
|
} catch (err) {
|
|
10830
|
-
console.warn(
|
|
10831
|
-
return `[Word Parsing Error for ${fileName}]`;
|
|
11343
|
+
console.warn("[DocumentParser] EPUB parsing failed:", err);
|
|
10832
11344
|
}
|
|
10833
11345
|
}
|
|
10834
11346
|
try {
|
|
@@ -10837,6 +11349,237 @@ ${csv.trim()}`);
|
|
|
10837
11349
|
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
10838
11350
|
}
|
|
10839
11351
|
}
|
|
11352
|
+
/**
|
|
11353
|
+
* Helper to strip XML tags and normalize inner text content.
|
|
11354
|
+
*/
|
|
11355
|
+
static stripXmlTags(xml) {
|
|
11356
|
+
return xml.replace(/<[^>]+>/g, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/\s+/g, " ").trim();
|
|
11357
|
+
}
|
|
11358
|
+
/**
|
|
11359
|
+
* Helper to strip HTML / XML tags while removing script and style blocks completely.
|
|
11360
|
+
*/
|
|
11361
|
+
static stripHtmlAndXml(html) {
|
|
11362
|
+
return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/ /gi, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
|
|
11363
|
+
}
|
|
11364
|
+
/**
|
|
11365
|
+
* Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
|
|
11366
|
+
*/
|
|
11367
|
+
static parseLegacyExcelOrFallback(buffer, fileName) {
|
|
11368
|
+
const rawStr = buffer.toString("utf-8");
|
|
11369
|
+
if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
|
|
11370
|
+
const rows = [];
|
|
11371
|
+
const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
|
|
11372
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11373
|
+
for (const rowHtml of rowMatches) {
|
|
11374
|
+
const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
|
|
11375
|
+
if (cells && cells.length > 0) {
|
|
11376
|
+
const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
|
|
11377
|
+
if (rowText) rows.push(rowText);
|
|
11378
|
+
}
|
|
11379
|
+
}
|
|
11380
|
+
}
|
|
11381
|
+
if (rows.length > 0) {
|
|
11382
|
+
return rows.join("\n");
|
|
11383
|
+
}
|
|
11384
|
+
const clean = this.stripHtmlAndXml(rawStr);
|
|
11385
|
+
if (clean.length > 10) return clean;
|
|
11386
|
+
}
|
|
11387
|
+
if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
|
|
11388
|
+
const rows = [];
|
|
11389
|
+
const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
|
|
11390
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11391
|
+
for (const rowXml of rowMatches) {
|
|
11392
|
+
const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
|
|
11393
|
+
if (dataMatches && dataMatches.length > 0) {
|
|
11394
|
+
const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
|
|
11395
|
+
if (rowText) rows.push(rowText);
|
|
11396
|
+
}
|
|
11397
|
+
}
|
|
11398
|
+
}
|
|
11399
|
+
if (rows.length > 0) {
|
|
11400
|
+
return rows.join("\n");
|
|
11401
|
+
}
|
|
11402
|
+
}
|
|
11403
|
+
const extractedStrings = [];
|
|
11404
|
+
const ignoredOleMarkers = /* @__PURE__ */ new Set([
|
|
11405
|
+
"root entry",
|
|
11406
|
+
"workbook",
|
|
11407
|
+
"compobj",
|
|
11408
|
+
"summaryinformation",
|
|
11409
|
+
"documentsummaryinformation",
|
|
11410
|
+
"microsoft excel",
|
|
11411
|
+
"worksheet",
|
|
11412
|
+
"biff8",
|
|
11413
|
+
"calcpr"
|
|
11414
|
+
]);
|
|
11415
|
+
let currentUtf16 = "";
|
|
11416
|
+
for (let i = 0; i < buffer.length - 1; i += 2) {
|
|
11417
|
+
const charCode = buffer[i];
|
|
11418
|
+
const zeroByte = buffer[i + 1];
|
|
11419
|
+
if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
|
|
11420
|
+
currentUtf16 += String.fromCharCode(charCode);
|
|
11421
|
+
} else {
|
|
11422
|
+
if (currentUtf16.length >= 2) {
|
|
11423
|
+
const trimmed = currentUtf16.trim();
|
|
11424
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11425
|
+
extractedStrings.push(trimmed);
|
|
11426
|
+
}
|
|
11427
|
+
}
|
|
11428
|
+
currentUtf16 = "";
|
|
11429
|
+
}
|
|
11430
|
+
}
|
|
11431
|
+
let currentAscii = "";
|
|
11432
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
11433
|
+
const byte = buffer[i];
|
|
11434
|
+
if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
|
|
11435
|
+
currentAscii += String.fromCharCode(byte);
|
|
11436
|
+
} else {
|
|
11437
|
+
if (currentAscii.length >= 2) {
|
|
11438
|
+
const trimmed = currentAscii.trim();
|
|
11439
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11440
|
+
extractedStrings.push(trimmed);
|
|
11441
|
+
}
|
|
11442
|
+
}
|
|
11443
|
+
currentAscii = "";
|
|
11444
|
+
}
|
|
11445
|
+
}
|
|
11446
|
+
const uniqueStrings = [];
|
|
11447
|
+
for (const str of extractedStrings) {
|
|
11448
|
+
if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
|
|
11449
|
+
uniqueStrings.push(str);
|
|
11450
|
+
}
|
|
11451
|
+
}
|
|
11452
|
+
if (uniqueStrings.length > 0) {
|
|
11453
|
+
return uniqueStrings.join(" | ");
|
|
11454
|
+
}
|
|
11455
|
+
const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
|
|
11456
|
+
return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
|
|
11457
|
+
}
|
|
11458
|
+
/**
|
|
11459
|
+
* Dynamically infer document domain and schema from extracted text, filename, and format.
|
|
11460
|
+
*/
|
|
11461
|
+
static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
|
|
11462
|
+
const lowerText = text.toLowerCase();
|
|
11463
|
+
const lowerName = fileName.toLowerCase();
|
|
11464
|
+
const financialTerms = [
|
|
11465
|
+
"price per share",
|
|
11466
|
+
"share price",
|
|
11467
|
+
"total cost of the deal",
|
|
11468
|
+
"cost of the deal",
|
|
11469
|
+
"lbo",
|
|
11470
|
+
"merger",
|
|
11471
|
+
"valuation",
|
|
11472
|
+
"d/e ratio",
|
|
11473
|
+
"ebit",
|
|
11474
|
+
"ebitda",
|
|
11475
|
+
"fcfe",
|
|
11476
|
+
"fcff",
|
|
11477
|
+
"wacc",
|
|
11478
|
+
"beta",
|
|
11479
|
+
"working capital",
|
|
11480
|
+
"cogs",
|
|
11481
|
+
"cap spending",
|
|
11482
|
+
"depreciation",
|
|
11483
|
+
"cashflow",
|
|
11484
|
+
"cash flow",
|
|
11485
|
+
"net income",
|
|
11486
|
+
"balance sheet",
|
|
11487
|
+
"income statement",
|
|
11488
|
+
"interest rate",
|
|
11489
|
+
"terminal year",
|
|
11490
|
+
"equity",
|
|
11491
|
+
"debt outstanding",
|
|
11492
|
+
"gross margin",
|
|
11493
|
+
"fiscal year",
|
|
11494
|
+
"fy20",
|
|
11495
|
+
"fy21",
|
|
11496
|
+
"fy22",
|
|
11497
|
+
"fy23",
|
|
11498
|
+
"fy24",
|
|
11499
|
+
"fy25",
|
|
11500
|
+
"q1",
|
|
11501
|
+
"q2",
|
|
11502
|
+
"q3",
|
|
11503
|
+
"q4"
|
|
11504
|
+
];
|
|
11505
|
+
let financialScore = 0;
|
|
11506
|
+
for (const term of financialTerms) {
|
|
11507
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11508
|
+
financialScore += 1;
|
|
11509
|
+
}
|
|
11510
|
+
}
|
|
11511
|
+
const ecommerceTerms = [
|
|
11512
|
+
"sku",
|
|
11513
|
+
"product_id",
|
|
11514
|
+
"product id",
|
|
11515
|
+
"item_id",
|
|
11516
|
+
"add to cart",
|
|
11517
|
+
"buy now",
|
|
11518
|
+
"in stock",
|
|
11519
|
+
"out of stock",
|
|
11520
|
+
"inventory count",
|
|
11521
|
+
"retail price",
|
|
11522
|
+
"unit price",
|
|
11523
|
+
"msrp",
|
|
11524
|
+
"cart",
|
|
11525
|
+
"shop",
|
|
11526
|
+
"catalog",
|
|
11527
|
+
"product_name",
|
|
11528
|
+
"product name",
|
|
11529
|
+
"brand name"
|
|
11530
|
+
];
|
|
11531
|
+
let ecommerceScore = 0;
|
|
11532
|
+
for (const term of ecommerceTerms) {
|
|
11533
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11534
|
+
ecommerceScore += 1;
|
|
11535
|
+
}
|
|
11536
|
+
}
|
|
11537
|
+
const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
|
|
11538
|
+
let legalScore = 0;
|
|
11539
|
+
for (const term of legalTerms) {
|
|
11540
|
+
if (lowerText.includes(term)) legalScore += 1;
|
|
11541
|
+
}
|
|
11542
|
+
const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
|
|
11543
|
+
let healthScore = 0;
|
|
11544
|
+
for (const term of healthTerms) {
|
|
11545
|
+
if (lowerText.includes(term)) healthScore += 1;
|
|
11546
|
+
}
|
|
11547
|
+
let schema;
|
|
11548
|
+
const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
11549
|
+
if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
|
|
11550
|
+
const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
|
|
11551
|
+
const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
|
|
11552
|
+
if (headers.length > 1 && lines.length > 1) {
|
|
11553
|
+
const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
|
|
11554
|
+
schema = {};
|
|
11555
|
+
headers.forEach((h, idx) => {
|
|
11556
|
+
const val = sampleRow[idx] || "";
|
|
11557
|
+
if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
|
|
11558
|
+
schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
|
|
11559
|
+
} else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
|
|
11560
|
+
schema[h] = "date";
|
|
11561
|
+
} else {
|
|
11562
|
+
schema[h] = "string";
|
|
11563
|
+
}
|
|
11564
|
+
});
|
|
11565
|
+
}
|
|
11566
|
+
}
|
|
11567
|
+
const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
|
|
11568
|
+
const isCatalog = ecommerceScore >= 2 && !isFinancial;
|
|
11569
|
+
let domain = "general";
|
|
11570
|
+
if (isFinancial) {
|
|
11571
|
+
domain = "finance";
|
|
11572
|
+
} else if (isCatalog) {
|
|
11573
|
+
domain = "ecommerce";
|
|
11574
|
+
} else if (legalScore >= 2) {
|
|
11575
|
+
domain = "legal";
|
|
11576
|
+
} else if (healthScore >= 2) {
|
|
11577
|
+
domain = "healthcare";
|
|
11578
|
+
} else if (schema && Object.keys(schema).length > 0) {
|
|
11579
|
+
domain = "tabular";
|
|
11580
|
+
}
|
|
11581
|
+
return { domain, schema, isCatalog, isFinancial };
|
|
11582
|
+
}
|
|
10840
11583
|
static async readAsText(file) {
|
|
10841
11584
|
if (Buffer.isBuffer(file)) {
|
|
10842
11585
|
return file.toString("utf-8");
|
|
@@ -11499,18 +12242,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
|
|
|
11499
12242
|
}
|
|
11500
12243
|
}
|
|
11501
12244
|
function getAllowedNamespaces(projectId) {
|
|
11502
|
-
const trimmed = projectId.trim();
|
|
12245
|
+
const trimmed = (projectId || "default").trim();
|
|
11503
12246
|
const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
|
|
11504
12247
|
return new Set([
|
|
11505
12248
|
trimmed,
|
|
11506
12249
|
withoutPrefix,
|
|
11507
|
-
`retrivora-${withoutPrefix}
|
|
12250
|
+
`retrivora-${withoutPrefix}`,
|
|
12251
|
+
"default",
|
|
12252
|
+
"retrivora-default"
|
|
11508
12253
|
].filter(Boolean));
|
|
11509
12254
|
}
|
|
11510
12255
|
function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
12256
|
+
var _a2;
|
|
11511
12257
|
const allowed = getAllowedNamespaces(payload.projectId);
|
|
11512
12258
|
const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
|
|
11513
|
-
|
|
12259
|
+
const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
|
|
12260
|
+
if (requested && !isDevOrFree && !allowed.has(requested)) {
|
|
11514
12261
|
return {
|
|
11515
12262
|
ok: false,
|
|
11516
12263
|
response: new Response(JSON.stringify({
|
|
@@ -11524,7 +12271,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
|
11524
12271
|
})
|
|
11525
12272
|
};
|
|
11526
12273
|
}
|
|
11527
|
-
const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
|
|
12274
|
+
const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
|
|
11528
12275
|
return { ok: true, namespace: requested || defaultNamespace };
|
|
11529
12276
|
}
|
|
11530
12277
|
var MAX_MESSAGE_LENGTH = 8e3;
|
|
@@ -11968,7 +12715,7 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
11968
12715
|
function createHealthHandler(configOrPlugin, options) {
|
|
11969
12716
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
11970
12717
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
11971
|
-
return async function GET(req) {
|
|
12718
|
+
return async function GET(req, context) {
|
|
11972
12719
|
if (req) {
|
|
11973
12720
|
const authResult = await checkAuth(req, onAuthorize);
|
|
11974
12721
|
if (authResult) return authResult;
|
|
@@ -12079,9 +12826,10 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12079
12826
|
}
|
|
12080
12827
|
const documents = [];
|
|
12081
12828
|
for (const file of files) {
|
|
12829
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
12082
12830
|
const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
|
|
12083
12831
|
if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
|
|
12084
|
-
const text =
|
|
12832
|
+
const text = buffer.toString("utf-8");
|
|
12085
12833
|
const Papa = await import("papaparse");
|
|
12086
12834
|
const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
|
|
12087
12835
|
if (parsed.data && parsed.data.length > 0) {
|
|
@@ -12121,49 +12869,70 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
12121
12869
|
}
|
|
12122
12870
|
}
|
|
12123
12871
|
} else if (isExcel) {
|
|
12124
|
-
|
|
12125
|
-
|
|
12126
|
-
|
|
12127
|
-
|
|
12128
|
-
|
|
12129
|
-
|
|
12130
|
-
|
|
12131
|
-
|
|
12132
|
-
|
|
12133
|
-
|
|
12134
|
-
|
|
12135
|
-
|
|
12136
|
-
|
|
12137
|
-
|
|
12138
|
-
|
|
12139
|
-
|
|
12140
|
-
|
|
12141
|
-
|
|
12142
|
-
|
|
12143
|
-
|
|
12144
|
-
|
|
12145
|
-
|
|
12146
|
-
|
|
12147
|
-
|
|
12148
|
-
|
|
12149
|
-
|
|
12150
|
-
|
|
12151
|
-
|
|
12152
|
-
|
|
12153
|
-
|
|
12154
|
-
|
|
12155
|
-
|
|
12156
|
-
|
|
12157
|
-
|
|
12158
|
-
|
|
12159
|
-
|
|
12160
|
-
|
|
12872
|
+
let excelLoaded = false;
|
|
12873
|
+
try {
|
|
12874
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
12875
|
+
if (isZip) {
|
|
12876
|
+
const ExcelJS = await import("exceljs");
|
|
12877
|
+
const workbook = new ExcelJS.Workbook();
|
|
12878
|
+
await workbook.xlsx.load(buffer);
|
|
12879
|
+
workbook.eachSheet((sheet, sheetId) => {
|
|
12880
|
+
const sheetName = sheet.name;
|
|
12881
|
+
let headers = [];
|
|
12882
|
+
let i = 0;
|
|
12883
|
+
sheet.eachRow((row, rowNumber) => {
|
|
12884
|
+
const values = Array.isArray(row.values) ? row.values.slice(1) : [];
|
|
12885
|
+
if (rowNumber === 1) {
|
|
12886
|
+
headers = values.map((v) => String(v));
|
|
12887
|
+
return;
|
|
12888
|
+
}
|
|
12889
|
+
i++;
|
|
12890
|
+
const contentParts = [];
|
|
12891
|
+
const rowData = {};
|
|
12892
|
+
values.forEach((val, index) => {
|
|
12893
|
+
const key = headers[index] || `Column${index + 1}`;
|
|
12894
|
+
rowData[key] = val;
|
|
12895
|
+
if (key && val !== void 0 && val !== null && String(val).trim()) {
|
|
12896
|
+
contentParts.push(`${key}: ${val}`);
|
|
12897
|
+
}
|
|
12898
|
+
});
|
|
12899
|
+
if (contentParts.length > 0) {
|
|
12900
|
+
documents.push({
|
|
12901
|
+
docId: `${file.name}-${sheetName}-row-${i}`,
|
|
12902
|
+
content: contentParts.join(", "),
|
|
12903
|
+
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
12904
|
+
fileName: file.name,
|
|
12905
|
+
sheetName,
|
|
12906
|
+
fileSize: file.size,
|
|
12907
|
+
fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
12908
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12909
|
+
}, dimension ? { dimension } : {}), {
|
|
12910
|
+
headers
|
|
12911
|
+
}), rowData)
|
|
12912
|
+
});
|
|
12913
|
+
}
|
|
12161
12914
|
});
|
|
12162
|
-
}
|
|
12915
|
+
});
|
|
12916
|
+
excelLoaded = documents.length > 0;
|
|
12917
|
+
}
|
|
12918
|
+
} catch (excelErr) {
|
|
12919
|
+
excelLoaded = false;
|
|
12920
|
+
}
|
|
12921
|
+
if (!excelLoaded) {
|
|
12922
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
12923
|
+
documents.push({
|
|
12924
|
+
docId: file.name,
|
|
12925
|
+
content,
|
|
12926
|
+
metadata: __spreadValues({
|
|
12927
|
+
fileName: file.name,
|
|
12928
|
+
fileSize: file.size,
|
|
12929
|
+
fileType: file.type || "application/vnd.ms-excel",
|
|
12930
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12931
|
+
}, dimension ? { dimension } : {})
|
|
12163
12932
|
});
|
|
12164
|
-
}
|
|
12933
|
+
}
|
|
12165
12934
|
} else {
|
|
12166
|
-
const content = await DocumentParser.parse(
|
|
12935
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
12167
12936
|
documents.push({
|
|
12168
12937
|
docId: file.name,
|
|
12169
12938
|
content,
|
|
@@ -12322,7 +13091,7 @@ function createSessionsHandler(configOrPlugin, options) {
|
|
|
12322
13091
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
12323
13092
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
12324
13093
|
const onResolveScope = options == null ? void 0 : options.onResolveScope;
|
|
12325
|
-
return async function GET(req) {
|
|
13094
|
+
return async function GET(req, context) {
|
|
12326
13095
|
if (req) {
|
|
12327
13096
|
const authResult = await checkAuth(req, onAuthorize);
|
|
12328
13097
|
if (authResult) return authResult;
|
|
@@ -13253,6 +14022,7 @@ export {
|
|
|
13253
14022
|
PineconeProvider,
|
|
13254
14023
|
Pipeline,
|
|
13255
14024
|
PostgreSQLProvider,
|
|
14025
|
+
PromptOptimizer,
|
|
13256
14026
|
ProviderHealthCheck,
|
|
13257
14027
|
ProviderNotFoundException,
|
|
13258
14028
|
ProviderRegistry,
|