@retrivora-ai/rag-engine 2.5.6 → 2.5.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{ILLMProvider-DwFj0h2u.d.mts → ILLMProvider-B2N3lMBc.d.mts} +10 -1
- package/dist/{ILLMProvider-DwFj0h2u.d.ts → ILLMProvider-B2N3lMBc.d.ts} +10 -1
- package/dist/handlers/index.d.mts +2 -2
- package/dist/handlers/index.d.ts +2 -2
- package/dist/handlers/index.js +961 -192
- package/dist/handlers/index.mjs +961 -192
- package/dist/{index-Cb0GsT16.d.ts → index-37V810a-.d.mts} +3 -3
- package/dist/{index-D1jN8KSF.d.mts → index-BxzDv2jd.d.ts} +3 -3
- package/dist/{index-Byv5MaYi.d.mts → index-CAGGPyI7.d.mts} +22 -2
- package/dist/{index-DP3_OpuB.d.ts → index-DdQxoIpu.d.ts} +22 -2
- package/dist/index.css +188 -4
- package/dist/index.d.mts +178 -20
- package/dist/index.d.ts +178 -20
- package/dist/index.js +735 -195
- package/dist/index.min.css +1 -1
- package/dist/index.mjs +731 -193
- package/dist/server.d.mts +100 -9
- package/dist/server.d.ts +100 -9
- package/dist/server.js +963 -192
- package/dist/server.mjs +962 -192
- package/package.json +6 -3
package/dist/handlers/index.mjs
CHANGED
|
@@ -88,8 +88,8 @@ function buildPayload(template, vars) {
|
|
|
88
88
|
let raw = template;
|
|
89
89
|
for (const [key, val] of Object.entries(vars)) {
|
|
90
90
|
const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
|
|
91
|
-
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
|
|
92
|
-
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
|
|
91
|
+
raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
|
|
92
|
+
raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
|
|
93
93
|
}
|
|
94
94
|
try {
|
|
95
95
|
return JSON.parse(raw);
|
|
@@ -312,7 +312,7 @@ var init_package = __esm({
|
|
|
312
312
|
"package.json"() {
|
|
313
313
|
package_default = {
|
|
314
314
|
name: "@retrivora-ai/rag-engine",
|
|
315
|
-
version: "2.5.
|
|
315
|
+
version: "2.5.8",
|
|
316
316
|
description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
|
|
317
317
|
author: "Abhinav Alkuchi",
|
|
318
318
|
license: "UNLICENSED",
|
|
@@ -415,9 +415,11 @@ var init_package = __esm({
|
|
|
415
415
|
"@google/genai": "^0.8.0",
|
|
416
416
|
"@google/generative-ai": "^0.24.1",
|
|
417
417
|
"@pinecone-database/pinecone": "^7.2.0",
|
|
418
|
-
"@retrivora-ai/billing-guards": "
|
|
418
|
+
"@retrivora-ai/billing-guards": "file:../billing-guards",
|
|
419
419
|
"@types/papaparse": "^5.5.2",
|
|
420
|
+
exceljs: "^4.4.0",
|
|
420
421
|
"framer-motion": "^13.1.0",
|
|
422
|
+
jszip: "^3.10.1",
|
|
421
423
|
katex: "^0.18.4",
|
|
422
424
|
"lucide-react": "^1.8.0",
|
|
423
425
|
"next-themes": "^0.4.6",
|
|
@@ -430,7 +432,7 @@ var init_package = __esm({
|
|
|
430
432
|
"rehype-katex": "^7.0.1",
|
|
431
433
|
"remark-gfm": "^4.0.1",
|
|
432
434
|
"remark-math": "^6.0.0",
|
|
433
|
-
|
|
435
|
+
xlsx: "^0.18.5",
|
|
434
436
|
zod: "^4.4.3"
|
|
435
437
|
},
|
|
436
438
|
optionalDependencies: {
|
|
@@ -448,6 +450,7 @@ var init_package = __esm({
|
|
|
448
450
|
"@tailwindcss/postcss": "^4",
|
|
449
451
|
"@types/estree": "^1.0.9",
|
|
450
452
|
"@types/jest": "^30.0.0",
|
|
453
|
+
"@types/jszip": "^3.4.1",
|
|
451
454
|
"@types/katex": "^0.16.8",
|
|
452
455
|
"@types/node": "^20",
|
|
453
456
|
"@types/pdf-parse": "^1.1.5",
|
|
@@ -2843,6 +2846,7 @@ var _LicenseVerifier = class _LicenseVerifier {
|
|
|
2843
2846
|
try {
|
|
2844
2847
|
response = await fetch(endpoint, {
|
|
2845
2848
|
method: "POST",
|
|
2849
|
+
signal: AbortSignal.timeout(3e3),
|
|
2846
2850
|
headers: {
|
|
2847
2851
|
"Content-Type": "application/json",
|
|
2848
2852
|
"x-license-key": licenseKey
|
|
@@ -5520,15 +5524,29 @@ Text to extract from:
|
|
|
5520
5524
|
const response = await this.llm.chat([
|
|
5521
5525
|
{ role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
|
|
5522
5526
|
{ role: "user", content: prompt }
|
|
5523
|
-
], "", { maxTokens:
|
|
5527
|
+
], "", { maxTokens: 800, temperature: 0.1 });
|
|
5524
5528
|
try {
|
|
5525
|
-
const
|
|
5526
|
-
const
|
|
5527
|
-
|
|
5529
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5530
|
+
const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
|
|
5531
|
+
const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
|
|
5532
|
+
const jsonMatch = candidate.match(/\{[\s\S]*\}/);
|
|
5533
|
+
const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
|
|
5534
|
+
const parsed = JSON.parse(cleanJson);
|
|
5535
|
+
return {
|
|
5536
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5537
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5538
|
+
};
|
|
5528
5539
|
} catch (e) {
|
|
5529
5540
|
try {
|
|
5530
|
-
const
|
|
5531
|
-
|
|
5541
|
+
const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
5542
|
+
const repaired = this.repairTruncatedJson(withoutThinking);
|
|
5543
|
+
if (repaired) {
|
|
5544
|
+
const parsed = JSON.parse(repaired);
|
|
5545
|
+
return {
|
|
5546
|
+
nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
|
|
5547
|
+
edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
|
|
5548
|
+
};
|
|
5549
|
+
}
|
|
5532
5550
|
} catch (e2) {
|
|
5533
5551
|
}
|
|
5534
5552
|
console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
|
|
@@ -5806,6 +5824,208 @@ ${error instanceof Error ? error.message : String(error)}`
|
|
|
5806
5824
|
}
|
|
5807
5825
|
};
|
|
5808
5826
|
|
|
5827
|
+
// src/rag/PromptOptimizer.ts
|
|
5828
|
+
var PromptOptimizer = class {
|
|
5829
|
+
/**
|
|
5830
|
+
* Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
|
|
5831
|
+
*/
|
|
5832
|
+
static optimize(rawPrompt, history = [], options = {}) {
|
|
5833
|
+
if (!rawPrompt || typeof rawPrompt !== "string") {
|
|
5834
|
+
return {
|
|
5835
|
+
optimizedPrompt: "",
|
|
5836
|
+
originalPrompt: rawPrompt || "",
|
|
5837
|
+
wasModified: false,
|
|
5838
|
+
transformations: []
|
|
5839
|
+
};
|
|
5840
|
+
}
|
|
5841
|
+
const { resolveHistory = true, expandKeywords = true } = options;
|
|
5842
|
+
const transformations = [];
|
|
5843
|
+
let current = rawPrompt;
|
|
5844
|
+
const cleaned = this.cleanText(current);
|
|
5845
|
+
if (cleaned !== current) {
|
|
5846
|
+
transformations.push("clean_whitespace_and_artifacts");
|
|
5847
|
+
current = cleaned;
|
|
5848
|
+
}
|
|
5849
|
+
const expandedAbbr = this.expandAbbreviations(current);
|
|
5850
|
+
if (expandedAbbr !== current) {
|
|
5851
|
+
transformations.push("expand_abbreviations");
|
|
5852
|
+
current = expandedAbbr;
|
|
5853
|
+
}
|
|
5854
|
+
if (expandKeywords) {
|
|
5855
|
+
const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
|
|
5856
|
+
if (this.TOPIC_EXPANSIONS[trimmedLower]) {
|
|
5857
|
+
current = this.TOPIC_EXPANSIONS[trimmedLower];
|
|
5858
|
+
transformations.push(`topic_expansion_${trimmedLower}`);
|
|
5859
|
+
}
|
|
5860
|
+
}
|
|
5861
|
+
if (resolveHistory && history && history.length > 0) {
|
|
5862
|
+
const resolved = this.resolveCoreferences(current, history);
|
|
5863
|
+
if (resolved !== current) {
|
|
5864
|
+
transformations.push("resolve_history_coreferences");
|
|
5865
|
+
current = resolved;
|
|
5866
|
+
}
|
|
5867
|
+
}
|
|
5868
|
+
if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
|
|
5869
|
+
if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
|
|
5870
|
+
current = `${current.trim()}?`;
|
|
5871
|
+
transformations.push("add_question_mark");
|
|
5872
|
+
}
|
|
5873
|
+
}
|
|
5874
|
+
return {
|
|
5875
|
+
optimizedPrompt: current,
|
|
5876
|
+
originalPrompt: rawPrompt,
|
|
5877
|
+
wasModified: current !== rawPrompt,
|
|
5878
|
+
transformations
|
|
5879
|
+
};
|
|
5880
|
+
}
|
|
5881
|
+
/**
|
|
5882
|
+
* Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
|
|
5883
|
+
*/
|
|
5884
|
+
static cleanText(text) {
|
|
5885
|
+
let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
|
|
5886
|
+
const doubleQuotesCount = (clean.match(/"/g) || []).length;
|
|
5887
|
+
if (doubleQuotesCount % 2 !== 0) {
|
|
5888
|
+
clean += '"';
|
|
5889
|
+
}
|
|
5890
|
+
const singleQuotesCount = (clean.match(/'/g) || []).length;
|
|
5891
|
+
if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
|
|
5892
|
+
clean += "'";
|
|
5893
|
+
}
|
|
5894
|
+
const openParen = (clean.match(/\(/g) || []).length;
|
|
5895
|
+
const closeParen = (clean.match(/\)/g) || []).length;
|
|
5896
|
+
if (openParen > closeParen) {
|
|
5897
|
+
clean += ")".repeat(openParen - closeParen);
|
|
5898
|
+
}
|
|
5899
|
+
return clean;
|
|
5900
|
+
}
|
|
5901
|
+
/**
|
|
5902
|
+
* Replaces shorthand chat abbreviations with clear natural language.
|
|
5903
|
+
*/
|
|
5904
|
+
static expandAbbreviations(text) {
|
|
5905
|
+
let expanded = text;
|
|
5906
|
+
for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
|
|
5907
|
+
expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
|
|
5908
|
+
}
|
|
5909
|
+
return expanded;
|
|
5910
|
+
}
|
|
5911
|
+
/**
|
|
5912
|
+
* Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
|
|
5913
|
+
*/
|
|
5914
|
+
static resolveCoreferences(prompt, history) {
|
|
5915
|
+
const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
|
|
5916
|
+
if (!pronounPattern.test(prompt)) {
|
|
5917
|
+
return prompt;
|
|
5918
|
+
}
|
|
5919
|
+
const recentMessages = [...history].slice(-4).reverse();
|
|
5920
|
+
let detectedEntity = null;
|
|
5921
|
+
for (const msg of recentMessages) {
|
|
5922
|
+
if (!msg.content) continue;
|
|
5923
|
+
const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
|
|
5924
|
+
if (quotedMatch) {
|
|
5925
|
+
detectedEntity = quotedMatch[1];
|
|
5926
|
+
break;
|
|
5927
|
+
}
|
|
5928
|
+
const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
|
|
5929
|
+
if (headingMatch) {
|
|
5930
|
+
detectedEntity = headingMatch[1].trim();
|
|
5931
|
+
break;
|
|
5932
|
+
}
|
|
5933
|
+
const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
|
|
5934
|
+
if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
|
|
5935
|
+
detectedEntity = entityMatches[1];
|
|
5936
|
+
break;
|
|
5937
|
+
}
|
|
5938
|
+
}
|
|
5939
|
+
if (detectedEntity) {
|
|
5940
|
+
let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
|
|
5941
|
+
return resolved;
|
|
5942
|
+
}
|
|
5943
|
+
return prompt;
|
|
5944
|
+
}
|
|
5945
|
+
/**
|
|
5946
|
+
* Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
|
|
5947
|
+
*/
|
|
5948
|
+
static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
|
|
5949
|
+
const fastResult = this.optimize(rawPrompt, history);
|
|
5950
|
+
if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
|
|
5951
|
+
return fastResult;
|
|
5952
|
+
}
|
|
5953
|
+
try {
|
|
5954
|
+
const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
|
|
5955
|
+
const prompt = `You are a query optimizer in an enterprise RAG system.
|
|
5956
|
+
Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
|
|
5957
|
+
- Do NOT answer the question.
|
|
5958
|
+
- Resolve any pronouns like "it", "they", "that" using the context.
|
|
5959
|
+
- Keep the user's intent 100% intact.
|
|
5960
|
+
- If the question is already clear, return it as is.
|
|
5961
|
+
- Return ONLY the optimized query string. No quotes or explanations.
|
|
5962
|
+
|
|
5963
|
+
Context:
|
|
5964
|
+
${historyContext || "No previous context."}
|
|
5965
|
+
|
|
5966
|
+
User Question: "${rawPrompt}"
|
|
5967
|
+
|
|
5968
|
+
Optimized Search Prompt:`;
|
|
5969
|
+
const response = await llmProvider.chat(
|
|
5970
|
+
[{ role: "user", content: prompt }],
|
|
5971
|
+
"",
|
|
5972
|
+
{
|
|
5973
|
+
systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
|
|
5974
|
+
temperature: 0,
|
|
5975
|
+
maxTokens: 100
|
|
5976
|
+
}
|
|
5977
|
+
);
|
|
5978
|
+
const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
|
|
5979
|
+
if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
|
|
5980
|
+
return {
|
|
5981
|
+
optimizedPrompt: cleaned,
|
|
5982
|
+
originalPrompt: rawPrompt,
|
|
5983
|
+
wasModified: cleaned !== rawPrompt,
|
|
5984
|
+
transformations: [...fastResult.transformations, "llm_deep_reformulation"]
|
|
5985
|
+
};
|
|
5986
|
+
}
|
|
5987
|
+
} catch (err) {
|
|
5988
|
+
console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
|
|
5989
|
+
}
|
|
5990
|
+
return fastResult;
|
|
5991
|
+
}
|
|
5992
|
+
};
|
|
5993
|
+
PromptOptimizer.COMMON_ABBREVIATIONS = {
|
|
5994
|
+
"\\bpls\\b": "please",
|
|
5995
|
+
"\\bplz\\b": "please",
|
|
5996
|
+
"\\bthx\\b": "thanks",
|
|
5997
|
+
"\\bty\\b": "thank you",
|
|
5998
|
+
"\\bapprox\\b": "approximately",
|
|
5999
|
+
"\\bw/o\\b": "without",
|
|
6000
|
+
"\\bw/\\b": "with",
|
|
6001
|
+
"\\bdiff\\b": "difference",
|
|
6002
|
+
"\\binfo\\b": "information",
|
|
6003
|
+
"\\bconfig\\b": "configuration",
|
|
6004
|
+
"\\bdoc\\b": "document",
|
|
6005
|
+
"\\bdocs\\b": "documentation",
|
|
6006
|
+
"\\bfeat\\b": "feature",
|
|
6007
|
+
"\\bfeats\\b": "features",
|
|
6008
|
+
"\\bvs\\b": "versus",
|
|
6009
|
+
"\\bu\\b": "you",
|
|
6010
|
+
"\\bur\\b": "your",
|
|
6011
|
+
"\\br\\b": "are"
|
|
6012
|
+
};
|
|
6013
|
+
PromptOptimizer.TOPIC_EXPANSIONS = {
|
|
6014
|
+
"pricing": "What are the available pricing plans, tiers, and cost structures?",
|
|
6015
|
+
"price": "What is the pricing and subscription cost?",
|
|
6016
|
+
"features": "What are the core features, capabilities, and tools provided?",
|
|
6017
|
+
"security": "What are the security, privacy, and compliance standards?",
|
|
6018
|
+
"privacy": "What is the privacy policy and how is data protected?",
|
|
6019
|
+
"refund": "What is the refund and cancellation policy?",
|
|
6020
|
+
"refunds": "What is the refund and cancellation policy?",
|
|
6021
|
+
"contact": "How can I contact customer support or sales?",
|
|
6022
|
+
"support": "What support channels and assistance are available?",
|
|
6023
|
+
"docs": "Where can I find the official documentation and setup guides?",
|
|
6024
|
+
"installation": "How do I install and configure the SDK or service?",
|
|
6025
|
+
"install": "What are the step-by-step installation instructions?",
|
|
6026
|
+
"faq": "What are the frequently asked questions and key answers?"
|
|
6027
|
+
};
|
|
6028
|
+
|
|
5809
6029
|
// src/core/mcp.ts
|
|
5810
6030
|
import { spawn } from "child_process";
|
|
5811
6031
|
var MCPClient = class {
|
|
@@ -7023,6 +7243,26 @@ var IntentClassifier = class {
|
|
|
7023
7243
|
static extractDataSignals(context) {
|
|
7024
7244
|
const docs = context.retrievedDocuments || [];
|
|
7025
7245
|
const rowCount = docs.length;
|
|
7246
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
7247
|
+
"filetype",
|
|
7248
|
+
"mimetype",
|
|
7249
|
+
"filename",
|
|
7250
|
+
"docid",
|
|
7251
|
+
"uploadedat",
|
|
7252
|
+
"dimension",
|
|
7253
|
+
"chunkindex",
|
|
7254
|
+
"totalchunks",
|
|
7255
|
+
"charactercount",
|
|
7256
|
+
"workspaceid",
|
|
7257
|
+
"pineconenamespace",
|
|
7258
|
+
"tier",
|
|
7259
|
+
"custommetadata",
|
|
7260
|
+
"csvheaders",
|
|
7261
|
+
"headers",
|
|
7262
|
+
"filesize",
|
|
7263
|
+
"content",
|
|
7264
|
+
"id"
|
|
7265
|
+
]);
|
|
7026
7266
|
let hasNumericFields = false;
|
|
7027
7267
|
let hasDateFields = false;
|
|
7028
7268
|
let hasCategoricalFields = false;
|
|
@@ -7038,6 +7278,7 @@ var IntentClassifier = class {
|
|
|
7038
7278
|
docs.forEach((doc) => {
|
|
7039
7279
|
const meta = doc.metadata || {};
|
|
7040
7280
|
Object.entries(meta).forEach(([key, val]) => {
|
|
7281
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
7041
7282
|
keys.add(key);
|
|
7042
7283
|
const kLower = key.toLowerCase();
|
|
7043
7284
|
if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
|
|
@@ -7046,7 +7287,7 @@ var IntentClassifier = class {
|
|
|
7046
7287
|
if (typeof val === "number") {
|
|
7047
7288
|
numericKeys.add(key);
|
|
7048
7289
|
} else if (typeof val === "string") {
|
|
7049
|
-
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
|
|
7290
|
+
if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
|
|
7050
7291
|
dateKeys.add(key);
|
|
7051
7292
|
} else if (val.length < 50 && !val.includes("\n")) {
|
|
7052
7293
|
catKeys.add(key);
|
|
@@ -7059,10 +7300,7 @@ var IntentClassifier = class {
|
|
|
7059
7300
|
hasCategoricalFields = catKeys.size > 0;
|
|
7060
7301
|
numericFieldCount = numericKeys.size;
|
|
7061
7302
|
categoricalFieldCount = catKeys.size;
|
|
7062
|
-
if (productKeyMatches >= 2
|
|
7063
|
-
var _a2, _b;
|
|
7064
|
-
return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
|
|
7065
|
-
})) {
|
|
7303
|
+
if (productKeyMatches >= 2) {
|
|
7066
7304
|
isProductLike = true;
|
|
7067
7305
|
}
|
|
7068
7306
|
}
|
|
@@ -7078,12 +7316,13 @@ var IntentClassifier = class {
|
|
|
7078
7316
|
}
|
|
7079
7317
|
// ─── Intent Heuristic Checkers ─────────────────────────────────────────────
|
|
7080
7318
|
static isInformationLookup(query) {
|
|
7081
|
-
|
|
7082
|
-
|
|
7319
|
+
const q = query.toLowerCase().trim();
|
|
7320
|
+
if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
|
|
7321
|
+
if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
|
|
7083
7322
|
return true;
|
|
7084
7323
|
}
|
|
7085
7324
|
}
|
|
7086
|
-
if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(
|
|
7325
|
+
if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
|
|
7087
7326
|
return true;
|
|
7088
7327
|
}
|
|
7089
7328
|
return false;
|
|
@@ -7098,13 +7337,13 @@ var IntentClassifier = class {
|
|
|
7098
7337
|
return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
|
|
7099
7338
|
}
|
|
7100
7339
|
static isTrendQuery(query) {
|
|
7101
|
-
return /\b(trend|trends|growth
|
|
7340
|
+
return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
|
|
7102
7341
|
}
|
|
7103
7342
|
static isRankingQuery(query) {
|
|
7104
7343
|
return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
|
|
7105
7344
|
}
|
|
7106
7345
|
static isAnalyticsQuery(query) {
|
|
7107
|
-
return /\b(
|
|
7346
|
+
return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
|
|
7108
7347
|
}
|
|
7109
7348
|
};
|
|
7110
7349
|
|
|
@@ -7116,8 +7355,8 @@ var Rule1SpecificInfoRule = class {
|
|
|
7116
7355
|
}
|
|
7117
7356
|
evaluate(context, intent) {
|
|
7118
7357
|
const q = (context.userQuery || "").toLowerCase().trim();
|
|
7119
|
-
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
|
|
7120
|
-
const isNonVisual = !/\b(
|
|
7358
|
+
const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
|
|
7359
|
+
const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
|
|
7121
7360
|
if (isFactQuery && isNonVisual) {
|
|
7122
7361
|
return {
|
|
7123
7362
|
renderType: "text",
|
|
@@ -7182,11 +7421,11 @@ var Rule4AnalyticalRule = class {
|
|
|
7182
7421
|
const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
|
|
7183
7422
|
if (!isAnalytical) return null;
|
|
7184
7423
|
let chartType = "bar";
|
|
7185
|
-
if (intent === "trend_analysis" || /\b(monthly|yearly|
|
|
7424
|
+
if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
|
|
7186
7425
|
chartType = "line";
|
|
7187
|
-
} else if (/\b(share|percentage|
|
|
7426
|
+
} else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
|
|
7188
7427
|
chartType = "pie";
|
|
7189
|
-
} else if (/\b(correlation|scatter|relationship
|
|
7428
|
+
} else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
|
|
7190
7429
|
chartType = "scatter";
|
|
7191
7430
|
} else {
|
|
7192
7431
|
chartType = "bar";
|
|
@@ -7949,12 +8188,20 @@ ${schemaProfileText}` : ""}`;
|
|
|
7949
8188
|
var _a3;
|
|
7950
8189
|
return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
|
|
7951
8190
|
}).filter(Boolean))) : this.detectCategories(data);
|
|
7952
|
-
if (categories.length
|
|
8191
|
+
if (categories.length <= 1) return null;
|
|
7953
8192
|
const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
|
|
8193
|
+
const hasInventoryData = data.some((item) => {
|
|
8194
|
+
const meta = item.metadata || {};
|
|
8195
|
+
return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
|
|
8196
|
+
});
|
|
7954
8197
|
const pieData = Object.entries(categoryData).map(([label, count]) => {
|
|
8198
|
+
if (!hasInventoryData) {
|
|
8199
|
+
return { label, value: count };
|
|
8200
|
+
}
|
|
7955
8201
|
const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
|
|
7956
8202
|
return { label, value: count, inStockCount, outOfStockCount };
|
|
7957
8203
|
});
|
|
8204
|
+
if (pieData.length <= 1) return null;
|
|
7958
8205
|
return {
|
|
7959
8206
|
type: "pie_chart",
|
|
7960
8207
|
title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
|
|
@@ -8261,41 +8508,34 @@ ${schemaProfileText}` : ""}`;
|
|
|
8261
8508
|
}
|
|
8262
8509
|
static _productQueryTest(query) {
|
|
8263
8510
|
const q = query.toLowerCase();
|
|
8264
|
-
const
|
|
8511
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
|
|
8512
|
+
if (financialTerms.test(q)) {
|
|
8513
|
+
return false;
|
|
8514
|
+
}
|
|
8515
|
+
const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
|
|
8265
8516
|
const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
|
|
8266
8517
|
const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
|
|
8267
8518
|
return productTerms && productAction && !nonProductEntityTerms;
|
|
8268
8519
|
}
|
|
8269
8520
|
static isProductData(item) {
|
|
8270
8521
|
if (!item) return false;
|
|
8271
|
-
const content = (item.content || "").toLowerCase();
|
|
8272
|
-
const productKeywords = [
|
|
8273
|
-
"product",
|
|
8274
|
-
"price",
|
|
8275
|
-
"stock",
|
|
8276
|
-
"sku",
|
|
8277
|
-
"brand",
|
|
8278
|
-
"msrp",
|
|
8279
|
-
"inventory",
|
|
8280
|
-
"buy",
|
|
8281
|
-
"shop",
|
|
8282
|
-
"beauty",
|
|
8283
|
-
"cosmetic",
|
|
8284
|
-
"facial",
|
|
8285
|
-
"cream",
|
|
8286
|
-
"serum",
|
|
8287
|
-
"mask",
|
|
8288
|
-
"makeup",
|
|
8289
|
-
"fragrance"
|
|
8290
|
-
];
|
|
8291
|
-
const hasKeywords = productKeywords.some((kw) => content.includes(kw));
|
|
8292
8522
|
const metadata = item.metadata || {};
|
|
8293
|
-
const
|
|
8294
|
-
|
|
8295
|
-
return
|
|
8296
|
-
}
|
|
8297
|
-
|
|
8298
|
-
|
|
8523
|
+
const domain = String(metadata.domain || "").toLowerCase();
|
|
8524
|
+
if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
|
|
8525
|
+
return false;
|
|
8526
|
+
}
|
|
8527
|
+
if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
|
|
8528
|
+
return true;
|
|
8529
|
+
}
|
|
8530
|
+
const content = (item.content || "").toLowerCase();
|
|
8531
|
+
const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
|
|
8532
|
+
if (financialTerms.test(content)) {
|
|
8533
|
+
return false;
|
|
8534
|
+
}
|
|
8535
|
+
const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
|
|
8536
|
+
const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
|
|
8537
|
+
const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
|
|
8538
|
+
return hasSku || hasProductIdentity && hasPrice;
|
|
8299
8539
|
}
|
|
8300
8540
|
static isTimeSeriesData(item) {
|
|
8301
8541
|
const metadata = item.metadata || {};
|
|
@@ -8314,10 +8554,31 @@ ${schemaProfileText}` : ""}`;
|
|
|
8314
8554
|
return fieldCount.size > 2;
|
|
8315
8555
|
}
|
|
8316
8556
|
static profileData(data) {
|
|
8557
|
+
const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
|
|
8558
|
+
"filetype",
|
|
8559
|
+
"mimetype",
|
|
8560
|
+
"filename",
|
|
8561
|
+
"docid",
|
|
8562
|
+
"uploadedat",
|
|
8563
|
+
"dimension",
|
|
8564
|
+
"chunkindex",
|
|
8565
|
+
"totalchunks",
|
|
8566
|
+
"charactercount",
|
|
8567
|
+
"workspaceid",
|
|
8568
|
+
"pineconenamespace",
|
|
8569
|
+
"tier",
|
|
8570
|
+
"custommetadata",
|
|
8571
|
+
"csvheaders",
|
|
8572
|
+
"headers",
|
|
8573
|
+
"filesize",
|
|
8574
|
+
"content",
|
|
8575
|
+
"id"
|
|
8576
|
+
]);
|
|
8317
8577
|
const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
|
|
8318
8578
|
var _a2, _b;
|
|
8319
8579
|
const fields2 = {};
|
|
8320
8580
|
Object.entries(item.metadata || {}).forEach(([key, value]) => {
|
|
8581
|
+
if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
|
|
8321
8582
|
const primitive = this.toPrimitive(value);
|
|
8322
8583
|
if (primitive !== null) fields2[key] = primitive;
|
|
8323
8584
|
});
|
|
@@ -9271,27 +9532,74 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9271
9532
|
* Ingest documents with automatic chunking, embedding, and batch upsert.
|
|
9272
9533
|
*/
|
|
9273
9534
|
async ingest(documents, namespace) {
|
|
9535
|
+
var _a2;
|
|
9274
9536
|
await this.initialize();
|
|
9275
9537
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9276
9538
|
const results = [];
|
|
9539
|
+
if (!documents || documents.length === 0) return results;
|
|
9540
|
+
const docChunksMap = [];
|
|
9541
|
+
const allChunks = [];
|
|
9277
9542
|
for (const doc of documents) {
|
|
9278
9543
|
try {
|
|
9279
9544
|
const chunks = await this.prepareChunks(doc);
|
|
9280
|
-
|
|
9545
|
+
docChunksMap.push({ doc, chunks });
|
|
9546
|
+
allChunks.push(...chunks);
|
|
9547
|
+
} catch (error) {
|
|
9548
|
+
Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
|
|
9549
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9550
|
+
}
|
|
9551
|
+
}
|
|
9552
|
+
if (allChunks.length === 0) {
|
|
9553
|
+
return results;
|
|
9554
|
+
}
|
|
9555
|
+
try {
|
|
9556
|
+
const allVectors = await this.processEmbeddings(allChunks);
|
|
9557
|
+
let vectorOffset = 0;
|
|
9558
|
+
const allUpsertDocs = [];
|
|
9559
|
+
const docSuccessCount = /* @__PURE__ */ new Map();
|
|
9560
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9561
|
+
const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
|
|
9562
|
+
vectorOffset += chunks.length;
|
|
9281
9563
|
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9282
9564
|
id: chunk.id,
|
|
9283
|
-
vector:
|
|
9565
|
+
vector: docVectors[i],
|
|
9284
9566
|
content: chunk.content,
|
|
9285
9567
|
metadata: chunk.metadata
|
|
9286
9568
|
}));
|
|
9287
|
-
|
|
9288
|
-
|
|
9569
|
+
allUpsertDocs.push(...upsertDocs);
|
|
9570
|
+
docSuccessCount.set(doc.docId, chunks.length);
|
|
9571
|
+
}
|
|
9572
|
+
const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
|
|
9573
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9574
|
+
results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
|
|
9289
9575
|
if (this.graphDB && this.entityExtractor) {
|
|
9290
|
-
|
|
9576
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9577
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9578
|
+
});
|
|
9579
|
+
}
|
|
9580
|
+
}
|
|
9581
|
+
} catch (batchErr) {
|
|
9582
|
+
Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
|
|
9583
|
+
for (const { doc, chunks } of docChunksMap) {
|
|
9584
|
+
try {
|
|
9585
|
+
const vectors = await this.processEmbeddings(chunks);
|
|
9586
|
+
const upsertDocs = chunks.map((chunk, i) => ({
|
|
9587
|
+
id: chunk.id,
|
|
9588
|
+
vector: vectors[i],
|
|
9589
|
+
content: chunk.content,
|
|
9590
|
+
metadata: chunk.metadata
|
|
9591
|
+
}));
|
|
9592
|
+
const totalProcessed = await this.processUpserts(upsertDocs, ns);
|
|
9593
|
+
results.push({ docId: doc.docId, chunksIngested: totalProcessed });
|
|
9594
|
+
if (this.graphDB && this.entityExtractor) {
|
|
9595
|
+
this.processGraphIngestion(doc.docId, chunks).catch((err) => {
|
|
9596
|
+
Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
|
|
9597
|
+
});
|
|
9598
|
+
}
|
|
9599
|
+
} catch (singleErr) {
|
|
9600
|
+
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
|
|
9601
|
+
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9291
9602
|
}
|
|
9292
|
-
} catch (error) {
|
|
9293
|
-
Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
|
|
9294
|
-
results.push({ docId: doc.docId, chunksIngested: 0 });
|
|
9295
9603
|
}
|
|
9296
9604
|
}
|
|
9297
9605
|
return results;
|
|
@@ -9311,19 +9619,40 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9311
9619
|
}
|
|
9312
9620
|
/**
|
|
9313
9621
|
* Step 2: Generate embeddings for chunks with retry logic.
|
|
9314
|
-
* Uses batchEmbed when available for efficiency; falls back to
|
|
9622
|
+
* Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
|
|
9315
9623
|
*/
|
|
9316
9624
|
async processEmbeddings(chunks) {
|
|
9317
|
-
|
|
9318
|
-
|
|
9319
|
-
|
|
9320
|
-
|
|
9321
|
-
|
|
9322
|
-
|
|
9323
|
-
|
|
9625
|
+
if (chunks.length === 0) return [];
|
|
9626
|
+
const texts = chunks.map((c) => c.content);
|
|
9627
|
+
if (typeof this.embeddingProvider.batchEmbed === "function") {
|
|
9628
|
+
try {
|
|
9629
|
+
const batchSize = 50;
|
|
9630
|
+
const allVectors = [];
|
|
9631
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
9632
|
+
const slice = texts.slice(i, i + batchSize);
|
|
9633
|
+
const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
|
|
9634
|
+
allVectors.push(...batchVectors);
|
|
9635
|
+
}
|
|
9636
|
+
if (allVectors.length === chunks.length) {
|
|
9637
|
+
return allVectors;
|
|
9638
|
+
}
|
|
9639
|
+
} catch (err) {
|
|
9640
|
+
Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
|
|
9641
|
+
error: err instanceof Error ? err.message : String(err)
|
|
9642
|
+
});
|
|
9643
|
+
}
|
|
9644
|
+
}
|
|
9645
|
+
const concurrentResult = await BatchProcessor.processConcurrent(
|
|
9646
|
+
texts,
|
|
9324
9647
|
(text) => this.embeddingProvider.embed(text, { taskType: "document" }),
|
|
9325
|
-
|
|
9648
|
+
10,
|
|
9649
|
+
{
|
|
9650
|
+
maxRetries: 3,
|
|
9651
|
+
initialDelayMs: 100,
|
|
9652
|
+
throwOnPartialFailure: false
|
|
9653
|
+
}
|
|
9326
9654
|
);
|
|
9655
|
+
const vectors = concurrentResult.results;
|
|
9327
9656
|
if (vectors.length !== chunks.length) {
|
|
9328
9657
|
throw new EmbeddingFailedException(
|
|
9329
9658
|
`[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
|
|
@@ -9350,25 +9679,24 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9350
9679
|
}
|
|
9351
9680
|
/** Step 4: Optional graph-based entity extraction and ingestion. */
|
|
9352
9681
|
async processGraphIngestion(docId, chunks) {
|
|
9353
|
-
const
|
|
9354
|
-
|
|
9355
|
-
maxRetries: 1,
|
|
9356
|
-
initialDelayMs: 500
|
|
9357
|
-
};
|
|
9358
|
-
await BatchProcessor.processBatch(
|
|
9682
|
+
const maxConcurrency = Math.min(3, chunks.length);
|
|
9683
|
+
await BatchProcessor.processConcurrent(
|
|
9359
9684
|
chunks,
|
|
9360
|
-
async (
|
|
9361
|
-
|
|
9362
|
-
|
|
9363
|
-
|
|
9364
|
-
|
|
9365
|
-
|
|
9366
|
-
}
|
|
9367
|
-
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9368
|
-
}
|
|
9685
|
+
async (chunk) => {
|
|
9686
|
+
try {
|
|
9687
|
+
const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
|
|
9688
|
+
if (nodes.length > 0) await this.graphDB.addNodes(nodes);
|
|
9689
|
+
if (edges.length > 0) await this.graphDB.addEdges(edges);
|
|
9690
|
+
} catch (err) {
|
|
9691
|
+
Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
|
|
9369
9692
|
}
|
|
9370
9693
|
},
|
|
9371
|
-
|
|
9694
|
+
maxConcurrency,
|
|
9695
|
+
{
|
|
9696
|
+
maxRetries: 1,
|
|
9697
|
+
initialDelayMs: 200,
|
|
9698
|
+
throwOnPartialFailure: false
|
|
9699
|
+
}
|
|
9372
9700
|
);
|
|
9373
9701
|
}
|
|
9374
9702
|
async runNormalQuery(question, history = [], namespace) {
|
|
@@ -9493,7 +9821,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9493
9821
|
*/
|
|
9494
9822
|
askStreamInternal(_0) {
|
|
9495
9823
|
return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
|
|
9496
|
-
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
|
|
9824
|
+
var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
|
|
9497
9825
|
yield new __await(this.initialize());
|
|
9498
9826
|
const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
|
|
9499
9827
|
const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
|
|
@@ -9501,29 +9829,35 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9501
9829
|
const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
|
|
9502
9830
|
const requestStart = performance.now();
|
|
9503
9831
|
try {
|
|
9504
|
-
let
|
|
9832
|
+
let effectiveQuestion = question;
|
|
9833
|
+
let promptOptResult;
|
|
9834
|
+
if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
|
|
9835
|
+
promptOptResult = PromptOptimizer.optimize(question, history);
|
|
9836
|
+
effectiveQuestion = promptOptResult.optimizedPrompt;
|
|
9837
|
+
}
|
|
9838
|
+
let searchQuery = effectiveQuestion;
|
|
9505
9839
|
let rewrittenQuery;
|
|
9506
|
-
if ((
|
|
9507
|
-
searchQuery = yield new __await(this.rewriteQuery(
|
|
9508
|
-
rewrittenQuery = searchQuery !==
|
|
9840
|
+
if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
|
|
9841
|
+
searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
|
|
9842
|
+
rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
|
|
9509
9843
|
}
|
|
9510
|
-
const hints = QueryProcessor.extractQueryFieldHints(
|
|
9511
|
-
const filter = QueryProcessor.buildQueryFilter(
|
|
9512
|
-
const numericPredicates = QueryProcessor.extractNumericPredicates(
|
|
9844
|
+
const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
|
|
9845
|
+
const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
|
|
9846
|
+
const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
|
|
9513
9847
|
if (numericPredicates.length > 0) {
|
|
9514
9848
|
filter.__numericPredicates = numericPredicates;
|
|
9515
9849
|
}
|
|
9516
9850
|
const embedStart = performance.now();
|
|
9517
9851
|
const cacheKey = `${ns}::${searchQuery}`;
|
|
9518
9852
|
const cachedVector = this.embeddingCache.get(cacheKey);
|
|
9519
|
-
const arch = (
|
|
9520
|
-
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((
|
|
9853
|
+
const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
|
|
9854
|
+
const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
|
|
9521
9855
|
const [strategyResult, embeddedVector] = yield new __await(Promise.all([
|
|
9522
9856
|
QueryProcessor.determineRetrievalStrategy(
|
|
9523
9857
|
searchQuery,
|
|
9524
9858
|
useGraph ? this.llmRouter.get("fast") : void 0,
|
|
9525
|
-
(
|
|
9526
|
-
(
|
|
9859
|
+
(_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
|
|
9860
|
+
(_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
|
|
9527
9861
|
),
|
|
9528
9862
|
// Embed immediately regardless of strategy — costs nothing if cached.
|
|
9529
9863
|
// If strategy turns out to be 'graph'-only we just won't use the vector.
|
|
@@ -9533,7 +9867,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9533
9867
|
if (!cachedVector && queryVector.length > 0) {
|
|
9534
9868
|
this.embeddingCache.set(cacheKey, queryVector);
|
|
9535
9869
|
}
|
|
9536
|
-
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((
|
|
9870
|
+
const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
|
|
9537
9871
|
const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
|
|
9538
9872
|
const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
|
|
9539
9873
|
const wantsExhaustiveList = true;
|
|
@@ -9550,7 +9884,7 @@ ${m.content}`).join("\n\n---\n\n");
|
|
|
9550
9884
|
return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
|
|
9551
9885
|
});
|
|
9552
9886
|
const rerankLimit = Math.max(retrievalLimit, fullSources.length);
|
|
9553
|
-
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((
|
|
9887
|
+
const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
|
|
9554
9888
|
if (!hasNumericPredicates && useReranking) {
|
|
9555
9889
|
fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
|
|
9556
9890
|
}
|
|
@@ -9578,7 +9912,7 @@ ${graphContext}
|
|
|
9578
9912
|
VECTOR CONTEXT:
|
|
9579
9913
|
${context}`;
|
|
9580
9914
|
}
|
|
9581
|
-
if ((
|
|
9915
|
+
if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
|
|
9582
9916
|
if (!this.coldContexts.has(ns)) {
|
|
9583
9917
|
yield new __await(this.loadColdContext(ns));
|
|
9584
9918
|
}
|
|
@@ -9610,31 +9944,31 @@ ${context}`;
|
|
|
9610
9944
|
SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
|
|
9611
9945
|
});
|
|
9612
9946
|
}
|
|
9613
|
-
const isProductQ = UITransformer.isProductQueryPublic(
|
|
9947
|
+
const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
|
|
9614
9948
|
const uiTransformationPromise = isProductQ ? Promise.resolve(
|
|
9615
9949
|
UITransformer.transform(
|
|
9616
|
-
|
|
9950
|
+
effectiveQuestion,
|
|
9617
9951
|
sources,
|
|
9618
9952
|
this.config,
|
|
9619
9953
|
cachedSchema,
|
|
9620
9954
|
{ visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
|
|
9621
9955
|
)
|
|
9622
9956
|
) : this.generateUiTransformation(
|
|
9623
|
-
|
|
9957
|
+
effectiveQuestion,
|
|
9624
9958
|
sources,
|
|
9625
9959
|
cachedSchema,
|
|
9626
9960
|
hasNumericPredicates
|
|
9627
9961
|
).catch((uiError) => {
|
|
9628
9962
|
Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
|
|
9629
|
-
return UITransformer.transform(
|
|
9963
|
+
return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
|
|
9630
9964
|
});
|
|
9631
9965
|
let hallucinationScoringPromise = Promise.resolve(void 0);
|
|
9632
9966
|
const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
|
|
9633
9967
|
const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
|
|
9634
|
-
const isNativeThinking = isClaude37 && (((
|
|
9968
|
+
const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
|
|
9635
9969
|
const modelNameLower = this.config.llm.model.toLowerCase();
|
|
9636
9970
|
const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
|
|
9637
|
-
const isSimulatedThinking = !isNativeThinking && (((
|
|
9971
|
+
const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
|
|
9638
9972
|
let finalRestrictionSuffix = restrictionSuffix;
|
|
9639
9973
|
if (isSimulatedThinking) {
|
|
9640
9974
|
finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
|
|
@@ -9643,9 +9977,9 @@ ${context}`;
|
|
|
9643
9977
|
role: m.role || "user",
|
|
9644
9978
|
content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
|
|
9645
9979
|
}));
|
|
9646
|
-
const userQuestion = { role: "user", content:
|
|
9980
|
+
const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
|
|
9647
9981
|
const messages = [...hardenedHistory, userQuestion];
|
|
9648
|
-
const systemPrompt = (
|
|
9982
|
+
const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
|
|
9649
9983
|
const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
|
|
9650
9984
|
let fullReply = "";
|
|
9651
9985
|
let thinkingText = "";
|
|
@@ -9756,7 +10090,7 @@ ${context}`;
|
|
|
9756
10090
|
}
|
|
9757
10091
|
yield fullReply;
|
|
9758
10092
|
}
|
|
9759
|
-
const runHallucination = ((
|
|
10093
|
+
const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
|
|
9760
10094
|
if (runHallucination) {
|
|
9761
10095
|
hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
|
|
9762
10096
|
}
|
|
@@ -9778,7 +10112,7 @@ ${context}`;
|
|
|
9778
10112
|
totalTokens: promptTokens + completionTokens,
|
|
9779
10113
|
estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
|
|
9780
10114
|
};
|
|
9781
|
-
const awaitHallucination = ((
|
|
10115
|
+
const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
|
|
9782
10116
|
const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
|
|
9783
10117
|
uiTransformationPromise,
|
|
9784
10118
|
awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
|
|
@@ -9786,9 +10120,9 @@ ${context}`;
|
|
|
9786
10120
|
const buildTrace = (hScore) => __spreadValues({
|
|
9787
10121
|
requestId,
|
|
9788
10122
|
query: question,
|
|
9789
|
-
rewrittenQuery,
|
|
10123
|
+
rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
|
|
9790
10124
|
systemPrompt,
|
|
9791
|
-
userPrompt:
|
|
10125
|
+
userPrompt: effectiveQuestion + finalRestrictionSuffix,
|
|
9792
10126
|
chunks: (sources || []).filter(Boolean).map((s) => {
|
|
9793
10127
|
var _a3, _b2;
|
|
9794
10128
|
return {
|
|
@@ -9807,10 +10141,10 @@ ${context}`;
|
|
|
9807
10141
|
hallucinationReason: hScore.reason
|
|
9808
10142
|
} : {});
|
|
9809
10143
|
const trace = buildTrace(hallucinationResult);
|
|
9810
|
-
const isTelemetryActive = (
|
|
10144
|
+
const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
|
|
9811
10145
|
if (isTelemetryActive) {
|
|
9812
10146
|
const defaultUrl = "https://www.retrivora.com/api/telemetry";
|
|
9813
|
-
const telemetryUrl = ((
|
|
10147
|
+
const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
|
|
9814
10148
|
const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
|
|
9815
10149
|
(async () => {
|
|
9816
10150
|
var _a3, _b2, _c2, _d2, _e2;
|
|
@@ -10386,13 +10720,81 @@ var VectorPlugin = class {
|
|
|
10386
10720
|
|
|
10387
10721
|
// src/utils/DocumentParser.ts
|
|
10388
10722
|
var DocumentParser = class {
|
|
10723
|
+
/**
|
|
10724
|
+
* Fast Magic-Bytes sniffer for instant file format identification (<1ms).
|
|
10725
|
+
*/
|
|
10726
|
+
static sniffMagicBytes(buffer) {
|
|
10727
|
+
if (!buffer || buffer.length < 4) return null;
|
|
10728
|
+
if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
|
|
10729
|
+
return "pdf";
|
|
10730
|
+
}
|
|
10731
|
+
if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
|
|
10732
|
+
return "zip";
|
|
10733
|
+
}
|
|
10734
|
+
if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
|
|
10735
|
+
return "ole2";
|
|
10736
|
+
}
|
|
10737
|
+
const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
|
|
10738
|
+
if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
|
|
10739
|
+
return "json";
|
|
10740
|
+
}
|
|
10741
|
+
return "text";
|
|
10742
|
+
}
|
|
10743
|
+
/**
|
|
10744
|
+
* Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
|
|
10745
|
+
* Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
|
|
10746
|
+
*/
|
|
10747
|
+
static formatSheetToContextualText(sheet, sheetName, XLSX) {
|
|
10748
|
+
const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
|
|
10749
|
+
if (!jsonRows || jsonRows.length === 0) {
|
|
10750
|
+
const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
|
|
10751
|
+
return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
|
|
10752
|
+
${csv.trim()}` : "";
|
|
10753
|
+
}
|
|
10754
|
+
const lines = [`--- Sheet: ${sheetName} ---`];
|
|
10755
|
+
let currentHeader = [];
|
|
10756
|
+
for (let r = 0; r < jsonRows.length; r++) {
|
|
10757
|
+
const row = jsonRows[r];
|
|
10758
|
+
if (!row || row.length === 0) continue;
|
|
10759
|
+
const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
|
|
10760
|
+
if (cells.length === 0) continue;
|
|
10761
|
+
if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
|
|
10762
|
+
lines.push(`
|
|
10763
|
+
## ${cells.join(" ")}`);
|
|
10764
|
+
continue;
|
|
10765
|
+
}
|
|
10766
|
+
const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
|
|
10767
|
+
if (isHeaderCandidate && cells.length >= 2) {
|
|
10768
|
+
currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
|
|
10769
|
+
lines.push(`| ${cells.join(" | ")} |`);
|
|
10770
|
+
lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
|
|
10771
|
+
continue;
|
|
10772
|
+
}
|
|
10773
|
+
if (currentHeader.length > 0 && cells.length > 1) {
|
|
10774
|
+
const fieldPairs = [];
|
|
10775
|
+
for (let c = 0; c < row.length; c++) {
|
|
10776
|
+
const h = currentHeader[c];
|
|
10777
|
+
const val = row[c];
|
|
10778
|
+
if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
|
|
10779
|
+
fieldPairs.push(`${h}: ${String(val).trim()}`);
|
|
10780
|
+
}
|
|
10781
|
+
}
|
|
10782
|
+
if (fieldPairs.length > 0) {
|
|
10783
|
+
lines.push(fieldPairs.join(" | "));
|
|
10784
|
+
continue;
|
|
10785
|
+
}
|
|
10786
|
+
}
|
|
10787
|
+
lines.push(cells.join(", "));
|
|
10788
|
+
}
|
|
10789
|
+
return lines.join("\n");
|
|
10790
|
+
}
|
|
10389
10791
|
/**
|
|
10390
10792
|
* Extract text from a File or Buffer based on its type.
|
|
10391
10793
|
*/
|
|
10392
|
-
static async parse(file, fileName, mimeType) {
|
|
10794
|
+
static async parse(file, fileName, mimeType = "") {
|
|
10393
10795
|
var _a2;
|
|
10394
10796
|
const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
|
|
10395
|
-
if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
10797
|
+
if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
|
|
10396
10798
|
return this.readAsText(file);
|
|
10397
10799
|
}
|
|
10398
10800
|
if (extension === "json" || mimeType === "application/json") {
|
|
@@ -10404,38 +10806,115 @@ var DocumentParser = class {
|
|
|
10404
10806
|
return text;
|
|
10405
10807
|
}
|
|
10406
10808
|
}
|
|
10407
|
-
if (extension === "
|
|
10809
|
+
if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
|
|
10810
|
+
return this.readAsText(file);
|
|
10811
|
+
}
|
|
10812
|
+
if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
|
|
10408
10813
|
return this.readAsText(file);
|
|
10409
10814
|
}
|
|
10410
|
-
if (extension === "
|
|
10815
|
+
if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
|
|
10411
10816
|
try {
|
|
10412
|
-
const
|
|
10817
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
10413
10818
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10414
|
-
const
|
|
10415
|
-
|
|
10416
|
-
|
|
10417
|
-
|
|
10418
|
-
const
|
|
10419
|
-
|
|
10420
|
-
|
|
10421
|
-
|
|
10422
|
-
|
|
10819
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
10820
|
+
const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
|
|
10821
|
+
var _a3, _b;
|
|
10822
|
+
const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
|
|
10823
|
+
const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
|
|
10824
|
+
return numA - numB;
|
|
10825
|
+
});
|
|
10826
|
+
if (slideFiles.length > 0) {
|
|
10827
|
+
const slidesText = [];
|
|
10828
|
+
for (let i = 0; i < slideFiles.length; i++) {
|
|
10829
|
+
const slideXml = await zip.files[slideFiles[i]].async("text");
|
|
10830
|
+
const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
|
|
10831
|
+
if (textMatches && textMatches.length > 0) {
|
|
10832
|
+
const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
|
|
10833
|
+
if (slideContent.trim()) {
|
|
10834
|
+
slidesText.push(`--- Slide ${i + 1} ---
|
|
10835
|
+
${slideContent.trim()}`);
|
|
10836
|
+
}
|
|
10423
10837
|
}
|
|
10424
|
-
});
|
|
10425
|
-
const csv = csvLines.join("\n");
|
|
10426
|
-
if (csv && csv.trim()) {
|
|
10427
|
-
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
10428
|
-
${csv.trim()}`);
|
|
10429
10838
|
}
|
|
10430
|
-
|
|
10431
|
-
|
|
10432
|
-
|
|
10839
|
+
if (slidesText.length > 0) {
|
|
10840
|
+
return slidesText.join("\n\n");
|
|
10841
|
+
}
|
|
10842
|
+
}
|
|
10843
|
+
if (zip.files["content.xml"]) {
|
|
10844
|
+
const contentXml = await zip.files["content.xml"].async("text");
|
|
10845
|
+
const cleanText = this.stripXmlTags(contentXml);
|
|
10846
|
+
if (cleanText.trim()) {
|
|
10847
|
+
return cleanText.trim();
|
|
10848
|
+
}
|
|
10849
|
+
}
|
|
10850
|
+
const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
|
|
10851
|
+
const extractedSegments = [];
|
|
10852
|
+
for (const xf of allXmlFiles) {
|
|
10853
|
+
const xml = await zip.files[xf].async("text");
|
|
10854
|
+
const stripped = this.stripXmlTags(xml);
|
|
10855
|
+
if (stripped.length > 10) extractedSegments.push(stripped);
|
|
10856
|
+
}
|
|
10857
|
+
if (extractedSegments.length > 0) {
|
|
10858
|
+
return extractedSegments.join("\n\n");
|
|
10859
|
+
}
|
|
10860
|
+
return `[PowerPoint Presentation: ${fileName}]`;
|
|
10861
|
+
} catch (err) {
|
|
10862
|
+
console.warn("[DocumentParser] PowerPoint parsing failed:", err);
|
|
10433
10863
|
try {
|
|
10434
10864
|
return await this.readAsText(file);
|
|
10435
|
-
} catch (
|
|
10436
|
-
return `[
|
|
10865
|
+
} catch (e) {
|
|
10866
|
+
return `[PowerPoint Presentation File: ${fileName}]`;
|
|
10867
|
+
}
|
|
10868
|
+
}
|
|
10869
|
+
}
|
|
10870
|
+
if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
|
|
10871
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10872
|
+
try {
|
|
10873
|
+
const XLSX = await import("xlsx");
|
|
10874
|
+
const workbook = XLSX.read(buffer, { type: "buffer" });
|
|
10875
|
+
const sheetsText = [];
|
|
10876
|
+
for (const sheetName of workbook.SheetNames) {
|
|
10877
|
+
const sheet = workbook.Sheets[sheetName];
|
|
10878
|
+
const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
|
|
10879
|
+
if (formatted && formatted.trim()) {
|
|
10880
|
+
sheetsText.push(formatted.trim());
|
|
10881
|
+
}
|
|
10882
|
+
}
|
|
10883
|
+
if (sheetsText.length > 0) {
|
|
10884
|
+
return sheetsText.join("\n\n");
|
|
10437
10885
|
}
|
|
10886
|
+
} catch (xlsxErr) {
|
|
10887
|
+
console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
|
|
10438
10888
|
}
|
|
10889
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
10890
|
+
if (isZip) {
|
|
10891
|
+
try {
|
|
10892
|
+
const ExcelJS = await import("exceljs");
|
|
10893
|
+
const workbook = new ExcelJS.Workbook();
|
|
10894
|
+
await workbook.xlsx.load(buffer);
|
|
10895
|
+
const sheetsText = [];
|
|
10896
|
+
workbook.eachSheet((sheet) => {
|
|
10897
|
+
const csvLines = [];
|
|
10898
|
+
sheet.eachRow((row) => {
|
|
10899
|
+
const values = row.values;
|
|
10900
|
+
if (Array.isArray(values)) {
|
|
10901
|
+
csvLines.push(values.slice(1).join(","));
|
|
10902
|
+
}
|
|
10903
|
+
});
|
|
10904
|
+
const csv = csvLines.join("\n");
|
|
10905
|
+
if (csv && csv.trim()) {
|
|
10906
|
+
sheetsText.push(`--- Sheet: ${sheet.name} ---
|
|
10907
|
+
${csv.trim()}`);
|
|
10908
|
+
}
|
|
10909
|
+
});
|
|
10910
|
+
if (sheetsText.length > 0) {
|
|
10911
|
+
return sheetsText.join("\n\n");
|
|
10912
|
+
}
|
|
10913
|
+
} catch (err) {
|
|
10914
|
+
console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
|
|
10915
|
+
}
|
|
10916
|
+
}
|
|
10917
|
+
return this.parseLegacyExcelOrFallback(buffer, fileName);
|
|
10439
10918
|
}
|
|
10440
10919
|
if (extension === "pdf" || mimeType === "application/pdf") {
|
|
10441
10920
|
try {
|
|
@@ -10449,15 +10928,48 @@ ${csv.trim()}`);
|
|
|
10449
10928
|
return `[PDF Parsing Error for ${fileName}]`;
|
|
10450
10929
|
}
|
|
10451
10930
|
}
|
|
10452
|
-
if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
|
|
10931
|
+
if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
|
|
10453
10932
|
try {
|
|
10454
10933
|
const mammoth = await import("mammoth");
|
|
10455
10934
|
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10456
10935
|
const result = await mammoth.extractRawText({ buffer });
|
|
10457
|
-
|
|
10936
|
+
if (result.value && result.value.trim()) {
|
|
10937
|
+
return result.value;
|
|
10938
|
+
}
|
|
10939
|
+
} catch (err) {
|
|
10940
|
+
console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
|
|
10941
|
+
}
|
|
10942
|
+
try {
|
|
10943
|
+
const raw = await this.readAsText(file);
|
|
10944
|
+
if (extension === "rtf" || raw.startsWith("{\\rtf")) {
|
|
10945
|
+
return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
|
|
10946
|
+
}
|
|
10947
|
+
return raw;
|
|
10948
|
+
} catch (e) {
|
|
10949
|
+
return `[Word Document File: ${fileName}]`;
|
|
10950
|
+
}
|
|
10951
|
+
}
|
|
10952
|
+
if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
|
|
10953
|
+
const raw = await this.readAsText(file);
|
|
10954
|
+
return this.stripHtmlAndXml(raw);
|
|
10955
|
+
}
|
|
10956
|
+
if (extension === "epub" || mimeType.includes("epub")) {
|
|
10957
|
+
try {
|
|
10958
|
+
const JSZip = (await import("jszip")).default || await import("jszip");
|
|
10959
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
10960
|
+
const zip = await JSZip.loadAsync(buffer);
|
|
10961
|
+
const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
|
|
10962
|
+
const chapters = [];
|
|
10963
|
+
for (const hf of htmlFiles) {
|
|
10964
|
+
const content = await zip.files[hf].async("text");
|
|
10965
|
+
const clean = this.stripHtmlAndXml(content);
|
|
10966
|
+
if (clean.length > 50) chapters.push(clean);
|
|
10967
|
+
}
|
|
10968
|
+
if (chapters.length > 0) {
|
|
10969
|
+
return chapters.join("\n\n---\n\n");
|
|
10970
|
+
}
|
|
10458
10971
|
} catch (err) {
|
|
10459
|
-
console.warn(
|
|
10460
|
-
return `[Word Parsing Error for ${fileName}]`;
|
|
10972
|
+
console.warn("[DocumentParser] EPUB parsing failed:", err);
|
|
10461
10973
|
}
|
|
10462
10974
|
}
|
|
10463
10975
|
try {
|
|
@@ -10466,6 +10978,237 @@ ${csv.trim()}`);
|
|
|
10466
10978
|
throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
|
|
10467
10979
|
}
|
|
10468
10980
|
}
|
|
10981
|
+
/**
|
|
10982
|
+
* Helper to strip XML tags and normalize inner text content.
|
|
10983
|
+
*/
|
|
10984
|
+
static stripXmlTags(xml) {
|
|
10985
|
+
return xml.replace(/<[^>]+>/g, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/'/g, "'").replace(/\s+/g, " ").trim();
|
|
10986
|
+
}
|
|
10987
|
+
/**
|
|
10988
|
+
* Helper to strip HTML / XML tags while removing script and style blocks completely.
|
|
10989
|
+
*/
|
|
10990
|
+
static stripHtmlAndXml(html) {
|
|
10991
|
+
return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/ /gi, " ").replace(/</g, "<").replace(/>/g, ">").replace(/&/g, "&").replace(/"/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
|
|
10992
|
+
}
|
|
10993
|
+
/**
|
|
10994
|
+
* Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
|
|
10995
|
+
*/
|
|
10996
|
+
static parseLegacyExcelOrFallback(buffer, fileName) {
|
|
10997
|
+
const rawStr = buffer.toString("utf-8");
|
|
10998
|
+
if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
|
|
10999
|
+
const rows = [];
|
|
11000
|
+
const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
|
|
11001
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11002
|
+
for (const rowHtml of rowMatches) {
|
|
11003
|
+
const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
|
|
11004
|
+
if (cells && cells.length > 0) {
|
|
11005
|
+
const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
|
|
11006
|
+
if (rowText) rows.push(rowText);
|
|
11007
|
+
}
|
|
11008
|
+
}
|
|
11009
|
+
}
|
|
11010
|
+
if (rows.length > 0) {
|
|
11011
|
+
return rows.join("\n");
|
|
11012
|
+
}
|
|
11013
|
+
const clean = this.stripHtmlAndXml(rawStr);
|
|
11014
|
+
if (clean.length > 10) return clean;
|
|
11015
|
+
}
|
|
11016
|
+
if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
|
|
11017
|
+
const rows = [];
|
|
11018
|
+
const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
|
|
11019
|
+
if (rowMatches && rowMatches.length > 0) {
|
|
11020
|
+
for (const rowXml of rowMatches) {
|
|
11021
|
+
const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
|
|
11022
|
+
if (dataMatches && dataMatches.length > 0) {
|
|
11023
|
+
const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
|
|
11024
|
+
if (rowText) rows.push(rowText);
|
|
11025
|
+
}
|
|
11026
|
+
}
|
|
11027
|
+
}
|
|
11028
|
+
if (rows.length > 0) {
|
|
11029
|
+
return rows.join("\n");
|
|
11030
|
+
}
|
|
11031
|
+
}
|
|
11032
|
+
const extractedStrings = [];
|
|
11033
|
+
const ignoredOleMarkers = /* @__PURE__ */ new Set([
|
|
11034
|
+
"root entry",
|
|
11035
|
+
"workbook",
|
|
11036
|
+
"compobj",
|
|
11037
|
+
"summaryinformation",
|
|
11038
|
+
"documentsummaryinformation",
|
|
11039
|
+
"microsoft excel",
|
|
11040
|
+
"worksheet",
|
|
11041
|
+
"biff8",
|
|
11042
|
+
"calcpr"
|
|
11043
|
+
]);
|
|
11044
|
+
let currentUtf16 = "";
|
|
11045
|
+
for (let i = 0; i < buffer.length - 1; i += 2) {
|
|
11046
|
+
const charCode = buffer[i];
|
|
11047
|
+
const zeroByte = buffer[i + 1];
|
|
11048
|
+
if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
|
|
11049
|
+
currentUtf16 += String.fromCharCode(charCode);
|
|
11050
|
+
} else {
|
|
11051
|
+
if (currentUtf16.length >= 2) {
|
|
11052
|
+
const trimmed = currentUtf16.trim();
|
|
11053
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11054
|
+
extractedStrings.push(trimmed);
|
|
11055
|
+
}
|
|
11056
|
+
}
|
|
11057
|
+
currentUtf16 = "";
|
|
11058
|
+
}
|
|
11059
|
+
}
|
|
11060
|
+
let currentAscii = "";
|
|
11061
|
+
for (let i = 0; i < buffer.length; i++) {
|
|
11062
|
+
const byte = buffer[i];
|
|
11063
|
+
if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
|
|
11064
|
+
currentAscii += String.fromCharCode(byte);
|
|
11065
|
+
} else {
|
|
11066
|
+
if (currentAscii.length >= 2) {
|
|
11067
|
+
const trimmed = currentAscii.trim();
|
|
11068
|
+
if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
|
|
11069
|
+
extractedStrings.push(trimmed);
|
|
11070
|
+
}
|
|
11071
|
+
}
|
|
11072
|
+
currentAscii = "";
|
|
11073
|
+
}
|
|
11074
|
+
}
|
|
11075
|
+
const uniqueStrings = [];
|
|
11076
|
+
for (const str of extractedStrings) {
|
|
11077
|
+
if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
|
|
11078
|
+
uniqueStrings.push(str);
|
|
11079
|
+
}
|
|
11080
|
+
}
|
|
11081
|
+
if (uniqueStrings.length > 0) {
|
|
11082
|
+
return uniqueStrings.join(" | ");
|
|
11083
|
+
}
|
|
11084
|
+
const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
|
|
11085
|
+
return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
|
|
11086
|
+
}
|
|
11087
|
+
/**
|
|
11088
|
+
* Dynamically infer document domain and schema from extracted text, filename, and format.
|
|
11089
|
+
*/
|
|
11090
|
+
static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
|
|
11091
|
+
const lowerText = text.toLowerCase();
|
|
11092
|
+
const lowerName = fileName.toLowerCase();
|
|
11093
|
+
const financialTerms = [
|
|
11094
|
+
"price per share",
|
|
11095
|
+
"share price",
|
|
11096
|
+
"total cost of the deal",
|
|
11097
|
+
"cost of the deal",
|
|
11098
|
+
"lbo",
|
|
11099
|
+
"merger",
|
|
11100
|
+
"valuation",
|
|
11101
|
+
"d/e ratio",
|
|
11102
|
+
"ebit",
|
|
11103
|
+
"ebitda",
|
|
11104
|
+
"fcfe",
|
|
11105
|
+
"fcff",
|
|
11106
|
+
"wacc",
|
|
11107
|
+
"beta",
|
|
11108
|
+
"working capital",
|
|
11109
|
+
"cogs",
|
|
11110
|
+
"cap spending",
|
|
11111
|
+
"depreciation",
|
|
11112
|
+
"cashflow",
|
|
11113
|
+
"cash flow",
|
|
11114
|
+
"net income",
|
|
11115
|
+
"balance sheet",
|
|
11116
|
+
"income statement",
|
|
11117
|
+
"interest rate",
|
|
11118
|
+
"terminal year",
|
|
11119
|
+
"equity",
|
|
11120
|
+
"debt outstanding",
|
|
11121
|
+
"gross margin",
|
|
11122
|
+
"fiscal year",
|
|
11123
|
+
"fy20",
|
|
11124
|
+
"fy21",
|
|
11125
|
+
"fy22",
|
|
11126
|
+
"fy23",
|
|
11127
|
+
"fy24",
|
|
11128
|
+
"fy25",
|
|
11129
|
+
"q1",
|
|
11130
|
+
"q2",
|
|
11131
|
+
"q3",
|
|
11132
|
+
"q4"
|
|
11133
|
+
];
|
|
11134
|
+
let financialScore = 0;
|
|
11135
|
+
for (const term of financialTerms) {
|
|
11136
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11137
|
+
financialScore += 1;
|
|
11138
|
+
}
|
|
11139
|
+
}
|
|
11140
|
+
const ecommerceTerms = [
|
|
11141
|
+
"sku",
|
|
11142
|
+
"product_id",
|
|
11143
|
+
"product id",
|
|
11144
|
+
"item_id",
|
|
11145
|
+
"add to cart",
|
|
11146
|
+
"buy now",
|
|
11147
|
+
"in stock",
|
|
11148
|
+
"out of stock",
|
|
11149
|
+
"inventory count",
|
|
11150
|
+
"retail price",
|
|
11151
|
+
"unit price",
|
|
11152
|
+
"msrp",
|
|
11153
|
+
"cart",
|
|
11154
|
+
"shop",
|
|
11155
|
+
"catalog",
|
|
11156
|
+
"product_name",
|
|
11157
|
+
"product name",
|
|
11158
|
+
"brand name"
|
|
11159
|
+
];
|
|
11160
|
+
let ecommerceScore = 0;
|
|
11161
|
+
for (const term of ecommerceTerms) {
|
|
11162
|
+
if (lowerText.includes(term) || lowerName.includes(term)) {
|
|
11163
|
+
ecommerceScore += 1;
|
|
11164
|
+
}
|
|
11165
|
+
}
|
|
11166
|
+
const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
|
|
11167
|
+
let legalScore = 0;
|
|
11168
|
+
for (const term of legalTerms) {
|
|
11169
|
+
if (lowerText.includes(term)) legalScore += 1;
|
|
11170
|
+
}
|
|
11171
|
+
const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
|
|
11172
|
+
let healthScore = 0;
|
|
11173
|
+
for (const term of healthTerms) {
|
|
11174
|
+
if (lowerText.includes(term)) healthScore += 1;
|
|
11175
|
+
}
|
|
11176
|
+
let schema;
|
|
11177
|
+
const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
|
|
11178
|
+
if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
|
|
11179
|
+
const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
|
|
11180
|
+
const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
|
|
11181
|
+
if (headers.length > 1 && lines.length > 1) {
|
|
11182
|
+
const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
|
|
11183
|
+
schema = {};
|
|
11184
|
+
headers.forEach((h, idx) => {
|
|
11185
|
+
const val = sampleRow[idx] || "";
|
|
11186
|
+
if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
|
|
11187
|
+
schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
|
|
11188
|
+
} else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
|
|
11189
|
+
schema[h] = "date";
|
|
11190
|
+
} else {
|
|
11191
|
+
schema[h] = "string";
|
|
11192
|
+
}
|
|
11193
|
+
});
|
|
11194
|
+
}
|
|
11195
|
+
}
|
|
11196
|
+
const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
|
|
11197
|
+
const isCatalog = ecommerceScore >= 2 && !isFinancial;
|
|
11198
|
+
let domain = "general";
|
|
11199
|
+
if (isFinancial) {
|
|
11200
|
+
domain = "finance";
|
|
11201
|
+
} else if (isCatalog) {
|
|
11202
|
+
domain = "ecommerce";
|
|
11203
|
+
} else if (legalScore >= 2) {
|
|
11204
|
+
domain = "legal";
|
|
11205
|
+
} else if (healthScore >= 2) {
|
|
11206
|
+
domain = "healthcare";
|
|
11207
|
+
} else if (schema && Object.keys(schema).length > 0) {
|
|
11208
|
+
domain = "tabular";
|
|
11209
|
+
}
|
|
11210
|
+
return { domain, schema, isCatalog, isFinancial };
|
|
11211
|
+
}
|
|
10469
11212
|
static async readAsText(file) {
|
|
10470
11213
|
if (Buffer.isBuffer(file)) {
|
|
10471
11214
|
return file.toString("utf-8");
|
|
@@ -11112,18 +11855,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
|
|
|
11112
11855
|
}
|
|
11113
11856
|
}
|
|
11114
11857
|
function getAllowedNamespaces(projectId) {
|
|
11115
|
-
const trimmed = projectId.trim();
|
|
11858
|
+
const trimmed = (projectId || "default").trim();
|
|
11116
11859
|
const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
|
|
11117
11860
|
return new Set([
|
|
11118
11861
|
trimmed,
|
|
11119
11862
|
withoutPrefix,
|
|
11120
|
-
`retrivora-${withoutPrefix}
|
|
11863
|
+
`retrivora-${withoutPrefix}`,
|
|
11864
|
+
"default",
|
|
11865
|
+
"retrivora-default"
|
|
11121
11866
|
].filter(Boolean));
|
|
11122
11867
|
}
|
|
11123
11868
|
function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
11869
|
+
var _a2;
|
|
11124
11870
|
const allowed = getAllowedNamespaces(payload.projectId);
|
|
11125
11871
|
const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
|
|
11126
|
-
|
|
11872
|
+
const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
|
|
11873
|
+
if (requested && !isDevOrFree && !allowed.has(requested)) {
|
|
11127
11874
|
return {
|
|
11128
11875
|
ok: false,
|
|
11129
11876
|
response: new Response(JSON.stringify({
|
|
@@ -11137,7 +11884,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
|
|
|
11137
11884
|
})
|
|
11138
11885
|
};
|
|
11139
11886
|
}
|
|
11140
|
-
const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
|
|
11887
|
+
const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
|
|
11141
11888
|
return { ok: true, namespace: requested || defaultNamespace };
|
|
11142
11889
|
}
|
|
11143
11890
|
var MAX_MESSAGE_LENGTH = 8e3;
|
|
@@ -11581,7 +12328,7 @@ function createIngestHandler(configOrPlugin, options) {
|
|
|
11581
12328
|
function createHealthHandler(configOrPlugin, options) {
|
|
11582
12329
|
const plugin = getOrCreatePlugin(configOrPlugin);
|
|
11583
12330
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
11584
|
-
return async function GET(req) {
|
|
12331
|
+
return async function GET(req, context) {
|
|
11585
12332
|
if (req) {
|
|
11586
12333
|
const authResult = await checkAuth(req, onAuthorize);
|
|
11587
12334
|
if (authResult) return authResult;
|
|
@@ -11692,9 +12439,10 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
11692
12439
|
}
|
|
11693
12440
|
const documents = [];
|
|
11694
12441
|
for (const file of files) {
|
|
12442
|
+
const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
|
|
11695
12443
|
const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
|
|
11696
12444
|
if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
|
|
11697
|
-
const text =
|
|
12445
|
+
const text = buffer.toString("utf-8");
|
|
11698
12446
|
const Papa = await import("papaparse");
|
|
11699
12447
|
const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
|
|
11700
12448
|
if (parsed.data && parsed.data.length > 0) {
|
|
@@ -11734,49 +12482,70 @@ function createUploadHandler(configOrPlugin, options) {
|
|
|
11734
12482
|
}
|
|
11735
12483
|
}
|
|
11736
12484
|
} else if (isExcel) {
|
|
11737
|
-
|
|
11738
|
-
|
|
11739
|
-
|
|
11740
|
-
|
|
11741
|
-
|
|
11742
|
-
|
|
11743
|
-
|
|
11744
|
-
|
|
11745
|
-
|
|
11746
|
-
|
|
11747
|
-
|
|
11748
|
-
|
|
11749
|
-
|
|
11750
|
-
|
|
11751
|
-
|
|
11752
|
-
|
|
11753
|
-
|
|
11754
|
-
|
|
11755
|
-
|
|
11756
|
-
|
|
11757
|
-
|
|
11758
|
-
|
|
11759
|
-
|
|
11760
|
-
|
|
11761
|
-
|
|
11762
|
-
|
|
11763
|
-
|
|
11764
|
-
|
|
11765
|
-
|
|
11766
|
-
|
|
11767
|
-
|
|
11768
|
-
|
|
11769
|
-
|
|
11770
|
-
|
|
11771
|
-
|
|
11772
|
-
|
|
11773
|
-
|
|
12485
|
+
let excelLoaded = false;
|
|
12486
|
+
try {
|
|
12487
|
+
const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
|
|
12488
|
+
if (isZip) {
|
|
12489
|
+
const ExcelJS = await import("exceljs");
|
|
12490
|
+
const workbook = new ExcelJS.Workbook();
|
|
12491
|
+
await workbook.xlsx.load(buffer);
|
|
12492
|
+
workbook.eachSheet((sheet, sheetId) => {
|
|
12493
|
+
const sheetName = sheet.name;
|
|
12494
|
+
let headers = [];
|
|
12495
|
+
let i = 0;
|
|
12496
|
+
sheet.eachRow((row, rowNumber) => {
|
|
12497
|
+
const values = Array.isArray(row.values) ? row.values.slice(1) : [];
|
|
12498
|
+
if (rowNumber === 1) {
|
|
12499
|
+
headers = values.map((v) => String(v));
|
|
12500
|
+
return;
|
|
12501
|
+
}
|
|
12502
|
+
i++;
|
|
12503
|
+
const contentParts = [];
|
|
12504
|
+
const rowData = {};
|
|
12505
|
+
values.forEach((val, index) => {
|
|
12506
|
+
const key = headers[index] || `Column${index + 1}`;
|
|
12507
|
+
rowData[key] = val;
|
|
12508
|
+
if (key && val !== void 0 && val !== null && String(val).trim()) {
|
|
12509
|
+
contentParts.push(`${key}: ${val}`);
|
|
12510
|
+
}
|
|
12511
|
+
});
|
|
12512
|
+
if (contentParts.length > 0) {
|
|
12513
|
+
documents.push({
|
|
12514
|
+
docId: `${file.name}-${sheetName}-row-${i}`,
|
|
12515
|
+
content: contentParts.join(", "),
|
|
12516
|
+
metadata: __spreadValues(__spreadProps(__spreadValues({
|
|
12517
|
+
fileName: file.name,
|
|
12518
|
+
sheetName,
|
|
12519
|
+
fileSize: file.size,
|
|
12520
|
+
fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
|
12521
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12522
|
+
}, dimension ? { dimension } : {}), {
|
|
12523
|
+
headers
|
|
12524
|
+
}), rowData)
|
|
12525
|
+
});
|
|
12526
|
+
}
|
|
11774
12527
|
});
|
|
11775
|
-
}
|
|
12528
|
+
});
|
|
12529
|
+
excelLoaded = documents.length > 0;
|
|
12530
|
+
}
|
|
12531
|
+
} catch (excelErr) {
|
|
12532
|
+
excelLoaded = false;
|
|
12533
|
+
}
|
|
12534
|
+
if (!excelLoaded) {
|
|
12535
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
12536
|
+
documents.push({
|
|
12537
|
+
docId: file.name,
|
|
12538
|
+
content,
|
|
12539
|
+
metadata: __spreadValues({
|
|
12540
|
+
fileName: file.name,
|
|
12541
|
+
fileSize: file.size,
|
|
12542
|
+
fileType: file.type || "application/vnd.ms-excel",
|
|
12543
|
+
uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
12544
|
+
}, dimension ? { dimension } : {})
|
|
11776
12545
|
});
|
|
11777
|
-
}
|
|
12546
|
+
}
|
|
11778
12547
|
} else {
|
|
11779
|
-
const content = await DocumentParser.parse(
|
|
12548
|
+
const content = await DocumentParser.parse(buffer, file.name, file.type);
|
|
11780
12549
|
documents.push({
|
|
11781
12550
|
docId: file.name,
|
|
11782
12551
|
content,
|
|
@@ -11935,7 +12704,7 @@ function createSessionsHandler(configOrPlugin, options) {
|
|
|
11935
12704
|
const storage = new DatabaseStorage(plugin.getConfig());
|
|
11936
12705
|
const onAuthorize = options == null ? void 0 : options.onAuthorize;
|
|
11937
12706
|
const onResolveScope = options == null ? void 0 : options.onResolveScope;
|
|
11938
|
-
return async function GET(req) {
|
|
12707
|
+
return async function GET(req, context) {
|
|
11939
12708
|
if (req) {
|
|
11940
12709
|
const authResult = await checkAuth(req, onAuthorize);
|
|
11941
12710
|
if (authResult) return authResult;
|