@retrivora-ai/rag-engine 2.5.7 → 2.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/dist/{ILLMProvider-DwFj0h2u.d.mts → ILLMProvider-B2N3lMBc.d.mts} +10 -1
  2. package/dist/{ILLMProvider-DwFj0h2u.d.ts → ILLMProvider-B2N3lMBc.d.ts} +10 -1
  3. package/dist/handlers/index.d.mts +2 -2
  4. package/dist/handlers/index.d.ts +2 -2
  5. package/dist/handlers/index.js +962 -193
  6. package/dist/handlers/index.mjs +12821 -24
  7. package/dist/{index-Cb0GsT16.d.ts → index-37V810a-.d.mts} +3 -3
  8. package/dist/{index-D1jN8KSF.d.mts → index-BxzDv2jd.d.ts} +3 -3
  9. package/dist/{index-9h_ENWQn.d.mts → index-CAGGPyI7.d.mts} +6 -2
  10. package/dist/{index-Xci6GfpI.d.ts → index-DdQxoIpu.d.ts} +6 -2
  11. package/dist/index.css +76 -4
  12. package/dist/index.d.mts +6 -6
  13. package/dist/index.d.ts +6 -6
  14. package/dist/index.js +138 -96
  15. package/dist/index.min.css +1 -1
  16. package/dist/index.mjs +1554 -130
  17. package/dist/server.d.mts +100 -9
  18. package/dist/server.d.ts +100 -9
  19. package/dist/server.js +964 -193
  20. package/dist/server.mjs +13378 -372
  21. package/package.json +5 -2
  22. package/dist/ChromaDBProvider-4RPJN7Y3.mjs +0 -8
  23. package/dist/MilvusProvider-WJLLZ6TW.mjs +0 -8
  24. package/dist/MongoDBProvider-QSERELZX.mjs +0 -8
  25. package/dist/MultiTablePostgresProvider-JF2HEUDZ.mjs +0 -9
  26. package/dist/PineconeProvider-HLZL7HKU.mjs +0 -10
  27. package/dist/PostgreSQLProvider-JQMCNHRO.mjs +0 -8
  28. package/dist/QdrantProvider-6MNFPEVC.mjs +0 -8
  29. package/dist/RedisProvider-ERQL6AEB.mjs +0 -8
  30. package/dist/SimpleGraphProvider-OP3HAJJM.mjs +0 -70
  31. package/dist/UniversalVectorProvider-INM3KH5F.mjs +0 -9
  32. package/dist/WeaviateProvider-VHWWICCC.mjs +0 -8
  33. package/dist/chunk-366EFBZC.mjs +0 -165
  34. package/dist/chunk-6MGVQTU4.mjs +0 -8883
  35. package/dist/chunk-B4AXIUM7.mjs +0 -49
  36. package/dist/chunk-BEPLYQBS.mjs +0 -89
  37. package/dist/chunk-BNOPQCZW.mjs +0 -238
  38. package/dist/chunk-FJ43J4FW.mjs +0 -654
  39. package/dist/chunk-GP2AJQRQ.mjs +0 -107
  40. package/dist/chunk-IMP6FUCY.mjs +0 -30
  41. package/dist/chunk-JRQWSJNF.mjs +0 -202
  42. package/dist/chunk-LYHCKUNF.mjs +0 -112
  43. package/dist/chunk-MFIWWUC7.mjs +0 -152
  44. package/dist/chunk-MWICNCB2.mjs +0 -157
  45. package/dist/chunk-N54YTFBN.mjs +0 -76
  46. package/dist/chunk-POOWU2HR.mjs +0 -211
  47. package/dist/chunk-QGDGPOZH.mjs +0 -217
  48. package/dist/chunk-QI3FTN6G.mjs +0 -281
  49. package/dist/chunk-QJIDGVG6.mjs +0 -334
  50. package/dist/chunk-W2DKVD3G.mjs +0 -81
  51. package/dist/chunk-XHSDHPKI.mjs +0 -336
@@ -94,8 +94,8 @@ function buildPayload(template, vars) {
94
94
  let raw = template;
95
95
  for (const [key, val] of Object.entries(vars)) {
96
96
  const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
97
- raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
98
- raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
97
+ raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
98
+ raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
99
99
  }
100
100
  try {
101
101
  return JSON.parse(raw);
@@ -318,7 +318,7 @@ var init_package = __esm({
318
318
  "package.json"() {
319
319
  package_default = {
320
320
  name: "@retrivora-ai/rag-engine",
321
- version: "2.5.7",
321
+ version: "2.5.9",
322
322
  description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
323
323
  author: "Abhinav Alkuchi",
324
324
  license: "UNLICENSED",
@@ -421,9 +421,11 @@ var init_package = __esm({
421
421
  "@google/genai": "^0.8.0",
422
422
  "@google/generative-ai": "^0.24.1",
423
423
  "@pinecone-database/pinecone": "^7.2.0",
424
- "@retrivora-ai/billing-guards": "*",
424
+ "@retrivora-ai/billing-guards": "file:../billing-guards",
425
425
  "@types/papaparse": "^5.5.2",
426
+ exceljs: "^4.4.0",
426
427
  "framer-motion": "^13.1.0",
428
+ jszip: "^3.10.1",
427
429
  katex: "^0.18.4",
428
430
  "lucide-react": "^1.8.0",
429
431
  "next-themes": "^0.4.6",
@@ -436,7 +438,7 @@ var init_package = __esm({
436
438
  "rehype-katex": "^7.0.1",
437
439
  "remark-gfm": "^4.0.1",
438
440
  "remark-math": "^6.0.0",
439
- exceljs: "^4.4.0",
441
+ xlsx: "^0.18.5",
440
442
  zod: "^4.4.3"
441
443
  },
442
444
  optionalDependencies: {
@@ -454,6 +456,7 @@ var init_package = __esm({
454
456
  "@tailwindcss/postcss": "^4",
455
457
  "@types/estree": "^1.0.9",
456
458
  "@types/jest": "^30.0.0",
459
+ "@types/jszip": "^3.4.1",
457
460
  "@types/katex": "^0.16.8",
458
461
  "@types/node": "^20",
459
462
  "@types/pdf-parse": "^1.1.5",
@@ -2798,7 +2801,7 @@ var _LicenseVerifier = class _LicenseVerifier {
2798
2801
  */
2799
2802
  static async generateMachineFingerprint() {
2800
2803
  if (typeof window !== "undefined" && typeof navigator !== "undefined") {
2801
- return navigator.userAgent || "browser-fallback-fingerprint";
2804
+ return "browser-client";
2802
2805
  }
2803
2806
  try {
2804
2807
  const os2 = await import("os");
@@ -2870,6 +2873,7 @@ var _LicenseVerifier = class _LicenseVerifier {
2870
2873
  try {
2871
2874
  response = await fetch(endpoint, {
2872
2875
  method: "POST",
2876
+ signal: AbortSignal.timeout(6e3),
2873
2877
  headers: {
2874
2878
  "Content-Type": "application/json",
2875
2879
  "x-license-key": licenseKey
@@ -5547,15 +5551,29 @@ Text to extract from:
5547
5551
  const response = await this.llm.chat([
5548
5552
  { role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
5549
5553
  { role: "user", content: prompt }
5550
- ], "", { maxTokens: 4096, temperature: 0.1 });
5554
+ ], "", { maxTokens: 800, temperature: 0.1 });
5551
5555
  try {
5552
- const jsonMatch = response.match(/\{[\s\S]*\}/);
5553
- const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
5554
- return JSON.parse(cleanJson);
5556
+ const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
5557
+ const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
5558
+ const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
5559
+ const jsonMatch = candidate.match(/\{[\s\S]*\}/);
5560
+ const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
5561
+ const parsed = JSON.parse(cleanJson);
5562
+ return {
5563
+ nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
5564
+ edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
5565
+ };
5555
5566
  } catch (e) {
5556
5567
  try {
5557
- const repaired = this.repairTruncatedJson(response);
5558
- if (repaired) return JSON.parse(repaired);
5568
+ const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
5569
+ const repaired = this.repairTruncatedJson(withoutThinking);
5570
+ if (repaired) {
5571
+ const parsed = JSON.parse(repaired);
5572
+ return {
5573
+ nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
5574
+ edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
5575
+ };
5576
+ }
5559
5577
  } catch (e2) {
5560
5578
  }
5561
5579
  console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
@@ -5833,6 +5851,208 @@ ${error instanceof Error ? error.message : String(error)}`
5833
5851
  }
5834
5852
  };
5835
5853
 
5854
+ // src/rag/PromptOptimizer.ts
5855
+ var PromptOptimizer = class {
5856
+ /**
5857
+ * Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
5858
+ */
5859
+ static optimize(rawPrompt, history = [], options = {}) {
5860
+ if (!rawPrompt || typeof rawPrompt !== "string") {
5861
+ return {
5862
+ optimizedPrompt: "",
5863
+ originalPrompt: rawPrompt || "",
5864
+ wasModified: false,
5865
+ transformations: []
5866
+ };
5867
+ }
5868
+ const { resolveHistory = true, expandKeywords = true } = options;
5869
+ const transformations = [];
5870
+ let current = rawPrompt;
5871
+ const cleaned = this.cleanText(current);
5872
+ if (cleaned !== current) {
5873
+ transformations.push("clean_whitespace_and_artifacts");
5874
+ current = cleaned;
5875
+ }
5876
+ const expandedAbbr = this.expandAbbreviations(current);
5877
+ if (expandedAbbr !== current) {
5878
+ transformations.push("expand_abbreviations");
5879
+ current = expandedAbbr;
5880
+ }
5881
+ if (expandKeywords) {
5882
+ const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
5883
+ if (this.TOPIC_EXPANSIONS[trimmedLower]) {
5884
+ current = this.TOPIC_EXPANSIONS[trimmedLower];
5885
+ transformations.push(`topic_expansion_${trimmedLower}`);
5886
+ }
5887
+ }
5888
+ if (resolveHistory && history && history.length > 0) {
5889
+ const resolved = this.resolveCoreferences(current, history);
5890
+ if (resolved !== current) {
5891
+ transformations.push("resolve_history_coreferences");
5892
+ current = resolved;
5893
+ }
5894
+ }
5895
+ if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
5896
+ if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
5897
+ current = `${current.trim()}?`;
5898
+ transformations.push("add_question_mark");
5899
+ }
5900
+ }
5901
+ return {
5902
+ optimizedPrompt: current,
5903
+ originalPrompt: rawPrompt,
5904
+ wasModified: current !== rawPrompt,
5905
+ transformations
5906
+ };
5907
+ }
5908
+ /**
5909
+ * Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
5910
+ */
5911
+ static cleanText(text) {
5912
+ let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
5913
+ const doubleQuotesCount = (clean.match(/"/g) || []).length;
5914
+ if (doubleQuotesCount % 2 !== 0) {
5915
+ clean += '"';
5916
+ }
5917
+ const singleQuotesCount = (clean.match(/'/g) || []).length;
5918
+ if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
5919
+ clean += "'";
5920
+ }
5921
+ const openParen = (clean.match(/\(/g) || []).length;
5922
+ const closeParen = (clean.match(/\)/g) || []).length;
5923
+ if (openParen > closeParen) {
5924
+ clean += ")".repeat(openParen - closeParen);
5925
+ }
5926
+ return clean;
5927
+ }
5928
+ /**
5929
+ * Replaces shorthand chat abbreviations with clear natural language.
5930
+ */
5931
+ static expandAbbreviations(text) {
5932
+ let expanded = text;
5933
+ for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
5934
+ expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
5935
+ }
5936
+ return expanded;
5937
+ }
5938
+ /**
5939
+ * Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
5940
+ */
5941
+ static resolveCoreferences(prompt, history) {
5942
+ const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
5943
+ if (!pronounPattern.test(prompt)) {
5944
+ return prompt;
5945
+ }
5946
+ const recentMessages = [...history].slice(-4).reverse();
5947
+ let detectedEntity = null;
5948
+ for (const msg of recentMessages) {
5949
+ if (!msg.content) continue;
5950
+ const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
5951
+ if (quotedMatch) {
5952
+ detectedEntity = quotedMatch[1];
5953
+ break;
5954
+ }
5955
+ const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
5956
+ if (headingMatch) {
5957
+ detectedEntity = headingMatch[1].trim();
5958
+ break;
5959
+ }
5960
+ const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
5961
+ if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
5962
+ detectedEntity = entityMatches[1];
5963
+ break;
5964
+ }
5965
+ }
5966
+ if (detectedEntity) {
5967
+ let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
5968
+ return resolved;
5969
+ }
5970
+ return prompt;
5971
+ }
5972
+ /**
5973
+ * Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
5974
+ */
5975
+ static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
5976
+ const fastResult = this.optimize(rawPrompt, history);
5977
+ if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
5978
+ return fastResult;
5979
+ }
5980
+ try {
5981
+ const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
5982
+ const prompt = `You are a query optimizer in an enterprise RAG system.
5983
+ Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
5984
+ - Do NOT answer the question.
5985
+ - Resolve any pronouns like "it", "they", "that" using the context.
5986
+ - Keep the user's intent 100% intact.
5987
+ - If the question is already clear, return it as is.
5988
+ - Return ONLY the optimized query string. No quotes or explanations.
5989
+
5990
+ Context:
5991
+ ${historyContext || "No previous context."}
5992
+
5993
+ User Question: "${rawPrompt}"
5994
+
5995
+ Optimized Search Prompt:`;
5996
+ const response = await llmProvider.chat(
5997
+ [{ role: "user", content: prompt }],
5998
+ "",
5999
+ {
6000
+ systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
6001
+ temperature: 0,
6002
+ maxTokens: 100
6003
+ }
6004
+ );
6005
+ const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
6006
+ if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
6007
+ return {
6008
+ optimizedPrompt: cleaned,
6009
+ originalPrompt: rawPrompt,
6010
+ wasModified: cleaned !== rawPrompt,
6011
+ transformations: [...fastResult.transformations, "llm_deep_reformulation"]
6012
+ };
6013
+ }
6014
+ } catch (err) {
6015
+ console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
6016
+ }
6017
+ return fastResult;
6018
+ }
6019
+ };
6020
+ PromptOptimizer.COMMON_ABBREVIATIONS = {
6021
+ "\\bpls\\b": "please",
6022
+ "\\bplz\\b": "please",
6023
+ "\\bthx\\b": "thanks",
6024
+ "\\bty\\b": "thank you",
6025
+ "\\bapprox\\b": "approximately",
6026
+ "\\bw/o\\b": "without",
6027
+ "\\bw/\\b": "with",
6028
+ "\\bdiff\\b": "difference",
6029
+ "\\binfo\\b": "information",
6030
+ "\\bconfig\\b": "configuration",
6031
+ "\\bdoc\\b": "document",
6032
+ "\\bdocs\\b": "documentation",
6033
+ "\\bfeat\\b": "feature",
6034
+ "\\bfeats\\b": "features",
6035
+ "\\bvs\\b": "versus",
6036
+ "\\bu\\b": "you",
6037
+ "\\bur\\b": "your",
6038
+ "\\br\\b": "are"
6039
+ };
6040
+ PromptOptimizer.TOPIC_EXPANSIONS = {
6041
+ "pricing": "What are the available pricing plans, tiers, and cost structures?",
6042
+ "price": "What is the pricing and subscription cost?",
6043
+ "features": "What are the core features, capabilities, and tools provided?",
6044
+ "security": "What are the security, privacy, and compliance standards?",
6045
+ "privacy": "What is the privacy policy and how is data protected?",
6046
+ "refund": "What is the refund and cancellation policy?",
6047
+ "refunds": "What is the refund and cancellation policy?",
6048
+ "contact": "How can I contact customer support or sales?",
6049
+ "support": "What support channels and assistance are available?",
6050
+ "docs": "Where can I find the official documentation and setup guides?",
6051
+ "installation": "How do I install and configure the SDK or service?",
6052
+ "install": "What are the step-by-step installation instructions?",
6053
+ "faq": "What are the frequently asked questions and key answers?"
6054
+ };
6055
+
5836
6056
  // src/core/mcp.ts
5837
6057
  var import_child_process = require("child_process");
5838
6058
  var MCPClient = class {
@@ -7050,6 +7270,26 @@ var IntentClassifier = class {
7050
7270
  static extractDataSignals(context) {
7051
7271
  const docs = context.retrievedDocuments || [];
7052
7272
  const rowCount = docs.length;
7273
+ const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
7274
+ "filetype",
7275
+ "mimetype",
7276
+ "filename",
7277
+ "docid",
7278
+ "uploadedat",
7279
+ "dimension",
7280
+ "chunkindex",
7281
+ "totalchunks",
7282
+ "charactercount",
7283
+ "workspaceid",
7284
+ "pineconenamespace",
7285
+ "tier",
7286
+ "custommetadata",
7287
+ "csvheaders",
7288
+ "headers",
7289
+ "filesize",
7290
+ "content",
7291
+ "id"
7292
+ ]);
7053
7293
  let hasNumericFields = false;
7054
7294
  let hasDateFields = false;
7055
7295
  let hasCategoricalFields = false;
@@ -7065,6 +7305,7 @@ var IntentClassifier = class {
7065
7305
  docs.forEach((doc) => {
7066
7306
  const meta = doc.metadata || {};
7067
7307
  Object.entries(meta).forEach(([key, val]) => {
7308
+ if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
7068
7309
  keys.add(key);
7069
7310
  const kLower = key.toLowerCase();
7070
7311
  if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
@@ -7073,7 +7314,7 @@ var IntentClassifier = class {
7073
7314
  if (typeof val === "number") {
7074
7315
  numericKeys.add(key);
7075
7316
  } else if (typeof val === "string") {
7076
- if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
7317
+ if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
7077
7318
  dateKeys.add(key);
7078
7319
  } else if (val.length < 50 && !val.includes("\n")) {
7079
7320
  catKeys.add(key);
@@ -7086,10 +7327,7 @@ var IntentClassifier = class {
7086
7327
  hasCategoricalFields = catKeys.size > 0;
7087
7328
  numericFieldCount = numericKeys.size;
7088
7329
  categoricalFieldCount = catKeys.size;
7089
- if (productKeyMatches >= 2 || docs.some((d) => {
7090
- var _a2, _b;
7091
- return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
7092
- })) {
7330
+ if (productKeyMatches >= 2) {
7093
7331
  isProductLike = true;
7094
7332
  }
7095
7333
  }
@@ -7105,12 +7343,13 @@ var IntentClassifier = class {
7105
7343
  }
7106
7344
  // ─── Intent Heuristic Checkers ─────────────────────────────────────────────
7107
7345
  static isInformationLookup(query) {
7108
- if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(query)) {
7109
- if (!/\b(compare|versus|vs\.?|trend|monthly|revenue|top 10|top selling|best|chart|graph|plot|breakdown|analytics)\b/i.test(query)) {
7346
+ const q = query.toLowerCase().trim();
7347
+ if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
7348
+ if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
7110
7349
  return true;
7111
7350
  }
7112
7351
  }
7113
- if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(query)) {
7352
+ if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
7114
7353
  return true;
7115
7354
  }
7116
7355
  return false;
@@ -7125,13 +7364,13 @@ var IntentClassifier = class {
7125
7364
  return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
7126
7365
  }
7127
7366
  static isTrendQuery(query) {
7128
- return /\b(trend|trends|growth|over time|historical|timeline|monthly|yearly|weekly|daily|revenue growth|sales growth)\b/i.test(query);
7367
+ return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
7129
7368
  }
7130
7369
  static isRankingQuery(query) {
7131
7370
  return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
7132
7371
  }
7133
7372
  static isAnalyticsQuery(query) {
7134
- return /\b(sales|revenue|analytics|insights|distribution|percentage|share|breakdown|breakup|scatter|correlation|metrics|performance|by region|by category)\b/i.test(query);
7373
+ return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
7135
7374
  }
7136
7375
  };
7137
7376
 
@@ -7143,8 +7382,8 @@ var Rule1SpecificInfoRule = class {
7143
7382
  }
7144
7383
  evaluate(context, intent) {
7145
7384
  const q = (context.userQuery || "").toLowerCase().trim();
7146
- const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
7147
- const isNonVisual = !/\b(compare|versus|vs\.?|trend|monthly|revenue|best|top 10|chart|table|list|grid)\b/i.test(q);
7385
+ const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
7386
+ const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
7148
7387
  if (isFactQuery && isNonVisual) {
7149
7388
  return {
7150
7389
  renderType: "text",
@@ -7209,11 +7448,11 @@ var Rule4AnalyticalRule = class {
7209
7448
  const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
7210
7449
  if (!isAnalytical) return null;
7211
7450
  let chartType = "bar";
7212
- if (intent === "trend_analysis" || /\b(monthly|yearly|weekly|growth|over time|trend|timeline|historical)\b/i.test(q)) {
7451
+ if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
7213
7452
  chartType = "line";
7214
- } else if (/\b(share|percentage|percent|breakup|breakdown|composition|pie|donut|proportion)\b/i.test(q)) {
7453
+ } else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
7215
7454
  chartType = "pie";
7216
- } else if (/\b(correlation|scatter|relationship|relation|impact of|depends on)\b/i.test(q)) {
7455
+ } else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
7217
7456
  chartType = "scatter";
7218
7457
  } else {
7219
7458
  chartType = "bar";
@@ -7976,12 +8215,20 @@ ${schemaProfileText}` : ""}`;
7976
8215
  var _a3;
7977
8216
  return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
7978
8217
  }).filter(Boolean))) : this.detectCategories(data);
7979
- if (categories.length === 0) return null;
8218
+ if (categories.length <= 1) return null;
7980
8219
  const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
8220
+ const hasInventoryData = data.some((item) => {
8221
+ const meta = item.metadata || {};
8222
+ return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
8223
+ });
7981
8224
  const pieData = Object.entries(categoryData).map(([label, count]) => {
8225
+ if (!hasInventoryData) {
8226
+ return { label, value: count };
8227
+ }
7982
8228
  const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
7983
8229
  return { label, value: count, inStockCount, outOfStockCount };
7984
8230
  });
8231
+ if (pieData.length <= 1) return null;
7985
8232
  return {
7986
8233
  type: "pie_chart",
7987
8234
  title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
@@ -8288,41 +8535,34 @@ ${schemaProfileText}` : ""}`;
8288
8535
  }
8289
8536
  static _productQueryTest(query) {
8290
8537
  const q = query.toLowerCase();
8291
- const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
8538
+ const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
8539
+ if (financialTerms.test(q)) {
8540
+ return false;
8541
+ }
8542
+ const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
8292
8543
  const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
8293
8544
  const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
8294
8545
  return productTerms && productAction && !nonProductEntityTerms;
8295
8546
  }
8296
8547
  static isProductData(item) {
8297
8548
  if (!item) return false;
8298
- const content = (item.content || "").toLowerCase();
8299
- const productKeywords = [
8300
- "product",
8301
- "price",
8302
- "stock",
8303
- "sku",
8304
- "brand",
8305
- "msrp",
8306
- "inventory",
8307
- "buy",
8308
- "shop",
8309
- "beauty",
8310
- "cosmetic",
8311
- "facial",
8312
- "cream",
8313
- "serum",
8314
- "mask",
8315
- "makeup",
8316
- "fragrance"
8317
- ];
8318
- const hasKeywords = productKeywords.some((kw) => content.includes(kw));
8319
8549
  const metadata = item.metadata || {};
8320
- const hasMetadataKey = Object.keys(metadata).some((k) => {
8321
- const val = metadata[k];
8322
- return ["price", "product", "sku", "brand", "cost"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
8323
- });
8324
- const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
8325
- return hasKeywords || hasMetadataKey || hasPricePattern;
8550
+ const domain = String(metadata.domain || "").toLowerCase();
8551
+ if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
8552
+ return false;
8553
+ }
8554
+ if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
8555
+ return true;
8556
+ }
8557
+ const content = (item.content || "").toLowerCase();
8558
+ const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
8559
+ if (financialTerms.test(content)) {
8560
+ return false;
8561
+ }
8562
+ const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
8563
+ const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
8564
+ const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
8565
+ return hasSku || hasProductIdentity && hasPrice;
8326
8566
  }
8327
8567
  static isTimeSeriesData(item) {
8328
8568
  const metadata = item.metadata || {};
@@ -8341,10 +8581,31 @@ ${schemaProfileText}` : ""}`;
8341
8581
  return fieldCount.size > 2;
8342
8582
  }
8343
8583
  static profileData(data) {
8584
+ const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
8585
+ "filetype",
8586
+ "mimetype",
8587
+ "filename",
8588
+ "docid",
8589
+ "uploadedat",
8590
+ "dimension",
8591
+ "chunkindex",
8592
+ "totalchunks",
8593
+ "charactercount",
8594
+ "workspaceid",
8595
+ "pineconenamespace",
8596
+ "tier",
8597
+ "custommetadata",
8598
+ "csvheaders",
8599
+ "headers",
8600
+ "filesize",
8601
+ "content",
8602
+ "id"
8603
+ ]);
8344
8604
  const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
8345
8605
  var _a2, _b;
8346
8606
  const fields2 = {};
8347
8607
  Object.entries(item.metadata || {}).forEach(([key, value]) => {
8608
+ if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
8348
8609
  const primitive = this.toPrimitive(value);
8349
8610
  if (primitive !== null) fields2[key] = primitive;
8350
8611
  });
@@ -9298,27 +9559,74 @@ ${m.content}`).join("\n\n---\n\n");
9298
9559
  * Ingest documents with automatic chunking, embedding, and batch upsert.
9299
9560
  */
9300
9561
  async ingest(documents, namespace) {
9562
+ var _a2;
9301
9563
  await this.initialize();
9302
9564
  const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
9303
9565
  const results = [];
9566
+ if (!documents || documents.length === 0) return results;
9567
+ const docChunksMap = [];
9568
+ const allChunks = [];
9304
9569
  for (const doc of documents) {
9305
9570
  try {
9306
9571
  const chunks = await this.prepareChunks(doc);
9307
- const vectors = await this.processEmbeddings(chunks);
9572
+ docChunksMap.push({ doc, chunks });
9573
+ allChunks.push(...chunks);
9574
+ } catch (error) {
9575
+ Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
9576
+ results.push({ docId: doc.docId, chunksIngested: 0 });
9577
+ }
9578
+ }
9579
+ if (allChunks.length === 0) {
9580
+ return results;
9581
+ }
9582
+ try {
9583
+ const allVectors = await this.processEmbeddings(allChunks);
9584
+ let vectorOffset = 0;
9585
+ const allUpsertDocs = [];
9586
+ const docSuccessCount = /* @__PURE__ */ new Map();
9587
+ for (const { doc, chunks } of docChunksMap) {
9588
+ const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
9589
+ vectorOffset += chunks.length;
9308
9590
  const upsertDocs = chunks.map((chunk, i) => ({
9309
9591
  id: chunk.id,
9310
- vector: vectors[i],
9592
+ vector: docVectors[i],
9311
9593
  content: chunk.content,
9312
9594
  metadata: chunk.metadata
9313
9595
  }));
9314
- const totalProcessed = await this.processUpserts(upsertDocs, ns);
9315
- results.push({ docId: doc.docId, chunksIngested: totalProcessed });
9596
+ allUpsertDocs.push(...upsertDocs);
9597
+ docSuccessCount.set(doc.docId, chunks.length);
9598
+ }
9599
+ const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
9600
+ for (const { doc, chunks } of docChunksMap) {
9601
+ results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
9316
9602
  if (this.graphDB && this.entityExtractor) {
9317
- await this.processGraphIngestion(doc.docId, chunks);
9603
+ this.processGraphIngestion(doc.docId, chunks).catch((err) => {
9604
+ Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
9605
+ });
9606
+ }
9607
+ }
9608
+ } catch (batchErr) {
9609
+ Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
9610
+ for (const { doc, chunks } of docChunksMap) {
9611
+ try {
9612
+ const vectors = await this.processEmbeddings(chunks);
9613
+ const upsertDocs = chunks.map((chunk, i) => ({
9614
+ id: chunk.id,
9615
+ vector: vectors[i],
9616
+ content: chunk.content,
9617
+ metadata: chunk.metadata
9618
+ }));
9619
+ const totalProcessed = await this.processUpserts(upsertDocs, ns);
9620
+ results.push({ docId: doc.docId, chunksIngested: totalProcessed });
9621
+ if (this.graphDB && this.entityExtractor) {
9622
+ this.processGraphIngestion(doc.docId, chunks).catch((err) => {
9623
+ Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
9624
+ });
9625
+ }
9626
+ } catch (singleErr) {
9627
+ Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
9628
+ results.push({ docId: doc.docId, chunksIngested: 0 });
9318
9629
  }
9319
- } catch (error) {
9320
- Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
9321
- results.push({ docId: doc.docId, chunksIngested: 0 });
9322
9630
  }
9323
9631
  }
9324
9632
  return results;
@@ -9338,19 +9646,40 @@ ${m.content}`).join("\n\n---\n\n");
9338
9646
  }
9339
9647
  /**
9340
9648
  * Step 2: Generate embeddings for chunks with retry logic.
9341
- * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
9649
+ * Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
9342
9650
  */
9343
9651
  async processEmbeddings(chunks) {
9344
- const embedBatchOptions = {
9345
- batchSize: 50,
9346
- maxRetries: 3,
9347
- initialDelayMs: 100
9348
- };
9349
- const vectors = await BatchProcessor.mapWithRetry(
9350
- chunks.map((c) => c.content),
9652
+ if (chunks.length === 0) return [];
9653
+ const texts = chunks.map((c) => c.content);
9654
+ if (typeof this.embeddingProvider.batchEmbed === "function") {
9655
+ try {
9656
+ const batchSize = 50;
9657
+ const allVectors = [];
9658
+ for (let i = 0; i < texts.length; i += batchSize) {
9659
+ const slice = texts.slice(i, i + batchSize);
9660
+ const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
9661
+ allVectors.push(...batchVectors);
9662
+ }
9663
+ if (allVectors.length === chunks.length) {
9664
+ return allVectors;
9665
+ }
9666
+ } catch (err) {
9667
+ Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
9668
+ error: err instanceof Error ? err.message : String(err)
9669
+ });
9670
+ }
9671
+ }
9672
+ const concurrentResult = await BatchProcessor.processConcurrent(
9673
+ texts,
9351
9674
  (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
9352
- embedBatchOptions
9675
+ 10,
9676
+ {
9677
+ maxRetries: 3,
9678
+ initialDelayMs: 100,
9679
+ throwOnPartialFailure: false
9680
+ }
9353
9681
  );
9682
+ const vectors = concurrentResult.results;
9354
9683
  if (vectors.length !== chunks.length) {
9355
9684
  throw new EmbeddingFailedException(
9356
9685
  `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
@@ -9377,25 +9706,24 @@ ${m.content}`).join("\n\n---\n\n");
9377
9706
  }
9378
9707
  /** Step 4: Optional graph-based entity extraction and ingestion. */
9379
9708
  async processGraphIngestion(docId, chunks) {
9380
- const extractionOptions = {
9381
- batchSize: 2,
9382
- maxRetries: 1,
9383
- initialDelayMs: 500
9384
- };
9385
- await BatchProcessor.processBatch(
9709
+ const maxConcurrency = Math.min(3, chunks.length);
9710
+ await BatchProcessor.processConcurrent(
9386
9711
  chunks,
9387
- async (batch) => {
9388
- for (const chunk of batch) {
9389
- try {
9390
- const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
9391
- if (nodes.length > 0) await this.graphDB.addNodes(nodes);
9392
- if (edges.length > 0) await this.graphDB.addEdges(edges);
9393
- } catch (err) {
9394
- Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
9395
- }
9712
+ async (chunk) => {
9713
+ try {
9714
+ const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
9715
+ if (nodes.length > 0) await this.graphDB.addNodes(nodes);
9716
+ if (edges.length > 0) await this.graphDB.addEdges(edges);
9717
+ } catch (err) {
9718
+ Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
9396
9719
  }
9397
9720
  },
9398
- extractionOptions
9721
+ maxConcurrency,
9722
+ {
9723
+ maxRetries: 1,
9724
+ initialDelayMs: 200,
9725
+ throwOnPartialFailure: false
9726
+ }
9399
9727
  );
9400
9728
  }
9401
9729
  async runNormalQuery(question, history = [], namespace) {
@@ -9520,7 +9848,7 @@ ${m.content}`).join("\n\n---\n\n");
9520
9848
  */
9521
9849
  askStreamInternal(_0) {
9522
9850
  return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
9523
- var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
9851
+ var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
9524
9852
  yield new __await(this.initialize());
9525
9853
  const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
9526
9854
  const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
@@ -9528,29 +9856,35 @@ ${m.content}`).join("\n\n---\n\n");
9528
9856
  const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
9529
9857
  const requestStart = performance.now();
9530
9858
  try {
9531
- let searchQuery = question;
9859
+ let effectiveQuestion = question;
9860
+ let promptOptResult;
9861
+ if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
9862
+ promptOptResult = PromptOptimizer.optimize(question, history);
9863
+ effectiveQuestion = promptOptResult.optimizedPrompt;
9864
+ }
9865
+ let searchQuery = effectiveQuestion;
9532
9866
  let rewrittenQuery;
9533
- if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
9534
- searchQuery = yield new __await(this.rewriteQuery(question, history));
9535
- rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
9867
+ if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
9868
+ searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
9869
+ rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
9536
9870
  }
9537
- const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
9538
- const filter = QueryProcessor.buildQueryFilter(question, hints);
9539
- const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
9871
+ const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
9872
+ const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
9873
+ const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
9540
9874
  if (numericPredicates.length > 0) {
9541
9875
  filter.__numericPredicates = numericPredicates;
9542
9876
  }
9543
9877
  const embedStart = performance.now();
9544
9878
  const cacheKey = `${ns}::${searchQuery}`;
9545
9879
  const cachedVector = this.embeddingCache.get(cacheKey);
9546
- const arch = (_h = this.config.rag) == null ? void 0 : _h.architecture;
9547
- const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_i = this.config.rag) == null ? void 0 : _i.useGraphRetrieval);
9880
+ const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
9881
+ const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
9548
9882
  const [strategyResult, embeddedVector] = yield new __await(Promise.all([
9549
9883
  QueryProcessor.determineRetrievalStrategy(
9550
9884
  searchQuery,
9551
9885
  useGraph ? this.llmRouter.get("fast") : void 0,
9552
- (_j = this.config.rag) == null ? void 0 : _j.graphKeywords,
9553
- (_k = this.config.rag) == null ? void 0 : _k.vectorKeywords
9886
+ (_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
9887
+ (_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
9554
9888
  ),
9555
9889
  // Embed immediately regardless of strategy — costs nothing if cached.
9556
9890
  // If strategy turns out to be 'graph'-only we just won't use the vector.
@@ -9560,7 +9894,7 @@ ${m.content}`).join("\n\n---\n\n");
9560
9894
  if (!cachedVector && queryVector.length > 0) {
9561
9895
  this.embeddingCache.set(cacheKey, queryVector);
9562
9896
  }
9563
- const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_l = this.config.rag) == null ? void 0 : _l.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
9897
+ const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
9564
9898
  const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
9565
9899
  const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
9566
9900
  const wantsExhaustiveList = true;
@@ -9577,7 +9911,7 @@ ${m.content}`).join("\n\n---\n\n");
9577
9911
  return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
9578
9912
  });
9579
9913
  const rerankLimit = Math.max(retrievalLimit, fullSources.length);
9580
- const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_m = this.config.rag) == null ? void 0 : _m.useReranking));
9914
+ const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
9581
9915
  if (!hasNumericPredicates && useReranking) {
9582
9916
  fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
9583
9917
  }
@@ -9605,7 +9939,7 @@ ${graphContext}
9605
9939
  VECTOR CONTEXT:
9606
9940
  ${context}`;
9607
9941
  }
9608
- if ((_o = (_n = this.config.rag) == null ? void 0 : _n.cag) == null ? void 0 : _o.enabled) {
9942
+ if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
9609
9943
  if (!this.coldContexts.has(ns)) {
9610
9944
  yield new __await(this.loadColdContext(ns));
9611
9945
  }
@@ -9637,31 +9971,31 @@ ${context}`;
9637
9971
  SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
9638
9972
  });
9639
9973
  }
9640
- const isProductQ = UITransformer.isProductQueryPublic(question);
9974
+ const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
9641
9975
  const uiTransformationPromise = isProductQ ? Promise.resolve(
9642
9976
  UITransformer.transform(
9643
- question,
9977
+ effectiveQuestion,
9644
9978
  sources,
9645
9979
  this.config,
9646
9980
  cachedSchema,
9647
9981
  { visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
9648
9982
  )
9649
9983
  ) : this.generateUiTransformation(
9650
- question,
9984
+ effectiveQuestion,
9651
9985
  sources,
9652
9986
  cachedSchema,
9653
9987
  hasNumericPredicates
9654
9988
  ).catch((uiError) => {
9655
9989
  Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
9656
- return UITransformer.transform(question, sources, this.config, cachedSchema);
9990
+ return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
9657
9991
  });
9658
9992
  let hallucinationScoringPromise = Promise.resolve(void 0);
9659
9993
  const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
9660
9994
  const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
9661
- const isNativeThinking = isClaude37 && (((_p = this.config.llm.options) == null ? void 0 : _p.thinking) === true || ((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === "enabled" || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) !== false);
9995
+ const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
9662
9996
  const modelNameLower = this.config.llm.model.toLowerCase();
9663
9997
  const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
9664
- const isSimulatedThinking = !isNativeThinking && (((_s = this.config.llm.options) == null ? void 0 : _s.thinking) === true || ((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === "enabled" || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) !== false);
9998
+ const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
9665
9999
  let finalRestrictionSuffix = restrictionSuffix;
9666
10000
  if (isSimulatedThinking) {
9667
10001
  finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
@@ -9670,9 +10004,9 @@ ${context}`;
9670
10004
  role: m.role || "user",
9671
10005
  content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
9672
10006
  }));
9673
- const userQuestion = { role: "user", content: question + finalRestrictionSuffix };
10007
+ const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
9674
10008
  const messages = [...hardenedHistory, userQuestion];
9675
- const systemPrompt = (_v = this.config.llm.systemPrompt) != null ? _v : "";
10009
+ const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
9676
10010
  const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
9677
10011
  let fullReply = "";
9678
10012
  let thinkingText = "";
@@ -9783,7 +10117,7 @@ ${context}`;
9783
10117
  }
9784
10118
  yield fullReply;
9785
10119
  }
9786
- const runHallucination = ((_w = this.config.llm.options) == null ? void 0 : _w.hallucinationScoring) === true;
10120
+ const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
9787
10121
  if (runHallucination) {
9788
10122
  hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
9789
10123
  }
@@ -9805,7 +10139,7 @@ ${context}`;
9805
10139
  totalTokens: promptTokens + completionTokens,
9806
10140
  estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
9807
10141
  };
9808
- const awaitHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.awaitHallucination) === true;
10142
+ const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
9809
10143
  const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
9810
10144
  uiTransformationPromise,
9811
10145
  awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
@@ -9813,9 +10147,9 @@ ${context}`;
9813
10147
  const buildTrace = (hScore) => __spreadValues({
9814
10148
  requestId,
9815
10149
  query: question,
9816
- rewrittenQuery,
10150
+ rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
9817
10151
  systemPrompt,
9818
- userPrompt: question + finalRestrictionSuffix,
10152
+ userPrompt: effectiveQuestion + finalRestrictionSuffix,
9819
10153
  chunks: (sources || []).filter(Boolean).map((s) => {
9820
10154
  var _a3, _b2;
9821
10155
  return {
@@ -9834,10 +10168,10 @@ ${context}`;
9834
10168
  hallucinationReason: hScore.reason
9835
10169
  } : {});
9836
10170
  const trace = buildTrace(hallucinationResult);
9837
- const isTelemetryActive = (_z = (_y = this.config.telemetry) == null ? void 0 : _y.enabled) != null ? _z : Boolean(this.config.licenseKey);
10171
+ const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
9838
10172
  if (isTelemetryActive) {
9839
10173
  const defaultUrl = "https://www.retrivora.com/api/telemetry";
9840
- const telemetryUrl = ((_A = this.config.telemetry) == null ? void 0 : _A.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
10174
+ const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
9841
10175
  const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
9842
10176
  (async () => {
9843
10177
  var _a3, _b2, _c2, _d2, _e2;
@@ -10412,13 +10746,81 @@ var VectorPlugin = class {
10412
10746
 
10413
10747
  // src/utils/DocumentParser.ts
10414
10748
  var DocumentParser = class {
10749
+ /**
10750
+ * Fast Magic-Bytes sniffer for instant file format identification (<1ms).
10751
+ */
10752
+ static sniffMagicBytes(buffer) {
10753
+ if (!buffer || buffer.length < 4) return null;
10754
+ if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
10755
+ return "pdf";
10756
+ }
10757
+ if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
10758
+ return "zip";
10759
+ }
10760
+ if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
10761
+ return "ole2";
10762
+ }
10763
+ const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
10764
+ if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
10765
+ return "json";
10766
+ }
10767
+ return "text";
10768
+ }
10769
+ /**
10770
+ * Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
10771
+ * Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
10772
+ */
10773
+ static formatSheetToContextualText(sheet, sheetName, XLSX) {
10774
+ const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
10775
+ if (!jsonRows || jsonRows.length === 0) {
10776
+ const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
10777
+ return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
10778
+ ${csv.trim()}` : "";
10779
+ }
10780
+ const lines = [`--- Sheet: ${sheetName} ---`];
10781
+ let currentHeader = [];
10782
+ for (let r = 0; r < jsonRows.length; r++) {
10783
+ const row = jsonRows[r];
10784
+ if (!row || row.length === 0) continue;
10785
+ const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
10786
+ if (cells.length === 0) continue;
10787
+ if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
10788
+ lines.push(`
10789
+ ## ${cells.join(" ")}`);
10790
+ continue;
10791
+ }
10792
+ const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
10793
+ if (isHeaderCandidate && cells.length >= 2) {
10794
+ currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
10795
+ lines.push(`| ${cells.join(" | ")} |`);
10796
+ lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
10797
+ continue;
10798
+ }
10799
+ if (currentHeader.length > 0 && cells.length > 1) {
10800
+ const fieldPairs = [];
10801
+ for (let c = 0; c < row.length; c++) {
10802
+ const h = currentHeader[c];
10803
+ const val = row[c];
10804
+ if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
10805
+ fieldPairs.push(`${h}: ${String(val).trim()}`);
10806
+ }
10807
+ }
10808
+ if (fieldPairs.length > 0) {
10809
+ lines.push(fieldPairs.join(" | "));
10810
+ continue;
10811
+ }
10812
+ }
10813
+ lines.push(cells.join(", "));
10814
+ }
10815
+ return lines.join("\n");
10816
+ }
10415
10817
  /**
10416
10818
  * Extract text from a File or Buffer based on its type.
10417
10819
  */
10418
- static async parse(file, fileName, mimeType) {
10820
+ static async parse(file, fileName, mimeType = "") {
10419
10821
  var _a2;
10420
10822
  const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
10421
- if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
10823
+ if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
10422
10824
  return this.readAsText(file);
10423
10825
  }
10424
10826
  if (extension === "json" || mimeType === "application/json") {
@@ -10430,38 +10832,115 @@ var DocumentParser = class {
10430
10832
  return text;
10431
10833
  }
10432
10834
  }
10433
- if (extension === "csv" || mimeType === "text/csv") {
10835
+ if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
10836
+ return this.readAsText(file);
10837
+ }
10838
+ if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
10434
10839
  return this.readAsText(file);
10435
10840
  }
10436
- if (extension === "xlsx" || extension === "xls" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
10841
+ if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
10437
10842
  try {
10438
- const ExcelJS = await import("exceljs");
10843
+ const JSZip = (await import("jszip")).default || await import("jszip");
10439
10844
  const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10440
- const workbook = new ExcelJS.Workbook();
10441
- await workbook.xlsx.load(buffer);
10442
- const sheetsText = [];
10443
- workbook.eachSheet((sheet, sheetId) => {
10444
- const csvLines = [];
10445
- sheet.eachRow((row, rowNumber) => {
10446
- const values = row.values;
10447
- if (Array.isArray(values)) {
10448
- csvLines.push(values.slice(1).join(","));
10845
+ const zip = await JSZip.loadAsync(buffer);
10846
+ const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
10847
+ var _a3, _b;
10848
+ const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
10849
+ const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
10850
+ return numA - numB;
10851
+ });
10852
+ if (slideFiles.length > 0) {
10853
+ const slidesText = [];
10854
+ for (let i = 0; i < slideFiles.length; i++) {
10855
+ const slideXml = await zip.files[slideFiles[i]].async("text");
10856
+ const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
10857
+ if (textMatches && textMatches.length > 0) {
10858
+ const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
10859
+ if (slideContent.trim()) {
10860
+ slidesText.push(`--- Slide ${i + 1} ---
10861
+ ${slideContent.trim()}`);
10862
+ }
10449
10863
  }
10450
- });
10451
- const csv = csvLines.join("\n");
10452
- if (csv && csv.trim()) {
10453
- sheetsText.push(`--- Sheet: ${sheet.name} ---
10454
- ${csv.trim()}`);
10455
10864
  }
10456
- });
10457
- return sheetsText.join("\n\n") || `[Empty Excel Workbook: ${fileName}]`;
10458
- } catch (e) {
10865
+ if (slidesText.length > 0) {
10866
+ return slidesText.join("\n\n");
10867
+ }
10868
+ }
10869
+ if (zip.files["content.xml"]) {
10870
+ const contentXml = await zip.files["content.xml"].async("text");
10871
+ const cleanText = this.stripXmlTags(contentXml);
10872
+ if (cleanText.trim()) {
10873
+ return cleanText.trim();
10874
+ }
10875
+ }
10876
+ const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
10877
+ const extractedSegments = [];
10878
+ for (const xf of allXmlFiles) {
10879
+ const xml = await zip.files[xf].async("text");
10880
+ const stripped = this.stripXmlTags(xml);
10881
+ if (stripped.length > 10) extractedSegments.push(stripped);
10882
+ }
10883
+ if (extractedSegments.length > 0) {
10884
+ return extractedSegments.join("\n\n");
10885
+ }
10886
+ return `[PowerPoint Presentation: ${fileName}]`;
10887
+ } catch (err) {
10888
+ console.warn("[DocumentParser] PowerPoint parsing failed:", err);
10459
10889
  try {
10460
10890
  return await this.readAsText(file);
10461
- } catch (e2) {
10462
- return `[Excel File Content: ${fileName}]`;
10891
+ } catch (e) {
10892
+ return `[PowerPoint Presentation File: ${fileName}]`;
10893
+ }
10894
+ }
10895
+ }
10896
+ if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
10897
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10898
+ try {
10899
+ const XLSX = await import("xlsx");
10900
+ const workbook = XLSX.read(buffer, { type: "buffer" });
10901
+ const sheetsText = [];
10902
+ for (const sheetName of workbook.SheetNames) {
10903
+ const sheet = workbook.Sheets[sheetName];
10904
+ const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
10905
+ if (formatted && formatted.trim()) {
10906
+ sheetsText.push(formatted.trim());
10907
+ }
10908
+ }
10909
+ if (sheetsText.length > 0) {
10910
+ return sheetsText.join("\n\n");
10463
10911
  }
10912
+ } catch (xlsxErr) {
10913
+ console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
10464
10914
  }
10915
+ const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
10916
+ if (isZip) {
10917
+ try {
10918
+ const ExcelJS = await import("exceljs");
10919
+ const workbook = new ExcelJS.Workbook();
10920
+ await workbook.xlsx.load(buffer);
10921
+ const sheetsText = [];
10922
+ workbook.eachSheet((sheet) => {
10923
+ const csvLines = [];
10924
+ sheet.eachRow((row) => {
10925
+ const values = row.values;
10926
+ if (Array.isArray(values)) {
10927
+ csvLines.push(values.slice(1).join(","));
10928
+ }
10929
+ });
10930
+ const csv = csvLines.join("\n");
10931
+ if (csv && csv.trim()) {
10932
+ sheetsText.push(`--- Sheet: ${sheet.name} ---
10933
+ ${csv.trim()}`);
10934
+ }
10935
+ });
10936
+ if (sheetsText.length > 0) {
10937
+ return sheetsText.join("\n\n");
10938
+ }
10939
+ } catch (err) {
10940
+ console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
10941
+ }
10942
+ }
10943
+ return this.parseLegacyExcelOrFallback(buffer, fileName);
10465
10944
  }
10466
10945
  if (extension === "pdf" || mimeType === "application/pdf") {
10467
10946
  try {
@@ -10475,15 +10954,48 @@ ${csv.trim()}`);
10475
10954
  return `[PDF Parsing Error for ${fileName}]`;
10476
10955
  }
10477
10956
  }
10478
- if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
10957
+ if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
10479
10958
  try {
10480
10959
  const mammoth = await import("mammoth");
10481
10960
  const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10482
10961
  const result = await mammoth.extractRawText({ buffer });
10483
- return result.value;
10962
+ if (result.value && result.value.trim()) {
10963
+ return result.value;
10964
+ }
10965
+ } catch (err) {
10966
+ console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
10967
+ }
10968
+ try {
10969
+ const raw = await this.readAsText(file);
10970
+ if (extension === "rtf" || raw.startsWith("{\\rtf")) {
10971
+ return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
10972
+ }
10973
+ return raw;
10974
+ } catch (e) {
10975
+ return `[Word Document File: ${fileName}]`;
10976
+ }
10977
+ }
10978
+ if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
10979
+ const raw = await this.readAsText(file);
10980
+ return this.stripHtmlAndXml(raw);
10981
+ }
10982
+ if (extension === "epub" || mimeType.includes("epub")) {
10983
+ try {
10984
+ const JSZip = (await import("jszip")).default || await import("jszip");
10985
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10986
+ const zip = await JSZip.loadAsync(buffer);
10987
+ const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
10988
+ const chapters = [];
10989
+ for (const hf of htmlFiles) {
10990
+ const content = await zip.files[hf].async("text");
10991
+ const clean = this.stripHtmlAndXml(content);
10992
+ if (clean.length > 50) chapters.push(clean);
10993
+ }
10994
+ if (chapters.length > 0) {
10995
+ return chapters.join("\n\n---\n\n");
10996
+ }
10484
10997
  } catch (err) {
10485
- console.warn('[DocumentParser] Word parsing failed. Make sure "mammoth" is installed.', err);
10486
- return `[Word Parsing Error for ${fileName}]`;
10998
+ console.warn("[DocumentParser] EPUB parsing failed:", err);
10487
10999
  }
10488
11000
  }
10489
11001
  try {
@@ -10492,6 +11004,237 @@ ${csv.trim()}`);
10492
11004
  throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
10493
11005
  }
10494
11006
  }
11007
+ /**
11008
+ * Helper to strip XML tags and normalize inner text content.
11009
+ */
11010
+ static stripXmlTags(xml) {
11011
+ return xml.replace(/<[^>]+>/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/\s+/g, " ").trim();
11012
+ }
11013
+ /**
11014
+ * Helper to strip HTML / XML tags while removing script and style blocks completely.
11015
+ */
11016
+ static stripHtmlAndXml(html) {
11017
+ return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
11018
+ }
11019
+ /**
11020
+ * Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
11021
+ */
11022
+ static parseLegacyExcelOrFallback(buffer, fileName) {
11023
+ const rawStr = buffer.toString("utf-8");
11024
+ if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
11025
+ const rows = [];
11026
+ const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
11027
+ if (rowMatches && rowMatches.length > 0) {
11028
+ for (const rowHtml of rowMatches) {
11029
+ const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
11030
+ if (cells && cells.length > 0) {
11031
+ const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
11032
+ if (rowText) rows.push(rowText);
11033
+ }
11034
+ }
11035
+ }
11036
+ if (rows.length > 0) {
11037
+ return rows.join("\n");
11038
+ }
11039
+ const clean = this.stripHtmlAndXml(rawStr);
11040
+ if (clean.length > 10) return clean;
11041
+ }
11042
+ if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
11043
+ const rows = [];
11044
+ const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
11045
+ if (rowMatches && rowMatches.length > 0) {
11046
+ for (const rowXml of rowMatches) {
11047
+ const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
11048
+ if (dataMatches && dataMatches.length > 0) {
11049
+ const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
11050
+ if (rowText) rows.push(rowText);
11051
+ }
11052
+ }
11053
+ }
11054
+ if (rows.length > 0) {
11055
+ return rows.join("\n");
11056
+ }
11057
+ }
11058
+ const extractedStrings = [];
11059
+ const ignoredOleMarkers = /* @__PURE__ */ new Set([
11060
+ "root entry",
11061
+ "workbook",
11062
+ "compobj",
11063
+ "summaryinformation",
11064
+ "documentsummaryinformation",
11065
+ "microsoft excel",
11066
+ "worksheet",
11067
+ "biff8",
11068
+ "calcpr"
11069
+ ]);
11070
+ let currentUtf16 = "";
11071
+ for (let i = 0; i < buffer.length - 1; i += 2) {
11072
+ const charCode = buffer[i];
11073
+ const zeroByte = buffer[i + 1];
11074
+ if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
11075
+ currentUtf16 += String.fromCharCode(charCode);
11076
+ } else {
11077
+ if (currentUtf16.length >= 2) {
11078
+ const trimmed = currentUtf16.trim();
11079
+ if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
11080
+ extractedStrings.push(trimmed);
11081
+ }
11082
+ }
11083
+ currentUtf16 = "";
11084
+ }
11085
+ }
11086
+ let currentAscii = "";
11087
+ for (let i = 0; i < buffer.length; i++) {
11088
+ const byte = buffer[i];
11089
+ if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
11090
+ currentAscii += String.fromCharCode(byte);
11091
+ } else {
11092
+ if (currentAscii.length >= 2) {
11093
+ const trimmed = currentAscii.trim();
11094
+ if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
11095
+ extractedStrings.push(trimmed);
11096
+ }
11097
+ }
11098
+ currentAscii = "";
11099
+ }
11100
+ }
11101
+ const uniqueStrings = [];
11102
+ for (const str of extractedStrings) {
11103
+ if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
11104
+ uniqueStrings.push(str);
11105
+ }
11106
+ }
11107
+ if (uniqueStrings.length > 0) {
11108
+ return uniqueStrings.join(" | ");
11109
+ }
11110
+ const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
11111
+ return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
11112
+ }
11113
+ /**
11114
+ * Dynamically infer document domain and schema from extracted text, filename, and format.
11115
+ */
11116
+ static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
11117
+ const lowerText = text.toLowerCase();
11118
+ const lowerName = fileName.toLowerCase();
11119
+ const financialTerms = [
11120
+ "price per share",
11121
+ "share price",
11122
+ "total cost of the deal",
11123
+ "cost of the deal",
11124
+ "lbo",
11125
+ "merger",
11126
+ "valuation",
11127
+ "d/e ratio",
11128
+ "ebit",
11129
+ "ebitda",
11130
+ "fcfe",
11131
+ "fcff",
11132
+ "wacc",
11133
+ "beta",
11134
+ "working capital",
11135
+ "cogs",
11136
+ "cap spending",
11137
+ "depreciation",
11138
+ "cashflow",
11139
+ "cash flow",
11140
+ "net income",
11141
+ "balance sheet",
11142
+ "income statement",
11143
+ "interest rate",
11144
+ "terminal year",
11145
+ "equity",
11146
+ "debt outstanding",
11147
+ "gross margin",
11148
+ "fiscal year",
11149
+ "fy20",
11150
+ "fy21",
11151
+ "fy22",
11152
+ "fy23",
11153
+ "fy24",
11154
+ "fy25",
11155
+ "q1",
11156
+ "q2",
11157
+ "q3",
11158
+ "q4"
11159
+ ];
11160
+ let financialScore = 0;
11161
+ for (const term of financialTerms) {
11162
+ if (lowerText.includes(term) || lowerName.includes(term)) {
11163
+ financialScore += 1;
11164
+ }
11165
+ }
11166
+ const ecommerceTerms = [
11167
+ "sku",
11168
+ "product_id",
11169
+ "product id",
11170
+ "item_id",
11171
+ "add to cart",
11172
+ "buy now",
11173
+ "in stock",
11174
+ "out of stock",
11175
+ "inventory count",
11176
+ "retail price",
11177
+ "unit price",
11178
+ "msrp",
11179
+ "cart",
11180
+ "shop",
11181
+ "catalog",
11182
+ "product_name",
11183
+ "product name",
11184
+ "brand name"
11185
+ ];
11186
+ let ecommerceScore = 0;
11187
+ for (const term of ecommerceTerms) {
11188
+ if (lowerText.includes(term) || lowerName.includes(term)) {
11189
+ ecommerceScore += 1;
11190
+ }
11191
+ }
11192
+ const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
11193
+ let legalScore = 0;
11194
+ for (const term of legalTerms) {
11195
+ if (lowerText.includes(term)) legalScore += 1;
11196
+ }
11197
+ const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
11198
+ let healthScore = 0;
11199
+ for (const term of healthTerms) {
11200
+ if (lowerText.includes(term)) healthScore += 1;
11201
+ }
11202
+ let schema;
11203
+ const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
11204
+ if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
11205
+ const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
11206
+ const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
11207
+ if (headers.length > 1 && lines.length > 1) {
11208
+ const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
11209
+ schema = {};
11210
+ headers.forEach((h, idx) => {
11211
+ const val = sampleRow[idx] || "";
11212
+ if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
11213
+ schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
11214
+ } else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
11215
+ schema[h] = "date";
11216
+ } else {
11217
+ schema[h] = "string";
11218
+ }
11219
+ });
11220
+ }
11221
+ }
11222
+ const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
11223
+ const isCatalog = ecommerceScore >= 2 && !isFinancial;
11224
+ let domain = "general";
11225
+ if (isFinancial) {
11226
+ domain = "finance";
11227
+ } else if (isCatalog) {
11228
+ domain = "ecommerce";
11229
+ } else if (legalScore >= 2) {
11230
+ domain = "legal";
11231
+ } else if (healthScore >= 2) {
11232
+ domain = "healthcare";
11233
+ } else if (schema && Object.keys(schema).length > 0) {
11234
+ domain = "tabular";
11235
+ }
11236
+ return { domain, schema, isCatalog, isFinancial };
11237
+ }
10495
11238
  static async readAsText(file) {
10496
11239
  if (Buffer.isBuffer(file)) {
10497
11240
  return file.toString("utf-8");
@@ -11138,18 +11881,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
11138
11881
  }
11139
11882
  }
11140
11883
  function getAllowedNamespaces(projectId) {
11141
- const trimmed = projectId.trim();
11884
+ const trimmed = (projectId || "default").trim();
11142
11885
  const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
11143
11886
  return new Set([
11144
11887
  trimmed,
11145
11888
  withoutPrefix,
11146
- `retrivora-${withoutPrefix}`
11889
+ `retrivora-${withoutPrefix}`,
11890
+ "default",
11891
+ "retrivora-default"
11147
11892
  ].filter(Boolean));
11148
11893
  }
11149
11894
  function resolveLicensedNamespace(payload, requestedNamespace) {
11895
+ var _a2;
11150
11896
  const allowed = getAllowedNamespaces(payload.projectId);
11151
11897
  const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
11152
- if (requested && !allowed.has(requested)) {
11898
+ const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
11899
+ if (requested && !isDevOrFree && !allowed.has(requested)) {
11153
11900
  return {
11154
11901
  ok: false,
11155
11902
  response: new Response(JSON.stringify({
@@ -11163,7 +11910,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
11163
11910
  })
11164
11911
  };
11165
11912
  }
11166
- const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
11913
+ const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
11167
11914
  return { ok: true, namespace: requested || defaultNamespace };
11168
11915
  }
11169
11916
  var MAX_MESSAGE_LENGTH = 8e3;
@@ -11607,7 +12354,7 @@ function createIngestHandler(configOrPlugin, options) {
11607
12354
  function createHealthHandler(configOrPlugin, options) {
11608
12355
  const plugin = getOrCreatePlugin(configOrPlugin);
11609
12356
  const onAuthorize = options == null ? void 0 : options.onAuthorize;
11610
- return async function GET(req) {
12357
+ return async function GET(req, context) {
11611
12358
  if (req) {
11612
12359
  const authResult = await checkAuth(req, onAuthorize);
11613
12360
  if (authResult) return authResult;
@@ -11718,9 +12465,10 @@ function createUploadHandler(configOrPlugin, options) {
11718
12465
  }
11719
12466
  const documents = [];
11720
12467
  for (const file of files) {
12468
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
11721
12469
  const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
11722
12470
  if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
11723
- const text = await file.text();
12471
+ const text = buffer.toString("utf-8");
11724
12472
  const Papa = await import("papaparse");
11725
12473
  const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
11726
12474
  if (parsed.data && parsed.data.length > 0) {
@@ -11760,49 +12508,70 @@ function createUploadHandler(configOrPlugin, options) {
11760
12508
  }
11761
12509
  }
11762
12510
  } else if (isExcel) {
11763
- const ExcelJS = await import("exceljs");
11764
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
11765
- const workbook = new ExcelJS.Workbook();
11766
- await workbook.xlsx.load(buffer);
11767
- workbook.eachSheet((sheet, sheetId) => {
11768
- const sheetName = sheet.name;
11769
- let headers = [];
11770
- let i = 0;
11771
- sheet.eachRow((row, rowNumber) => {
11772
- const values = Array.isArray(row.values) ? row.values.slice(1) : [];
11773
- if (rowNumber === 1) {
11774
- headers = values.map((v) => String(v));
11775
- return;
11776
- }
11777
- i++;
11778
- const contentParts = [];
11779
- const rowData = {};
11780
- values.forEach((val, index) => {
11781
- const key = headers[index] || `Column${index + 1}`;
11782
- rowData[key] = val;
11783
- if (key && val !== void 0 && val !== null && String(val).trim()) {
11784
- contentParts.push(`${key}: ${val}`);
11785
- }
11786
- });
11787
- if (contentParts.length > 0) {
11788
- documents.push({
11789
- docId: `${file.name}-${sheetName}-row-${i}`,
11790
- content: contentParts.join(", "),
11791
- metadata: __spreadValues(__spreadProps(__spreadValues({
11792
- fileName: file.name,
11793
- sheetName,
11794
- fileSize: file.size,
11795
- fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
11796
- uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
11797
- }, dimension ? { dimension } : {}), {
11798
- headers
11799
- }), rowData)
12511
+ let excelLoaded = false;
12512
+ try {
12513
+ const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
12514
+ if (isZip) {
12515
+ const ExcelJS = await import("exceljs");
12516
+ const workbook = new ExcelJS.Workbook();
12517
+ await workbook.xlsx.load(buffer);
12518
+ workbook.eachSheet((sheet, sheetId) => {
12519
+ const sheetName = sheet.name;
12520
+ let headers = [];
12521
+ let i = 0;
12522
+ sheet.eachRow((row, rowNumber) => {
12523
+ const values = Array.isArray(row.values) ? row.values.slice(1) : [];
12524
+ if (rowNumber === 1) {
12525
+ headers = values.map((v) => String(v));
12526
+ return;
12527
+ }
12528
+ i++;
12529
+ const contentParts = [];
12530
+ const rowData = {};
12531
+ values.forEach((val, index) => {
12532
+ const key = headers[index] || `Column${index + 1}`;
12533
+ rowData[key] = val;
12534
+ if (key && val !== void 0 && val !== null && String(val).trim()) {
12535
+ contentParts.push(`${key}: ${val}`);
12536
+ }
12537
+ });
12538
+ if (contentParts.length > 0) {
12539
+ documents.push({
12540
+ docId: `${file.name}-${sheetName}-row-${i}`,
12541
+ content: contentParts.join(", "),
12542
+ metadata: __spreadValues(__spreadProps(__spreadValues({
12543
+ fileName: file.name,
12544
+ sheetName,
12545
+ fileSize: file.size,
12546
+ fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
12547
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12548
+ }, dimension ? { dimension } : {}), {
12549
+ headers
12550
+ }), rowData)
12551
+ });
12552
+ }
11800
12553
  });
11801
- }
12554
+ });
12555
+ excelLoaded = documents.length > 0;
12556
+ }
12557
+ } catch (excelErr) {
12558
+ excelLoaded = false;
12559
+ }
12560
+ if (!excelLoaded) {
12561
+ const content = await DocumentParser.parse(buffer, file.name, file.type);
12562
+ documents.push({
12563
+ docId: file.name,
12564
+ content,
12565
+ metadata: __spreadValues({
12566
+ fileName: file.name,
12567
+ fileSize: file.size,
12568
+ fileType: file.type || "application/vnd.ms-excel",
12569
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12570
+ }, dimension ? { dimension } : {})
11802
12571
  });
11803
- });
12572
+ }
11804
12573
  } else {
11805
- const content = await DocumentParser.parse(file, file.name, file.type);
12574
+ const content = await DocumentParser.parse(buffer, file.name, file.type);
11806
12575
  documents.push({
11807
12576
  docId: file.name,
11808
12577
  content,
@@ -11961,7 +12730,7 @@ function createSessionsHandler(configOrPlugin, options) {
11961
12730
  const storage = new DatabaseStorage(plugin.getConfig());
11962
12731
  const onAuthorize = options == null ? void 0 : options.onAuthorize;
11963
12732
  const onResolveScope = options == null ? void 0 : options.onResolveScope;
11964
- return async function GET(req) {
12733
+ return async function GET(req, context) {
11965
12734
  if (req) {
11966
12735
  const authResult = await checkAuth(req, onAuthorize);
11967
12736
  if (authResult) return authResult;