@retrivora-ai/rag-engine 2.5.6 → 2.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/server.mjs CHANGED
@@ -88,8 +88,8 @@ function buildPayload(template, vars) {
88
88
  let raw = template;
89
89
  for (const [key, val] of Object.entries(vars)) {
90
90
  const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
91
- raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
92
- raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
91
+ raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
92
+ raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
93
93
  }
94
94
  try {
95
95
  return JSON.parse(raw);
@@ -326,7 +326,7 @@ var init_package = __esm({
326
326
  "package.json"() {
327
327
  package_default = {
328
328
  name: "@retrivora-ai/rag-engine",
329
- version: "2.5.6",
329
+ version: "2.5.8",
330
330
  description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
331
331
  author: "Abhinav Alkuchi",
332
332
  license: "UNLICENSED",
@@ -429,9 +429,11 @@ var init_package = __esm({
429
429
  "@google/genai": "^0.8.0",
430
430
  "@google/generative-ai": "^0.24.1",
431
431
  "@pinecone-database/pinecone": "^7.2.0",
432
- "@retrivora-ai/billing-guards": "*",
432
+ "@retrivora-ai/billing-guards": "file:../billing-guards",
433
433
  "@types/papaparse": "^5.5.2",
434
+ exceljs: "^4.4.0",
434
435
  "framer-motion": "^13.1.0",
436
+ jszip: "^3.10.1",
435
437
  katex: "^0.18.4",
436
438
  "lucide-react": "^1.8.0",
437
439
  "next-themes": "^0.4.6",
@@ -444,7 +446,7 @@ var init_package = __esm({
444
446
  "rehype-katex": "^7.0.1",
445
447
  "remark-gfm": "^4.0.1",
446
448
  "remark-math": "^6.0.0",
447
- exceljs: "^4.4.0",
449
+ xlsx: "^0.18.5",
448
450
  zod: "^4.4.3"
449
451
  },
450
452
  optionalDependencies: {
@@ -462,6 +464,7 @@ var init_package = __esm({
462
464
  "@tailwindcss/postcss": "^4",
463
465
  "@types/estree": "^1.0.9",
464
466
  "@types/jest": "^30.0.0",
467
+ "@types/jszip": "^3.4.1",
465
468
  "@types/katex": "^0.16.8",
466
469
  "@types/node": "^20",
467
470
  "@types/pdf-parse": "^1.1.5",
@@ -2864,6 +2867,7 @@ var _LicenseVerifier = class _LicenseVerifier {
2864
2867
  try {
2865
2868
  response = await fetch(endpoint, {
2866
2869
  method: "POST",
2870
+ signal: AbortSignal.timeout(3e3),
2867
2871
  headers: {
2868
2872
  "Content-Type": "application/json",
2869
2873
  "x-license-key": licenseKey
@@ -5589,15 +5593,29 @@ Text to extract from:
5589
5593
  const response = await this.llm.chat([
5590
5594
  { role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
5591
5595
  { role: "user", content: prompt }
5592
- ], "", { maxTokens: 4096, temperature: 0.1 });
5596
+ ], "", { maxTokens: 800, temperature: 0.1 });
5593
5597
  try {
5594
- const jsonMatch = response.match(/\{[\s\S]*\}/);
5595
- const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
5596
- return JSON.parse(cleanJson);
5598
+ const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
5599
+ const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
5600
+ const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
5601
+ const jsonMatch = candidate.match(/\{[\s\S]*\}/);
5602
+ const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
5603
+ const parsed = JSON.parse(cleanJson);
5604
+ return {
5605
+ nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
5606
+ edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
5607
+ };
5597
5608
  } catch (e) {
5598
5609
  try {
5599
- const repaired = this.repairTruncatedJson(response);
5600
- if (repaired) return JSON.parse(repaired);
5610
+ const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
5611
+ const repaired = this.repairTruncatedJson(withoutThinking);
5612
+ if (repaired) {
5613
+ const parsed = JSON.parse(repaired);
5614
+ return {
5615
+ nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
5616
+ edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
5617
+ };
5618
+ }
5601
5619
  } catch (e2) {
5602
5620
  }
5603
5621
  console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
@@ -5875,6 +5893,208 @@ ${error instanceof Error ? error.message : String(error)}`
5875
5893
  }
5876
5894
  };
5877
5895
 
5896
+ // src/rag/PromptOptimizer.ts
5897
+ var PromptOptimizer = class {
5898
+ /**
5899
+ * Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
5900
+ */
5901
+ static optimize(rawPrompt, history = [], options = {}) {
5902
+ if (!rawPrompt || typeof rawPrompt !== "string") {
5903
+ return {
5904
+ optimizedPrompt: "",
5905
+ originalPrompt: rawPrompt || "",
5906
+ wasModified: false,
5907
+ transformations: []
5908
+ };
5909
+ }
5910
+ const { resolveHistory = true, expandKeywords = true } = options;
5911
+ const transformations = [];
5912
+ let current = rawPrompt;
5913
+ const cleaned = this.cleanText(current);
5914
+ if (cleaned !== current) {
5915
+ transformations.push("clean_whitespace_and_artifacts");
5916
+ current = cleaned;
5917
+ }
5918
+ const expandedAbbr = this.expandAbbreviations(current);
5919
+ if (expandedAbbr !== current) {
5920
+ transformations.push("expand_abbreviations");
5921
+ current = expandedAbbr;
5922
+ }
5923
+ if (expandKeywords) {
5924
+ const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
5925
+ if (this.TOPIC_EXPANSIONS[trimmedLower]) {
5926
+ current = this.TOPIC_EXPANSIONS[trimmedLower];
5927
+ transformations.push(`topic_expansion_${trimmedLower}`);
5928
+ }
5929
+ }
5930
+ if (resolveHistory && history && history.length > 0) {
5931
+ const resolved = this.resolveCoreferences(current, history);
5932
+ if (resolved !== current) {
5933
+ transformations.push("resolve_history_coreferences");
5934
+ current = resolved;
5935
+ }
5936
+ }
5937
+ if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
5938
+ if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
5939
+ current = `${current.trim()}?`;
5940
+ transformations.push("add_question_mark");
5941
+ }
5942
+ }
5943
+ return {
5944
+ optimizedPrompt: current,
5945
+ originalPrompt: rawPrompt,
5946
+ wasModified: current !== rawPrompt,
5947
+ transformations
5948
+ };
5949
+ }
5950
+ /**
5951
+ * Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
5952
+ */
5953
+ static cleanText(text) {
5954
+ let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
5955
+ const doubleQuotesCount = (clean.match(/"/g) || []).length;
5956
+ if (doubleQuotesCount % 2 !== 0) {
5957
+ clean += '"';
5958
+ }
5959
+ const singleQuotesCount = (clean.match(/'/g) || []).length;
5960
+ if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
5961
+ clean += "'";
5962
+ }
5963
+ const openParen = (clean.match(/\(/g) || []).length;
5964
+ const closeParen = (clean.match(/\)/g) || []).length;
5965
+ if (openParen > closeParen) {
5966
+ clean += ")".repeat(openParen - closeParen);
5967
+ }
5968
+ return clean;
5969
+ }
5970
+ /**
5971
+ * Replaces shorthand chat abbreviations with clear natural language.
5972
+ */
5973
+ static expandAbbreviations(text) {
5974
+ let expanded = text;
5975
+ for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
5976
+ expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
5977
+ }
5978
+ return expanded;
5979
+ }
5980
+ /**
5981
+ * Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
5982
+ */
5983
+ static resolveCoreferences(prompt, history) {
5984
+ const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
5985
+ if (!pronounPattern.test(prompt)) {
5986
+ return prompt;
5987
+ }
5988
+ const recentMessages = [...history].slice(-4).reverse();
5989
+ let detectedEntity = null;
5990
+ for (const msg of recentMessages) {
5991
+ if (!msg.content) continue;
5992
+ const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
5993
+ if (quotedMatch) {
5994
+ detectedEntity = quotedMatch[1];
5995
+ break;
5996
+ }
5997
+ const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
5998
+ if (headingMatch) {
5999
+ detectedEntity = headingMatch[1].trim();
6000
+ break;
6001
+ }
6002
+ const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
6003
+ if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
6004
+ detectedEntity = entityMatches[1];
6005
+ break;
6006
+ }
6007
+ }
6008
+ if (detectedEntity) {
6009
+ let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
6010
+ return resolved;
6011
+ }
6012
+ return prompt;
6013
+ }
6014
+ /**
6015
+ * Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
6016
+ */
6017
+ static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
6018
+ const fastResult = this.optimize(rawPrompt, history);
6019
+ if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
6020
+ return fastResult;
6021
+ }
6022
+ try {
6023
+ const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
6024
+ const prompt = `You are a query optimizer in an enterprise RAG system.
6025
+ Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
6026
+ - Do NOT answer the question.
6027
+ - Resolve any pronouns like "it", "they", "that" using the context.
6028
+ - Keep the user's intent 100% intact.
6029
+ - If the question is already clear, return it as is.
6030
+ - Return ONLY the optimized query string. No quotes or explanations.
6031
+
6032
+ Context:
6033
+ ${historyContext || "No previous context."}
6034
+
6035
+ User Question: "${rawPrompt}"
6036
+
6037
+ Optimized Search Prompt:`;
6038
+ const response = await llmProvider.chat(
6039
+ [{ role: "user", content: prompt }],
6040
+ "",
6041
+ {
6042
+ systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
6043
+ temperature: 0,
6044
+ maxTokens: 100
6045
+ }
6046
+ );
6047
+ const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
6048
+ if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
6049
+ return {
6050
+ optimizedPrompt: cleaned,
6051
+ originalPrompt: rawPrompt,
6052
+ wasModified: cleaned !== rawPrompt,
6053
+ transformations: [...fastResult.transformations, "llm_deep_reformulation"]
6054
+ };
6055
+ }
6056
+ } catch (err) {
6057
+ console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
6058
+ }
6059
+ return fastResult;
6060
+ }
6061
+ };
6062
+ PromptOptimizer.COMMON_ABBREVIATIONS = {
6063
+ "\\bpls\\b": "please",
6064
+ "\\bplz\\b": "please",
6065
+ "\\bthx\\b": "thanks",
6066
+ "\\bty\\b": "thank you",
6067
+ "\\bapprox\\b": "approximately",
6068
+ "\\bw/o\\b": "without",
6069
+ "\\bw/\\b": "with",
6070
+ "\\bdiff\\b": "difference",
6071
+ "\\binfo\\b": "information",
6072
+ "\\bconfig\\b": "configuration",
6073
+ "\\bdoc\\b": "document",
6074
+ "\\bdocs\\b": "documentation",
6075
+ "\\bfeat\\b": "feature",
6076
+ "\\bfeats\\b": "features",
6077
+ "\\bvs\\b": "versus",
6078
+ "\\bu\\b": "you",
6079
+ "\\bur\\b": "your",
6080
+ "\\br\\b": "are"
6081
+ };
6082
+ PromptOptimizer.TOPIC_EXPANSIONS = {
6083
+ "pricing": "What are the available pricing plans, tiers, and cost structures?",
6084
+ "price": "What is the pricing and subscription cost?",
6085
+ "features": "What are the core features, capabilities, and tools provided?",
6086
+ "security": "What are the security, privacy, and compliance standards?",
6087
+ "privacy": "What is the privacy policy and how is data protected?",
6088
+ "refund": "What is the refund and cancellation policy?",
6089
+ "refunds": "What is the refund and cancellation policy?",
6090
+ "contact": "How can I contact customer support or sales?",
6091
+ "support": "What support channels and assistance are available?",
6092
+ "docs": "Where can I find the official documentation and setup guides?",
6093
+ "installation": "How do I install and configure the SDK or service?",
6094
+ "install": "What are the step-by-step installation instructions?",
6095
+ "faq": "What are the frequently asked questions and key answers?"
6096
+ };
6097
+
5878
6098
  // src/core/mcp.ts
5879
6099
  import { spawn } from "child_process";
5880
6100
  var MCPClient = class {
@@ -7098,6 +7318,26 @@ var IntentClassifier = class {
7098
7318
  static extractDataSignals(context) {
7099
7319
  const docs = context.retrievedDocuments || [];
7100
7320
  const rowCount = docs.length;
7321
+ const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
7322
+ "filetype",
7323
+ "mimetype",
7324
+ "filename",
7325
+ "docid",
7326
+ "uploadedat",
7327
+ "dimension",
7328
+ "chunkindex",
7329
+ "totalchunks",
7330
+ "charactercount",
7331
+ "workspaceid",
7332
+ "pineconenamespace",
7333
+ "tier",
7334
+ "custommetadata",
7335
+ "csvheaders",
7336
+ "headers",
7337
+ "filesize",
7338
+ "content",
7339
+ "id"
7340
+ ]);
7101
7341
  let hasNumericFields = false;
7102
7342
  let hasDateFields = false;
7103
7343
  let hasCategoricalFields = false;
@@ -7113,6 +7353,7 @@ var IntentClassifier = class {
7113
7353
  docs.forEach((doc) => {
7114
7354
  const meta = doc.metadata || {};
7115
7355
  Object.entries(meta).forEach(([key, val]) => {
7356
+ if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
7116
7357
  keys.add(key);
7117
7358
  const kLower = key.toLowerCase();
7118
7359
  if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
@@ -7121,7 +7362,7 @@ var IntentClassifier = class {
7121
7362
  if (typeof val === "number") {
7122
7363
  numericKeys.add(key);
7123
7364
  } else if (typeof val === "string") {
7124
- if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
7365
+ if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
7125
7366
  dateKeys.add(key);
7126
7367
  } else if (val.length < 50 && !val.includes("\n")) {
7127
7368
  catKeys.add(key);
@@ -7134,10 +7375,7 @@ var IntentClassifier = class {
7134
7375
  hasCategoricalFields = catKeys.size > 0;
7135
7376
  numericFieldCount = numericKeys.size;
7136
7377
  categoricalFieldCount = catKeys.size;
7137
- if (productKeyMatches >= 2 || docs.some((d) => {
7138
- var _a2, _b;
7139
- return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
7140
- })) {
7378
+ if (productKeyMatches >= 2) {
7141
7379
  isProductLike = true;
7142
7380
  }
7143
7381
  }
@@ -7153,12 +7391,13 @@ var IntentClassifier = class {
7153
7391
  }
7154
7392
  // ─── Intent Heuristic Checkers ─────────────────────────────────────────────
7155
7393
  static isInformationLookup(query) {
7156
- if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(query)) {
7157
- if (!/\b(compare|versus|vs\.?|trend|monthly|revenue|top 10|top selling|best|chart|graph|plot|breakdown|analytics)\b/i.test(query)) {
7394
+ const q = query.toLowerCase().trim();
7395
+ if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
7396
+ if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
7158
7397
  return true;
7159
7398
  }
7160
7399
  }
7161
- if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(query)) {
7400
+ if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
7162
7401
  return true;
7163
7402
  }
7164
7403
  return false;
@@ -7173,13 +7412,13 @@ var IntentClassifier = class {
7173
7412
  return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
7174
7413
  }
7175
7414
  static isTrendQuery(query) {
7176
- return /\b(trend|trends|growth|over time|historical|timeline|monthly|yearly|weekly|daily|revenue growth|sales growth)\b/i.test(query);
7415
+ return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
7177
7416
  }
7178
7417
  static isRankingQuery(query) {
7179
7418
  return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
7180
7419
  }
7181
7420
  static isAnalyticsQuery(query) {
7182
- return /\b(sales|revenue|analytics|insights|distribution|percentage|share|breakdown|breakup|scatter|correlation|metrics|performance|by region|by category)\b/i.test(query);
7421
+ return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
7183
7422
  }
7184
7423
  };
7185
7424
 
@@ -7191,8 +7430,8 @@ var Rule1SpecificInfoRule = class {
7191
7430
  }
7192
7431
  evaluate(context, intent) {
7193
7432
  const q = (context.userQuery || "").toLowerCase().trim();
7194
- const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
7195
- const isNonVisual = !/\b(compare|versus|vs\.?|trend|monthly|revenue|best|top 10|chart|table|list|grid)\b/i.test(q);
7433
+ const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
7434
+ const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
7196
7435
  if (isFactQuery && isNonVisual) {
7197
7436
  return {
7198
7437
  renderType: "text",
@@ -7257,11 +7496,11 @@ var Rule4AnalyticalRule = class {
7257
7496
  const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
7258
7497
  if (!isAnalytical) return null;
7259
7498
  let chartType = "bar";
7260
- if (intent === "trend_analysis" || /\b(monthly|yearly|weekly|growth|over time|trend|timeline|historical)\b/i.test(q)) {
7499
+ if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
7261
7500
  chartType = "line";
7262
- } else if (/\b(share|percentage|percent|breakup|breakdown|composition|pie|donut|proportion)\b/i.test(q)) {
7501
+ } else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
7263
7502
  chartType = "pie";
7264
- } else if (/\b(correlation|scatter|relationship|relation|impact of|depends on)\b/i.test(q)) {
7503
+ } else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
7265
7504
  chartType = "scatter";
7266
7505
  } else {
7267
7506
  chartType = "bar";
@@ -8032,12 +8271,20 @@ ${schemaProfileText}` : ""}`;
8032
8271
  var _a3;
8033
8272
  return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
8034
8273
  }).filter(Boolean))) : this.detectCategories(data);
8035
- if (categories.length === 0) return null;
8274
+ if (categories.length <= 1) return null;
8036
8275
  const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
8276
+ const hasInventoryData = data.some((item) => {
8277
+ const meta = item.metadata || {};
8278
+ return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
8279
+ });
8037
8280
  const pieData = Object.entries(categoryData).map(([label, count]) => {
8281
+ if (!hasInventoryData) {
8282
+ return { label, value: count };
8283
+ }
8038
8284
  const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
8039
8285
  return { label, value: count, inStockCount, outOfStockCount };
8040
8286
  });
8287
+ if (pieData.length <= 1) return null;
8041
8288
  return {
8042
8289
  type: "pie_chart",
8043
8290
  title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
@@ -8344,41 +8591,34 @@ ${schemaProfileText}` : ""}`;
8344
8591
  }
8345
8592
  static _productQueryTest(query) {
8346
8593
  const q = query.toLowerCase();
8347
- const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
8594
+ const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
8595
+ if (financialTerms.test(q)) {
8596
+ return false;
8597
+ }
8598
+ const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
8348
8599
  const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
8349
8600
  const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
8350
8601
  return productTerms && productAction && !nonProductEntityTerms;
8351
8602
  }
8352
8603
  static isProductData(item) {
8353
8604
  if (!item) return false;
8354
- const content = (item.content || "").toLowerCase();
8355
- const productKeywords = [
8356
- "product",
8357
- "price",
8358
- "stock",
8359
- "sku",
8360
- "brand",
8361
- "msrp",
8362
- "inventory",
8363
- "buy",
8364
- "shop",
8365
- "beauty",
8366
- "cosmetic",
8367
- "facial",
8368
- "cream",
8369
- "serum",
8370
- "mask",
8371
- "makeup",
8372
- "fragrance"
8373
- ];
8374
- const hasKeywords = productKeywords.some((kw) => content.includes(kw));
8375
8605
  const metadata = item.metadata || {};
8376
- const hasMetadataKey = Object.keys(metadata).some((k) => {
8377
- const val = metadata[k];
8378
- return ["price", "product", "sku", "brand", "cost"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
8379
- });
8380
- const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
8381
- return hasKeywords || hasMetadataKey || hasPricePattern;
8606
+ const domain = String(metadata.domain || "").toLowerCase();
8607
+ if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
8608
+ return false;
8609
+ }
8610
+ if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
8611
+ return true;
8612
+ }
8613
+ const content = (item.content || "").toLowerCase();
8614
+ const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
8615
+ if (financialTerms.test(content)) {
8616
+ return false;
8617
+ }
8618
+ const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
8619
+ const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
8620
+ const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
8621
+ return hasSku || hasProductIdentity && hasPrice;
8382
8622
  }
8383
8623
  static isTimeSeriesData(item) {
8384
8624
  const metadata = item.metadata || {};
@@ -8397,10 +8637,31 @@ ${schemaProfileText}` : ""}`;
8397
8637
  return fieldCount.size > 2;
8398
8638
  }
8399
8639
  static profileData(data) {
8640
+ const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
8641
+ "filetype",
8642
+ "mimetype",
8643
+ "filename",
8644
+ "docid",
8645
+ "uploadedat",
8646
+ "dimension",
8647
+ "chunkindex",
8648
+ "totalchunks",
8649
+ "charactercount",
8650
+ "workspaceid",
8651
+ "pineconenamespace",
8652
+ "tier",
8653
+ "custommetadata",
8654
+ "csvheaders",
8655
+ "headers",
8656
+ "filesize",
8657
+ "content",
8658
+ "id"
8659
+ ]);
8400
8660
  const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
8401
8661
  var _a2, _b;
8402
8662
  const fields2 = {};
8403
8663
  Object.entries(item.metadata || {}).forEach(([key, value]) => {
8664
+ if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
8404
8665
  const primitive = this.toPrimitive(value);
8405
8666
  if (primitive !== null) fields2[key] = primitive;
8406
8667
  });
@@ -9354,27 +9615,74 @@ ${m.content}`).join("\n\n---\n\n");
9354
9615
  * Ingest documents with automatic chunking, embedding, and batch upsert.
9355
9616
  */
9356
9617
  async ingest(documents, namespace) {
9618
+ var _a2;
9357
9619
  await this.initialize();
9358
9620
  const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
9359
9621
  const results = [];
9622
+ if (!documents || documents.length === 0) return results;
9623
+ const docChunksMap = [];
9624
+ const allChunks = [];
9360
9625
  for (const doc of documents) {
9361
9626
  try {
9362
9627
  const chunks = await this.prepareChunks(doc);
9363
- const vectors = await this.processEmbeddings(chunks);
9628
+ docChunksMap.push({ doc, chunks });
9629
+ allChunks.push(...chunks);
9630
+ } catch (error) {
9631
+ Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
9632
+ results.push({ docId: doc.docId, chunksIngested: 0 });
9633
+ }
9634
+ }
9635
+ if (allChunks.length === 0) {
9636
+ return results;
9637
+ }
9638
+ try {
9639
+ const allVectors = await this.processEmbeddings(allChunks);
9640
+ let vectorOffset = 0;
9641
+ const allUpsertDocs = [];
9642
+ const docSuccessCount = /* @__PURE__ */ new Map();
9643
+ for (const { doc, chunks } of docChunksMap) {
9644
+ const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
9645
+ vectorOffset += chunks.length;
9364
9646
  const upsertDocs = chunks.map((chunk, i) => ({
9365
9647
  id: chunk.id,
9366
- vector: vectors[i],
9648
+ vector: docVectors[i],
9367
9649
  content: chunk.content,
9368
9650
  metadata: chunk.metadata
9369
9651
  }));
9370
- const totalProcessed = await this.processUpserts(upsertDocs, ns);
9371
- results.push({ docId: doc.docId, chunksIngested: totalProcessed });
9652
+ allUpsertDocs.push(...upsertDocs);
9653
+ docSuccessCount.set(doc.docId, chunks.length);
9654
+ }
9655
+ const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
9656
+ for (const { doc, chunks } of docChunksMap) {
9657
+ results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
9372
9658
  if (this.graphDB && this.entityExtractor) {
9373
- await this.processGraphIngestion(doc.docId, chunks);
9659
+ this.processGraphIngestion(doc.docId, chunks).catch((err) => {
9660
+ Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
9661
+ });
9662
+ }
9663
+ }
9664
+ } catch (batchErr) {
9665
+ Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
9666
+ for (const { doc, chunks } of docChunksMap) {
9667
+ try {
9668
+ const vectors = await this.processEmbeddings(chunks);
9669
+ const upsertDocs = chunks.map((chunk, i) => ({
9670
+ id: chunk.id,
9671
+ vector: vectors[i],
9672
+ content: chunk.content,
9673
+ metadata: chunk.metadata
9674
+ }));
9675
+ const totalProcessed = await this.processUpserts(upsertDocs, ns);
9676
+ results.push({ docId: doc.docId, chunksIngested: totalProcessed });
9677
+ if (this.graphDB && this.entityExtractor) {
9678
+ this.processGraphIngestion(doc.docId, chunks).catch((err) => {
9679
+ Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
9680
+ });
9681
+ }
9682
+ } catch (singleErr) {
9683
+ Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
9684
+ results.push({ docId: doc.docId, chunksIngested: 0 });
9374
9685
  }
9375
- } catch (error) {
9376
- Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
9377
- results.push({ docId: doc.docId, chunksIngested: 0 });
9378
9686
  }
9379
9687
  }
9380
9688
  return results;
@@ -9394,19 +9702,40 @@ ${m.content}`).join("\n\n---\n\n");
9394
9702
  }
9395
9703
  /**
9396
9704
  * Step 2: Generate embeddings for chunks with retry logic.
9397
- * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
9705
+ * Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
9398
9706
  */
9399
9707
  async processEmbeddings(chunks) {
9400
- const embedBatchOptions = {
9401
- batchSize: 50,
9402
- maxRetries: 3,
9403
- initialDelayMs: 100
9404
- };
9405
- const vectors = await BatchProcessor.mapWithRetry(
9406
- chunks.map((c) => c.content),
9708
+ if (chunks.length === 0) return [];
9709
+ const texts = chunks.map((c) => c.content);
9710
+ if (typeof this.embeddingProvider.batchEmbed === "function") {
9711
+ try {
9712
+ const batchSize = 50;
9713
+ const allVectors = [];
9714
+ for (let i = 0; i < texts.length; i += batchSize) {
9715
+ const slice = texts.slice(i, i + batchSize);
9716
+ const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
9717
+ allVectors.push(...batchVectors);
9718
+ }
9719
+ if (allVectors.length === chunks.length) {
9720
+ return allVectors;
9721
+ }
9722
+ } catch (err) {
9723
+ Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
9724
+ error: err instanceof Error ? err.message : String(err)
9725
+ });
9726
+ }
9727
+ }
9728
+ const concurrentResult = await BatchProcessor.processConcurrent(
9729
+ texts,
9407
9730
  (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
9408
- embedBatchOptions
9731
+ 10,
9732
+ {
9733
+ maxRetries: 3,
9734
+ initialDelayMs: 100,
9735
+ throwOnPartialFailure: false
9736
+ }
9409
9737
  );
9738
+ const vectors = concurrentResult.results;
9410
9739
  if (vectors.length !== chunks.length) {
9411
9740
  throw new EmbeddingFailedException(
9412
9741
  `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
@@ -9433,25 +9762,24 @@ ${m.content}`).join("\n\n---\n\n");
9433
9762
  }
9434
9763
  /** Step 4: Optional graph-based entity extraction and ingestion. */
9435
9764
  async processGraphIngestion(docId, chunks) {
9436
- const extractionOptions = {
9437
- batchSize: 2,
9438
- maxRetries: 1,
9439
- initialDelayMs: 500
9440
- };
9441
- await BatchProcessor.processBatch(
9765
+ const maxConcurrency = Math.min(3, chunks.length);
9766
+ await BatchProcessor.processConcurrent(
9442
9767
  chunks,
9443
- async (batch) => {
9444
- for (const chunk of batch) {
9445
- try {
9446
- const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
9447
- if (nodes.length > 0) await this.graphDB.addNodes(nodes);
9448
- if (edges.length > 0) await this.graphDB.addEdges(edges);
9449
- } catch (err) {
9450
- Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
9451
- }
9768
+ async (chunk) => {
9769
+ try {
9770
+ const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
9771
+ if (nodes.length > 0) await this.graphDB.addNodes(nodes);
9772
+ if (edges.length > 0) await this.graphDB.addEdges(edges);
9773
+ } catch (err) {
9774
+ Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
9452
9775
  }
9453
9776
  },
9454
- extractionOptions
9777
+ maxConcurrency,
9778
+ {
9779
+ maxRetries: 1,
9780
+ initialDelayMs: 200,
9781
+ throwOnPartialFailure: false
9782
+ }
9455
9783
  );
9456
9784
  }
9457
9785
  async runNormalQuery(question, history = [], namespace) {
@@ -9576,7 +9904,7 @@ ${m.content}`).join("\n\n---\n\n");
9576
9904
  */
9577
9905
  askStreamInternal(_0) {
9578
9906
  return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
9579
- var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
9907
+ var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
9580
9908
  yield new __await(this.initialize());
9581
9909
  const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
9582
9910
  const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
@@ -9584,29 +9912,35 @@ ${m.content}`).join("\n\n---\n\n");
9584
9912
  const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
9585
9913
  const requestStart = performance.now();
9586
9914
  try {
9587
- let searchQuery = question;
9915
+ let effectiveQuestion = question;
9916
+ let promptOptResult;
9917
+ if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
9918
+ promptOptResult = PromptOptimizer.optimize(question, history);
9919
+ effectiveQuestion = promptOptResult.optimizedPrompt;
9920
+ }
9921
+ let searchQuery = effectiveQuestion;
9588
9922
  let rewrittenQuery;
9589
- if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
9590
- searchQuery = yield new __await(this.rewriteQuery(question, history));
9591
- rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
9923
+ if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
9924
+ searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
9925
+ rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
9592
9926
  }
9593
- const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
9594
- const filter = QueryProcessor.buildQueryFilter(question, hints);
9595
- const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
9927
+ const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
9928
+ const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
9929
+ const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
9596
9930
  if (numericPredicates.length > 0) {
9597
9931
  filter.__numericPredicates = numericPredicates;
9598
9932
  }
9599
9933
  const embedStart = performance.now();
9600
9934
  const cacheKey = `${ns}::${searchQuery}`;
9601
9935
  const cachedVector = this.embeddingCache.get(cacheKey);
9602
- const arch = (_h = this.config.rag) == null ? void 0 : _h.architecture;
9603
- const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_i = this.config.rag) == null ? void 0 : _i.useGraphRetrieval);
9936
+ const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
9937
+ const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
9604
9938
  const [strategyResult, embeddedVector] = yield new __await(Promise.all([
9605
9939
  QueryProcessor.determineRetrievalStrategy(
9606
9940
  searchQuery,
9607
9941
  useGraph ? this.llmRouter.get("fast") : void 0,
9608
- (_j = this.config.rag) == null ? void 0 : _j.graphKeywords,
9609
- (_k = this.config.rag) == null ? void 0 : _k.vectorKeywords
9942
+ (_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
9943
+ (_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
9610
9944
  ),
9611
9945
  // Embed immediately regardless of strategy — costs nothing if cached.
9612
9946
  // If strategy turns out to be 'graph'-only we just won't use the vector.
@@ -9616,7 +9950,7 @@ ${m.content}`).join("\n\n---\n\n");
9616
9950
  if (!cachedVector && queryVector.length > 0) {
9617
9951
  this.embeddingCache.set(cacheKey, queryVector);
9618
9952
  }
9619
- const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_l = this.config.rag) == null ? void 0 : _l.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
9953
+ const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
9620
9954
  const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
9621
9955
  const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
9622
9956
  const wantsExhaustiveList = true;
@@ -9633,7 +9967,7 @@ ${m.content}`).join("\n\n---\n\n");
9633
9967
  return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
9634
9968
  });
9635
9969
  const rerankLimit = Math.max(retrievalLimit, fullSources.length);
9636
- const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_m = this.config.rag) == null ? void 0 : _m.useReranking));
9970
+ const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
9637
9971
  if (!hasNumericPredicates && useReranking) {
9638
9972
  fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
9639
9973
  }
@@ -9661,7 +9995,7 @@ ${graphContext}
9661
9995
  VECTOR CONTEXT:
9662
9996
  ${context}`;
9663
9997
  }
9664
- if ((_o = (_n = this.config.rag) == null ? void 0 : _n.cag) == null ? void 0 : _o.enabled) {
9998
+ if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
9665
9999
  if (!this.coldContexts.has(ns)) {
9666
10000
  yield new __await(this.loadColdContext(ns));
9667
10001
  }
@@ -9693,31 +10027,31 @@ ${context}`;
9693
10027
  SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
9694
10028
  });
9695
10029
  }
9696
- const isProductQ = UITransformer.isProductQueryPublic(question);
10030
+ const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
9697
10031
  const uiTransformationPromise = isProductQ ? Promise.resolve(
9698
10032
  UITransformer.transform(
9699
- question,
10033
+ effectiveQuestion,
9700
10034
  sources,
9701
10035
  this.config,
9702
10036
  cachedSchema,
9703
10037
  { visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
9704
10038
  )
9705
10039
  ) : this.generateUiTransformation(
9706
- question,
10040
+ effectiveQuestion,
9707
10041
  sources,
9708
10042
  cachedSchema,
9709
10043
  hasNumericPredicates
9710
10044
  ).catch((uiError) => {
9711
10045
  Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
9712
- return UITransformer.transform(question, sources, this.config, cachedSchema);
10046
+ return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
9713
10047
  });
9714
10048
  let hallucinationScoringPromise = Promise.resolve(void 0);
9715
10049
  const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
9716
10050
  const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
9717
- const isNativeThinking = isClaude37 && (((_p = this.config.llm.options) == null ? void 0 : _p.thinking) === true || ((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === "enabled" || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) !== false);
10051
+ const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
9718
10052
  const modelNameLower = this.config.llm.model.toLowerCase();
9719
10053
  const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
9720
- const isSimulatedThinking = !isNativeThinking && (((_s = this.config.llm.options) == null ? void 0 : _s.thinking) === true || ((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === "enabled" || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) !== false);
10054
+ const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
9721
10055
  let finalRestrictionSuffix = restrictionSuffix;
9722
10056
  if (isSimulatedThinking) {
9723
10057
  finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
@@ -9726,9 +10060,9 @@ ${context}`;
9726
10060
  role: m.role || "user",
9727
10061
  content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
9728
10062
  }));
9729
- const userQuestion = { role: "user", content: question + finalRestrictionSuffix };
10063
+ const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
9730
10064
  const messages = [...hardenedHistory, userQuestion];
9731
- const systemPrompt = (_v = this.config.llm.systemPrompt) != null ? _v : "";
10065
+ const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
9732
10066
  const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
9733
10067
  let fullReply = "";
9734
10068
  let thinkingText = "";
@@ -9839,7 +10173,7 @@ ${context}`;
9839
10173
  }
9840
10174
  yield fullReply;
9841
10175
  }
9842
- const runHallucination = ((_w = this.config.llm.options) == null ? void 0 : _w.hallucinationScoring) === true;
10176
+ const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
9843
10177
  if (runHallucination) {
9844
10178
  hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
9845
10179
  }
@@ -9861,7 +10195,7 @@ ${context}`;
9861
10195
  totalTokens: promptTokens + completionTokens,
9862
10196
  estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
9863
10197
  };
9864
- const awaitHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.awaitHallucination) === true;
10198
+ const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
9865
10199
  const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
9866
10200
  uiTransformationPromise,
9867
10201
  awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
@@ -9869,9 +10203,9 @@ ${context}`;
9869
10203
  const buildTrace = (hScore) => __spreadValues({
9870
10204
  requestId,
9871
10205
  query: question,
9872
- rewrittenQuery,
10206
+ rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
9873
10207
  systemPrompt,
9874
- userPrompt: question + finalRestrictionSuffix,
10208
+ userPrompt: effectiveQuestion + finalRestrictionSuffix,
9875
10209
  chunks: (sources || []).filter(Boolean).map((s) => {
9876
10210
  var _a3, _b2;
9877
10211
  return {
@@ -9890,10 +10224,10 @@ ${context}`;
9890
10224
  hallucinationReason: hScore.reason
9891
10225
  } : {});
9892
10226
  const trace = buildTrace(hallucinationResult);
9893
- const isTelemetryActive = (_z = (_y = this.config.telemetry) == null ? void 0 : _y.enabled) != null ? _z : Boolean(this.config.licenseKey);
10227
+ const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
9894
10228
  if (isTelemetryActive) {
9895
10229
  const defaultUrl = "https://www.retrivora.com/api/telemetry";
9896
- const telemetryUrl = ((_A = this.config.telemetry) == null ? void 0 : _A.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
10230
+ const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
9897
10231
  const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
9898
10232
  (async () => {
9899
10233
  var _a3, _b2, _c2, _d2, _e2;
@@ -10757,13 +11091,81 @@ function createFromPreset(presetName) {
10757
11091
 
10758
11092
  // src/utils/DocumentParser.ts
10759
11093
  var DocumentParser = class {
11094
+ /**
11095
+ * Fast Magic-Bytes sniffer for instant file format identification (<1ms).
11096
+ */
11097
+ static sniffMagicBytes(buffer) {
11098
+ if (!buffer || buffer.length < 4) return null;
11099
+ if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
11100
+ return "pdf";
11101
+ }
11102
+ if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
11103
+ return "zip";
11104
+ }
11105
+ if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
11106
+ return "ole2";
11107
+ }
11108
+ const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
11109
+ if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
11110
+ return "json";
11111
+ }
11112
+ return "text";
11113
+ }
11114
+ /**
11115
+ * Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
11116
+ * Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
11117
+ */
11118
+ static formatSheetToContextualText(sheet, sheetName, XLSX) {
11119
+ const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
11120
+ if (!jsonRows || jsonRows.length === 0) {
11121
+ const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
11122
+ return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
11123
+ ${csv.trim()}` : "";
11124
+ }
11125
+ const lines = [`--- Sheet: ${sheetName} ---`];
11126
+ let currentHeader = [];
11127
+ for (let r = 0; r < jsonRows.length; r++) {
11128
+ const row = jsonRows[r];
11129
+ if (!row || row.length === 0) continue;
11130
+ const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
11131
+ if (cells.length === 0) continue;
11132
+ if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
11133
+ lines.push(`
11134
+ ## ${cells.join(" ")}`);
11135
+ continue;
11136
+ }
11137
+ const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
11138
+ if (isHeaderCandidate && cells.length >= 2) {
11139
+ currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
11140
+ lines.push(`| ${cells.join(" | ")} |`);
11141
+ lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
11142
+ continue;
11143
+ }
11144
+ if (currentHeader.length > 0 && cells.length > 1) {
11145
+ const fieldPairs = [];
11146
+ for (let c = 0; c < row.length; c++) {
11147
+ const h = currentHeader[c];
11148
+ const val = row[c];
11149
+ if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
11150
+ fieldPairs.push(`${h}: ${String(val).trim()}`);
11151
+ }
11152
+ }
11153
+ if (fieldPairs.length > 0) {
11154
+ lines.push(fieldPairs.join(" | "));
11155
+ continue;
11156
+ }
11157
+ }
11158
+ lines.push(cells.join(", "));
11159
+ }
11160
+ return lines.join("\n");
11161
+ }
10760
11162
  /**
10761
11163
  * Extract text from a File or Buffer based on its type.
10762
11164
  */
10763
- static async parse(file, fileName, mimeType) {
11165
+ static async parse(file, fileName, mimeType = "") {
10764
11166
  var _a2;
10765
11167
  const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
10766
- if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
11168
+ if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
10767
11169
  return this.readAsText(file);
10768
11170
  }
10769
11171
  if (extension === "json" || mimeType === "application/json") {
@@ -10775,39 +11177,116 @@ var DocumentParser = class {
10775
11177
  return text;
10776
11178
  }
10777
11179
  }
10778
- if (extension === "csv" || mimeType === "text/csv") {
11180
+ if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
10779
11181
  return this.readAsText(file);
10780
11182
  }
10781
- if (extension === "xlsx" || extension === "xls" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
11183
+ if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
11184
+ return this.readAsText(file);
11185
+ }
11186
+ if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
10782
11187
  try {
10783
- const ExcelJS = await import("exceljs");
11188
+ const JSZip = (await import("jszip")).default || await import("jszip");
10784
11189
  const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10785
- const workbook = new ExcelJS.Workbook();
10786
- await workbook.xlsx.load(buffer);
10787
- const sheetsText = [];
10788
- workbook.eachSheet((sheet, sheetId) => {
10789
- const csvLines = [];
10790
- sheet.eachRow((row, rowNumber) => {
10791
- const values = row.values;
10792
- if (Array.isArray(values)) {
10793
- csvLines.push(values.slice(1).join(","));
11190
+ const zip = await JSZip.loadAsync(buffer);
11191
+ const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
11192
+ var _a3, _b;
11193
+ const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
11194
+ const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
11195
+ return numA - numB;
11196
+ });
11197
+ if (slideFiles.length > 0) {
11198
+ const slidesText = [];
11199
+ for (let i = 0; i < slideFiles.length; i++) {
11200
+ const slideXml = await zip.files[slideFiles[i]].async("text");
11201
+ const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
11202
+ if (textMatches && textMatches.length > 0) {
11203
+ const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
11204
+ if (slideContent.trim()) {
11205
+ slidesText.push(`--- Slide ${i + 1} ---
11206
+ ${slideContent.trim()}`);
11207
+ }
10794
11208
  }
10795
- });
10796
- const csv = csvLines.join("\n");
10797
- if (csv && csv.trim()) {
10798
- sheetsText.push(`--- Sheet: ${sheet.name} ---
10799
- ${csv.trim()}`);
10800
11209
  }
10801
- });
10802
- return sheetsText.join("\n\n") || `[Empty Excel Workbook: ${fileName}]`;
10803
- } catch (e) {
11210
+ if (slidesText.length > 0) {
11211
+ return slidesText.join("\n\n");
11212
+ }
11213
+ }
11214
+ if (zip.files["content.xml"]) {
11215
+ const contentXml = await zip.files["content.xml"].async("text");
11216
+ const cleanText = this.stripXmlTags(contentXml);
11217
+ if (cleanText.trim()) {
11218
+ return cleanText.trim();
11219
+ }
11220
+ }
11221
+ const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
11222
+ const extractedSegments = [];
11223
+ for (const xf of allXmlFiles) {
11224
+ const xml = await zip.files[xf].async("text");
11225
+ const stripped = this.stripXmlTags(xml);
11226
+ if (stripped.length > 10) extractedSegments.push(stripped);
11227
+ }
11228
+ if (extractedSegments.length > 0) {
11229
+ return extractedSegments.join("\n\n");
11230
+ }
11231
+ return `[PowerPoint Presentation: ${fileName}]`;
11232
+ } catch (err) {
11233
+ console.warn("[DocumentParser] PowerPoint parsing failed:", err);
10804
11234
  try {
10805
11235
  return await this.readAsText(file);
10806
- } catch (e2) {
10807
- return `[Excel File Content: ${fileName}]`;
11236
+ } catch (e) {
11237
+ return `[PowerPoint Presentation File: ${fileName}]`;
10808
11238
  }
10809
11239
  }
10810
11240
  }
11241
+ if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
11242
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
11243
+ try {
11244
+ const XLSX = await import("xlsx");
11245
+ const workbook = XLSX.read(buffer, { type: "buffer" });
11246
+ const sheetsText = [];
11247
+ for (const sheetName of workbook.SheetNames) {
11248
+ const sheet = workbook.Sheets[sheetName];
11249
+ const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
11250
+ if (formatted && formatted.trim()) {
11251
+ sheetsText.push(formatted.trim());
11252
+ }
11253
+ }
11254
+ if (sheetsText.length > 0) {
11255
+ return sheetsText.join("\n\n");
11256
+ }
11257
+ } catch (xlsxErr) {
11258
+ console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
11259
+ }
11260
+ const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
11261
+ if (isZip) {
11262
+ try {
11263
+ const ExcelJS = await import("exceljs");
11264
+ const workbook = new ExcelJS.Workbook();
11265
+ await workbook.xlsx.load(buffer);
11266
+ const sheetsText = [];
11267
+ workbook.eachSheet((sheet) => {
11268
+ const csvLines = [];
11269
+ sheet.eachRow((row) => {
11270
+ const values = row.values;
11271
+ if (Array.isArray(values)) {
11272
+ csvLines.push(values.slice(1).join(","));
11273
+ }
11274
+ });
11275
+ const csv = csvLines.join("\n");
11276
+ if (csv && csv.trim()) {
11277
+ sheetsText.push(`--- Sheet: ${sheet.name} ---
11278
+ ${csv.trim()}`);
11279
+ }
11280
+ });
11281
+ if (sheetsText.length > 0) {
11282
+ return sheetsText.join("\n\n");
11283
+ }
11284
+ } catch (err) {
11285
+ console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
11286
+ }
11287
+ }
11288
+ return this.parseLegacyExcelOrFallback(buffer, fileName);
11289
+ }
10811
11290
  if (extension === "pdf" || mimeType === "application/pdf") {
10812
11291
  try {
10813
11292
  const pdfModule = await import("pdf-parse");
@@ -10820,15 +11299,48 @@ ${csv.trim()}`);
10820
11299
  return `[PDF Parsing Error for ${fileName}]`;
10821
11300
  }
10822
11301
  }
10823
- if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
11302
+ if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
10824
11303
  try {
10825
11304
  const mammoth = await import("mammoth");
10826
11305
  const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10827
11306
  const result = await mammoth.extractRawText({ buffer });
10828
- return result.value;
11307
+ if (result.value && result.value.trim()) {
11308
+ return result.value;
11309
+ }
11310
+ } catch (err) {
11311
+ console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
11312
+ }
11313
+ try {
11314
+ const raw = await this.readAsText(file);
11315
+ if (extension === "rtf" || raw.startsWith("{\\rtf")) {
11316
+ return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
11317
+ }
11318
+ return raw;
11319
+ } catch (e) {
11320
+ return `[Word Document File: ${fileName}]`;
11321
+ }
11322
+ }
11323
+ if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
11324
+ const raw = await this.readAsText(file);
11325
+ return this.stripHtmlAndXml(raw);
11326
+ }
11327
+ if (extension === "epub" || mimeType.includes("epub")) {
11328
+ try {
11329
+ const JSZip = (await import("jszip")).default || await import("jszip");
11330
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
11331
+ const zip = await JSZip.loadAsync(buffer);
11332
+ const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
11333
+ const chapters = [];
11334
+ for (const hf of htmlFiles) {
11335
+ const content = await zip.files[hf].async("text");
11336
+ const clean = this.stripHtmlAndXml(content);
11337
+ if (clean.length > 50) chapters.push(clean);
11338
+ }
11339
+ if (chapters.length > 0) {
11340
+ return chapters.join("\n\n---\n\n");
11341
+ }
10829
11342
  } catch (err) {
10830
- console.warn('[DocumentParser] Word parsing failed. Make sure "mammoth" is installed.', err);
10831
- return `[Word Parsing Error for ${fileName}]`;
11343
+ console.warn("[DocumentParser] EPUB parsing failed:", err);
10832
11344
  }
10833
11345
  }
10834
11346
  try {
@@ -10837,6 +11349,237 @@ ${csv.trim()}`);
10837
11349
  throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
10838
11350
  }
10839
11351
  }
11352
+ /**
11353
+ * Helper to strip XML tags and normalize inner text content.
11354
+ */
11355
+ static stripXmlTags(xml) {
11356
+ return xml.replace(/<[^>]+>/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/\s+/g, " ").trim();
11357
+ }
11358
+ /**
11359
+ * Helper to strip HTML / XML tags while removing script and style blocks completely.
11360
+ */
11361
+ static stripHtmlAndXml(html) {
11362
+ return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
11363
+ }
11364
+ /**
11365
+ * Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
11366
+ */
11367
+ static parseLegacyExcelOrFallback(buffer, fileName) {
11368
+ const rawStr = buffer.toString("utf-8");
11369
+ if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
11370
+ const rows = [];
11371
+ const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
11372
+ if (rowMatches && rowMatches.length > 0) {
11373
+ for (const rowHtml of rowMatches) {
11374
+ const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
11375
+ if (cells && cells.length > 0) {
11376
+ const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
11377
+ if (rowText) rows.push(rowText);
11378
+ }
11379
+ }
11380
+ }
11381
+ if (rows.length > 0) {
11382
+ return rows.join("\n");
11383
+ }
11384
+ const clean = this.stripHtmlAndXml(rawStr);
11385
+ if (clean.length > 10) return clean;
11386
+ }
11387
+ if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
11388
+ const rows = [];
11389
+ const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
11390
+ if (rowMatches && rowMatches.length > 0) {
11391
+ for (const rowXml of rowMatches) {
11392
+ const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
11393
+ if (dataMatches && dataMatches.length > 0) {
11394
+ const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
11395
+ if (rowText) rows.push(rowText);
11396
+ }
11397
+ }
11398
+ }
11399
+ if (rows.length > 0) {
11400
+ return rows.join("\n");
11401
+ }
11402
+ }
11403
+ const extractedStrings = [];
11404
+ const ignoredOleMarkers = /* @__PURE__ */ new Set([
11405
+ "root entry",
11406
+ "workbook",
11407
+ "compobj",
11408
+ "summaryinformation",
11409
+ "documentsummaryinformation",
11410
+ "microsoft excel",
11411
+ "worksheet",
11412
+ "biff8",
11413
+ "calcpr"
11414
+ ]);
11415
+ let currentUtf16 = "";
11416
+ for (let i = 0; i < buffer.length - 1; i += 2) {
11417
+ const charCode = buffer[i];
11418
+ const zeroByte = buffer[i + 1];
11419
+ if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
11420
+ currentUtf16 += String.fromCharCode(charCode);
11421
+ } else {
11422
+ if (currentUtf16.length >= 2) {
11423
+ const trimmed = currentUtf16.trim();
11424
+ if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
11425
+ extractedStrings.push(trimmed);
11426
+ }
11427
+ }
11428
+ currentUtf16 = "";
11429
+ }
11430
+ }
11431
+ let currentAscii = "";
11432
+ for (let i = 0; i < buffer.length; i++) {
11433
+ const byte = buffer[i];
11434
+ if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
11435
+ currentAscii += String.fromCharCode(byte);
11436
+ } else {
11437
+ if (currentAscii.length >= 2) {
11438
+ const trimmed = currentAscii.trim();
11439
+ if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
11440
+ extractedStrings.push(trimmed);
11441
+ }
11442
+ }
11443
+ currentAscii = "";
11444
+ }
11445
+ }
11446
+ const uniqueStrings = [];
11447
+ for (const str of extractedStrings) {
11448
+ if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
11449
+ uniqueStrings.push(str);
11450
+ }
11451
+ }
11452
+ if (uniqueStrings.length > 0) {
11453
+ return uniqueStrings.join(" | ");
11454
+ }
11455
+ const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
11456
+ return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
11457
+ }
11458
+ /**
11459
+ * Dynamically infer document domain and schema from extracted text, filename, and format.
11460
+ */
11461
+ static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
11462
+ const lowerText = text.toLowerCase();
11463
+ const lowerName = fileName.toLowerCase();
11464
+ const financialTerms = [
11465
+ "price per share",
11466
+ "share price",
11467
+ "total cost of the deal",
11468
+ "cost of the deal",
11469
+ "lbo",
11470
+ "merger",
11471
+ "valuation",
11472
+ "d/e ratio",
11473
+ "ebit",
11474
+ "ebitda",
11475
+ "fcfe",
11476
+ "fcff",
11477
+ "wacc",
11478
+ "beta",
11479
+ "working capital",
11480
+ "cogs",
11481
+ "cap spending",
11482
+ "depreciation",
11483
+ "cashflow",
11484
+ "cash flow",
11485
+ "net income",
11486
+ "balance sheet",
11487
+ "income statement",
11488
+ "interest rate",
11489
+ "terminal year",
11490
+ "equity",
11491
+ "debt outstanding",
11492
+ "gross margin",
11493
+ "fiscal year",
11494
+ "fy20",
11495
+ "fy21",
11496
+ "fy22",
11497
+ "fy23",
11498
+ "fy24",
11499
+ "fy25",
11500
+ "q1",
11501
+ "q2",
11502
+ "q3",
11503
+ "q4"
11504
+ ];
11505
+ let financialScore = 0;
11506
+ for (const term of financialTerms) {
11507
+ if (lowerText.includes(term) || lowerName.includes(term)) {
11508
+ financialScore += 1;
11509
+ }
11510
+ }
11511
+ const ecommerceTerms = [
11512
+ "sku",
11513
+ "product_id",
11514
+ "product id",
11515
+ "item_id",
11516
+ "add to cart",
11517
+ "buy now",
11518
+ "in stock",
11519
+ "out of stock",
11520
+ "inventory count",
11521
+ "retail price",
11522
+ "unit price",
11523
+ "msrp",
11524
+ "cart",
11525
+ "shop",
11526
+ "catalog",
11527
+ "product_name",
11528
+ "product name",
11529
+ "brand name"
11530
+ ];
11531
+ let ecommerceScore = 0;
11532
+ for (const term of ecommerceTerms) {
11533
+ if (lowerText.includes(term) || lowerName.includes(term)) {
11534
+ ecommerceScore += 1;
11535
+ }
11536
+ }
11537
+ const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
11538
+ let legalScore = 0;
11539
+ for (const term of legalTerms) {
11540
+ if (lowerText.includes(term)) legalScore += 1;
11541
+ }
11542
+ const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
11543
+ let healthScore = 0;
11544
+ for (const term of healthTerms) {
11545
+ if (lowerText.includes(term)) healthScore += 1;
11546
+ }
11547
+ let schema;
11548
+ const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
11549
+ if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
11550
+ const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
11551
+ const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
11552
+ if (headers.length > 1 && lines.length > 1) {
11553
+ const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
11554
+ schema = {};
11555
+ headers.forEach((h, idx) => {
11556
+ const val = sampleRow[idx] || "";
11557
+ if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
11558
+ schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
11559
+ } else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
11560
+ schema[h] = "date";
11561
+ } else {
11562
+ schema[h] = "string";
11563
+ }
11564
+ });
11565
+ }
11566
+ }
11567
+ const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
11568
+ const isCatalog = ecommerceScore >= 2 && !isFinancial;
11569
+ let domain = "general";
11570
+ if (isFinancial) {
11571
+ domain = "finance";
11572
+ } else if (isCatalog) {
11573
+ domain = "ecommerce";
11574
+ } else if (legalScore >= 2) {
11575
+ domain = "legal";
11576
+ } else if (healthScore >= 2) {
11577
+ domain = "healthcare";
11578
+ } else if (schema && Object.keys(schema).length > 0) {
11579
+ domain = "tabular";
11580
+ }
11581
+ return { domain, schema, isCatalog, isFinancial };
11582
+ }
10840
11583
  static async readAsText(file) {
10841
11584
  if (Buffer.isBuffer(file)) {
10842
11585
  return file.toString("utf-8");
@@ -11499,18 +12242,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
11499
12242
  }
11500
12243
  }
11501
12244
  function getAllowedNamespaces(projectId) {
11502
- const trimmed = projectId.trim();
12245
+ const trimmed = (projectId || "default").trim();
11503
12246
  const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
11504
12247
  return new Set([
11505
12248
  trimmed,
11506
12249
  withoutPrefix,
11507
- `retrivora-${withoutPrefix}`
12250
+ `retrivora-${withoutPrefix}`,
12251
+ "default",
12252
+ "retrivora-default"
11508
12253
  ].filter(Boolean));
11509
12254
  }
11510
12255
  function resolveLicensedNamespace(payload, requestedNamespace) {
12256
+ var _a2;
11511
12257
  const allowed = getAllowedNamespaces(payload.projectId);
11512
12258
  const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
11513
- if (requested && !allowed.has(requested)) {
12259
+ const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
12260
+ if (requested && !isDevOrFree && !allowed.has(requested)) {
11514
12261
  return {
11515
12262
  ok: false,
11516
12263
  response: new Response(JSON.stringify({
@@ -11524,7 +12271,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
11524
12271
  })
11525
12272
  };
11526
12273
  }
11527
- const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
12274
+ const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
11528
12275
  return { ok: true, namespace: requested || defaultNamespace };
11529
12276
  }
11530
12277
  var MAX_MESSAGE_LENGTH = 8e3;
@@ -11968,7 +12715,7 @@ function createIngestHandler(configOrPlugin, options) {
11968
12715
  function createHealthHandler(configOrPlugin, options) {
11969
12716
  const plugin = getOrCreatePlugin(configOrPlugin);
11970
12717
  const onAuthorize = options == null ? void 0 : options.onAuthorize;
11971
- return async function GET(req) {
12718
+ return async function GET(req, context) {
11972
12719
  if (req) {
11973
12720
  const authResult = await checkAuth(req, onAuthorize);
11974
12721
  if (authResult) return authResult;
@@ -12079,9 +12826,10 @@ function createUploadHandler(configOrPlugin, options) {
12079
12826
  }
12080
12827
  const documents = [];
12081
12828
  for (const file of files) {
12829
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
12082
12830
  const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
12083
12831
  if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
12084
- const text = await file.text();
12832
+ const text = buffer.toString("utf-8");
12085
12833
  const Papa = await import("papaparse");
12086
12834
  const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
12087
12835
  if (parsed.data && parsed.data.length > 0) {
@@ -12121,49 +12869,70 @@ function createUploadHandler(configOrPlugin, options) {
12121
12869
  }
12122
12870
  }
12123
12871
  } else if (isExcel) {
12124
- const ExcelJS = await import("exceljs");
12125
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
12126
- const workbook = new ExcelJS.Workbook();
12127
- await workbook.xlsx.load(buffer);
12128
- workbook.eachSheet((sheet, sheetId) => {
12129
- const sheetName = sheet.name;
12130
- let headers = [];
12131
- let i = 0;
12132
- sheet.eachRow((row, rowNumber) => {
12133
- const values = Array.isArray(row.values) ? row.values.slice(1) : [];
12134
- if (rowNumber === 1) {
12135
- headers = values.map((v) => String(v));
12136
- return;
12137
- }
12138
- i++;
12139
- const contentParts = [];
12140
- const rowData = {};
12141
- values.forEach((val, index) => {
12142
- const key = headers[index] || `Column${index + 1}`;
12143
- rowData[key] = val;
12144
- if (key && val !== void 0 && val !== null && String(val).trim()) {
12145
- contentParts.push(`${key}: ${val}`);
12146
- }
12147
- });
12148
- if (contentParts.length > 0) {
12149
- documents.push({
12150
- docId: `${file.name}-${sheetName}-row-${i}`,
12151
- content: contentParts.join(", "),
12152
- metadata: __spreadValues(__spreadProps(__spreadValues({
12153
- fileName: file.name,
12154
- sheetName,
12155
- fileSize: file.size,
12156
- fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
12157
- uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12158
- }, dimension ? { dimension } : {}), {
12159
- headers
12160
- }), rowData)
12872
+ let excelLoaded = false;
12873
+ try {
12874
+ const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
12875
+ if (isZip) {
12876
+ const ExcelJS = await import("exceljs");
12877
+ const workbook = new ExcelJS.Workbook();
12878
+ await workbook.xlsx.load(buffer);
12879
+ workbook.eachSheet((sheet, sheetId) => {
12880
+ const sheetName = sheet.name;
12881
+ let headers = [];
12882
+ let i = 0;
12883
+ sheet.eachRow((row, rowNumber) => {
12884
+ const values = Array.isArray(row.values) ? row.values.slice(1) : [];
12885
+ if (rowNumber === 1) {
12886
+ headers = values.map((v) => String(v));
12887
+ return;
12888
+ }
12889
+ i++;
12890
+ const contentParts = [];
12891
+ const rowData = {};
12892
+ values.forEach((val, index) => {
12893
+ const key = headers[index] || `Column${index + 1}`;
12894
+ rowData[key] = val;
12895
+ if (key && val !== void 0 && val !== null && String(val).trim()) {
12896
+ contentParts.push(`${key}: ${val}`);
12897
+ }
12898
+ });
12899
+ if (contentParts.length > 0) {
12900
+ documents.push({
12901
+ docId: `${file.name}-${sheetName}-row-${i}`,
12902
+ content: contentParts.join(", "),
12903
+ metadata: __spreadValues(__spreadProps(__spreadValues({
12904
+ fileName: file.name,
12905
+ sheetName,
12906
+ fileSize: file.size,
12907
+ fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
12908
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12909
+ }, dimension ? { dimension } : {}), {
12910
+ headers
12911
+ }), rowData)
12912
+ });
12913
+ }
12161
12914
  });
12162
- }
12915
+ });
12916
+ excelLoaded = documents.length > 0;
12917
+ }
12918
+ } catch (excelErr) {
12919
+ excelLoaded = false;
12920
+ }
12921
+ if (!excelLoaded) {
12922
+ const content = await DocumentParser.parse(buffer, file.name, file.type);
12923
+ documents.push({
12924
+ docId: file.name,
12925
+ content,
12926
+ metadata: __spreadValues({
12927
+ fileName: file.name,
12928
+ fileSize: file.size,
12929
+ fileType: file.type || "application/vnd.ms-excel",
12930
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12931
+ }, dimension ? { dimension } : {})
12163
12932
  });
12164
- });
12933
+ }
12165
12934
  } else {
12166
- const content = await DocumentParser.parse(file, file.name, file.type);
12935
+ const content = await DocumentParser.parse(buffer, file.name, file.type);
12167
12936
  documents.push({
12168
12937
  docId: file.name,
12169
12938
  content,
@@ -12322,7 +13091,7 @@ function createSessionsHandler(configOrPlugin, options) {
12322
13091
  const storage = new DatabaseStorage(plugin.getConfig());
12323
13092
  const onAuthorize = options == null ? void 0 : options.onAuthorize;
12324
13093
  const onResolveScope = options == null ? void 0 : options.onResolveScope;
12325
- return async function GET(req) {
13094
+ return async function GET(req, context) {
12326
13095
  if (req) {
12327
13096
  const authResult = await checkAuth(req, onAuthorize);
12328
13097
  if (authResult) return authResult;
@@ -13253,6 +14022,7 @@ export {
13253
14022
  PineconeProvider,
13254
14023
  Pipeline,
13255
14024
  PostgreSQLProvider,
14025
+ PromptOptimizer,
13256
14026
  ProviderHealthCheck,
13257
14027
  ProviderNotFoundException,
13258
14028
  ProviderRegistry,