@retrivora-ai/rag-engine 2.5.6 → 2.5.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -88,8 +88,8 @@ function buildPayload(template, vars) {
88
88
  let raw = template;
89
89
  for (const [key, val] of Object.entries(vars)) {
90
90
  const stringifiedVal = val === void 0 ? "null" : JSON.stringify(val);
91
- raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), stringifiedVal);
92
- raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), stringifiedVal);
91
+ raw = raw.replace(new RegExp(`"\\{\\{${key}\\}\\}"`, "g"), () => stringifiedVal);
92
+ raw = raw.replace(new RegExp(`\\{\\{${key}\\}\\}`, "g"), () => stringifiedVal);
93
93
  }
94
94
  try {
95
95
  return JSON.parse(raw);
@@ -312,7 +312,7 @@ var init_package = __esm({
312
312
  "package.json"() {
313
313
  package_default = {
314
314
  name: "@retrivora-ai/rag-engine",
315
- version: "2.5.6",
315
+ version: "2.5.8",
316
316
  description: "Retrivora AI is a plug-and-play AI engine for RAG chat experiences \u2014 generic vector DB + LLM provider, embeddable or standalone.",
317
317
  author: "Abhinav Alkuchi",
318
318
  license: "UNLICENSED",
@@ -415,9 +415,11 @@ var init_package = __esm({
415
415
  "@google/genai": "^0.8.0",
416
416
  "@google/generative-ai": "^0.24.1",
417
417
  "@pinecone-database/pinecone": "^7.2.0",
418
- "@retrivora-ai/billing-guards": "*",
418
+ "@retrivora-ai/billing-guards": "file:../billing-guards",
419
419
  "@types/papaparse": "^5.5.2",
420
+ exceljs: "^4.4.0",
420
421
  "framer-motion": "^13.1.0",
422
+ jszip: "^3.10.1",
421
423
  katex: "^0.18.4",
422
424
  "lucide-react": "^1.8.0",
423
425
  "next-themes": "^0.4.6",
@@ -430,7 +432,7 @@ var init_package = __esm({
430
432
  "rehype-katex": "^7.0.1",
431
433
  "remark-gfm": "^4.0.1",
432
434
  "remark-math": "^6.0.0",
433
- exceljs: "^4.4.0",
435
+ xlsx: "^0.18.5",
434
436
  zod: "^4.4.3"
435
437
  },
436
438
  optionalDependencies: {
@@ -448,6 +450,7 @@ var init_package = __esm({
448
450
  "@tailwindcss/postcss": "^4",
449
451
  "@types/estree": "^1.0.9",
450
452
  "@types/jest": "^30.0.0",
453
+ "@types/jszip": "^3.4.1",
451
454
  "@types/katex": "^0.16.8",
452
455
  "@types/node": "^20",
453
456
  "@types/pdf-parse": "^1.1.5",
@@ -2843,6 +2846,7 @@ var _LicenseVerifier = class _LicenseVerifier {
2843
2846
  try {
2844
2847
  response = await fetch(endpoint, {
2845
2848
  method: "POST",
2849
+ signal: AbortSignal.timeout(3e3),
2846
2850
  headers: {
2847
2851
  "Content-Type": "application/json",
2848
2852
  "x-license-key": licenseKey
@@ -5520,15 +5524,29 @@ Text to extract from:
5520
5524
  const response = await this.llm.chat([
5521
5525
  { role: "system", content: "You are a precise knowledge graph extraction engine. You always output valid JSON and nothing else." },
5522
5526
  { role: "user", content: prompt }
5523
- ], "", { maxTokens: 4096, temperature: 0.1 });
5527
+ ], "", { maxTokens: 800, temperature: 0.1 });
5524
5528
  try {
5525
- const jsonMatch = response.match(/\{[\s\S]*\}/);
5526
- const cleanJson = jsonMatch ? jsonMatch[0] : response.trim();
5527
- return JSON.parse(cleanJson);
5529
+ const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
5530
+ const codeFenceMatch = withoutThinking.match(/```(?:json)?\s*([\s\S]*?)\s*```/i);
5531
+ const candidate = codeFenceMatch ? codeFenceMatch[1] : withoutThinking;
5532
+ const jsonMatch = candidate.match(/\{[\s\S]*\}/);
5533
+ const cleanJson = jsonMatch ? jsonMatch[0] : candidate.trim();
5534
+ const parsed = JSON.parse(cleanJson);
5535
+ return {
5536
+ nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
5537
+ edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
5538
+ };
5528
5539
  } catch (e) {
5529
5540
  try {
5530
- const repaired = this.repairTruncatedJson(response);
5531
- if (repaired) return JSON.parse(repaired);
5541
+ const withoutThinking = response.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
5542
+ const repaired = this.repairTruncatedJson(withoutThinking);
5543
+ if (repaired) {
5544
+ const parsed = JSON.parse(repaired);
5545
+ return {
5546
+ nodes: Array.isArray(parsed == null ? void 0 : parsed.nodes) ? parsed.nodes : [],
5547
+ edges: Array.isArray(parsed == null ? void 0 : parsed.edges) ? parsed.edges : []
5548
+ };
5549
+ }
5532
5550
  } catch (e2) {
5533
5551
  }
5534
5552
  console.warn(`[EntityExtractor] Failed to parse LLM response. Length: ${response.length} chars.`);
@@ -5806,6 +5824,208 @@ ${error instanceof Error ? error.message : String(error)}`
5806
5824
  }
5807
5825
  };
5808
5826
 
5827
+ // src/rag/PromptOptimizer.ts
5828
+ var PromptOptimizer = class {
5829
+ /**
5830
+ * Optimizes a user prompt synchronously using fast heuristic rules and history resolution.
5831
+ */
5832
+ static optimize(rawPrompt, history = [], options = {}) {
5833
+ if (!rawPrompt || typeof rawPrompt !== "string") {
5834
+ return {
5835
+ optimizedPrompt: "",
5836
+ originalPrompt: rawPrompt || "",
5837
+ wasModified: false,
5838
+ transformations: []
5839
+ };
5840
+ }
5841
+ const { resolveHistory = true, expandKeywords = true } = options;
5842
+ const transformations = [];
5843
+ let current = rawPrompt;
5844
+ const cleaned = this.cleanText(current);
5845
+ if (cleaned !== current) {
5846
+ transformations.push("clean_whitespace_and_artifacts");
5847
+ current = cleaned;
5848
+ }
5849
+ const expandedAbbr = this.expandAbbreviations(current);
5850
+ if (expandedAbbr !== current) {
5851
+ transformations.push("expand_abbreviations");
5852
+ current = expandedAbbr;
5853
+ }
5854
+ if (expandKeywords) {
5855
+ const trimmedLower = current.toLowerCase().replace(/[^a-z0-9]/g, "");
5856
+ if (this.TOPIC_EXPANSIONS[trimmedLower]) {
5857
+ current = this.TOPIC_EXPANSIONS[trimmedLower];
5858
+ transformations.push(`topic_expansion_${trimmedLower}`);
5859
+ }
5860
+ }
5861
+ if (resolveHistory && history && history.length > 0) {
5862
+ const resolved = this.resolveCoreferences(current, history);
5863
+ if (resolved !== current) {
5864
+ transformations.push("resolve_history_coreferences");
5865
+ current = resolved;
5866
+ }
5867
+ }
5868
+ if (!/[?.!]$/.test(current.trim()) && current.length > 10) {
5869
+ if (/^(what|why|how|when|where|who|which|can|could|is|are|will|would|do|does|did|should)\b/i.test(current)) {
5870
+ current = `${current.trim()}?`;
5871
+ transformations.push("add_question_mark");
5872
+ }
5873
+ }
5874
+ return {
5875
+ optimizedPrompt: current,
5876
+ originalPrompt: rawPrompt,
5877
+ wasModified: current !== rawPrompt,
5878
+ transformations
5879
+ };
5880
+ }
5881
+ /**
5882
+ * Cleans text artifacts, fixes unclosed quotes/parentheses, and collapses whitespace.
5883
+ */
5884
+ static cleanText(text) {
5885
+ let clean = text.replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "").replace(/\?{2,}/g, "?").replace(/!{2,}/g, "!").replace(/\.{3,}/g, "...").replace(/\s+/g, " ").trim();
5886
+ const doubleQuotesCount = (clean.match(/"/g) || []).length;
5887
+ if (doubleQuotesCount % 2 !== 0) {
5888
+ clean += '"';
5889
+ }
5890
+ const singleQuotesCount = (clean.match(/'/g) || []).length;
5891
+ if (singleQuotesCount % 2 !== 0 && !/'\w/.test(clean)) {
5892
+ clean += "'";
5893
+ }
5894
+ const openParen = (clean.match(/\(/g) || []).length;
5895
+ const closeParen = (clean.match(/\)/g) || []).length;
5896
+ if (openParen > closeParen) {
5897
+ clean += ")".repeat(openParen - closeParen);
5898
+ }
5899
+ return clean;
5900
+ }
5901
+ /**
5902
+ * Replaces shorthand chat abbreviations with clear natural language.
5903
+ */
5904
+ static expandAbbreviations(text) {
5905
+ let expanded = text;
5906
+ for (const [pattern, replacement] of Object.entries(this.COMMON_ABBREVIATIONS)) {
5907
+ expanded = expanded.replace(new RegExp(pattern, "gi"), replacement);
5908
+ }
5909
+ return expanded;
5910
+ }
5911
+ /**
5912
+ * Resolves ambiguous pronouns ('it', 'this', 'they', 'the last one') against recent chat history.
5913
+ */
5914
+ static resolveCoreferences(prompt, history) {
5915
+ const pronounPattern = /\b(it|its|this|that|these|those|they|them|their|the previous one|the last one)\b/i;
5916
+ if (!pronounPattern.test(prompt)) {
5917
+ return prompt;
5918
+ }
5919
+ const recentMessages = [...history].slice(-4).reverse();
5920
+ let detectedEntity = null;
5921
+ for (const msg of recentMessages) {
5922
+ if (!msg.content) continue;
5923
+ const quotedMatch = msg.content.match(/"([^"]{2,60})"/);
5924
+ if (quotedMatch) {
5925
+ detectedEntity = quotedMatch[1];
5926
+ break;
5927
+ }
5928
+ const headingMatch = msg.content.match(/^#{1,3}\s+([^\n]{2,60})/m);
5929
+ if (headingMatch) {
5930
+ detectedEntity = headingMatch[1].trim();
5931
+ break;
5932
+ }
5933
+ const entityMatches = msg.content.match(/\b([A-Z][a-z0-9]+(?:\s+[A-Z][a-z0-9]+){1,3})\b/);
5934
+ if (entityMatches && !/^(Retrivora|Assistant|User|Sure|Here|Please)\b/i.test(entityMatches[1])) {
5935
+ detectedEntity = entityMatches[1];
5936
+ break;
5937
+ }
5938
+ }
5939
+ if (detectedEntity) {
5940
+ let resolved = prompt.replace(/\b(how much is it|how much does it cost)\b/gi, `What is the price of ${detectedEntity}`).replace(/\b(what is it|what does it do)\b/gi, `What is ${detectedEntity} and what are its features`).replace(/\b(explain it|tell me more about it)\b/gi, `Tell me more details about ${detectedEntity}`).replace(/\b(compare it|compare them)\b/gi, `Compare ${detectedEntity} with alternatives`).replace(/\b(it's|its)\b/gi, `${detectedEntity}'s`).replace(/\b(it|this|that)\b/gi, detectedEntity);
5941
+ return resolved;
5942
+ }
5943
+ return prompt;
5944
+ }
5945
+ /**
5946
+ * Deep LLM-powered prompt optimizer for complex, multi-hop, or highly ambiguous user queries.
5947
+ */
5948
+ static async optimizeWithLLM(rawPrompt, history = [], llmProvider) {
5949
+ const fastResult = this.optimize(rawPrompt, history);
5950
+ if (!llmProvider || !rawPrompt || rawPrompt.trim().length > 250) {
5951
+ return fastResult;
5952
+ }
5953
+ try {
5954
+ const historyContext = (history || []).slice(-4).map((m) => `${m.role === "user" ? "User" : "Assistant"}: ${m.content.slice(0, 150)}`).join("\n");
5955
+ const prompt = `You are a query optimizer in an enterprise RAG system.
5956
+ Given the conversation context and user's new question, rewrite the question into a clear, standalone, complete search prompt that extracts the core user intent.
5957
+ - Do NOT answer the question.
5958
+ - Resolve any pronouns like "it", "they", "that" using the context.
5959
+ - Keep the user's intent 100% intact.
5960
+ - If the question is already clear, return it as is.
5961
+ - Return ONLY the optimized query string. No quotes or explanations.
5962
+
5963
+ Context:
5964
+ ${historyContext || "No previous context."}
5965
+
5966
+ User Question: "${rawPrompt}"
5967
+
5968
+ Optimized Search Prompt:`;
5969
+ const response = await llmProvider.chat(
5970
+ [{ role: "user", content: prompt }],
5971
+ "",
5972
+ {
5973
+ systemPrompt: "You are an expert search query optimizer. Return ONLY the rewritten query string.",
5974
+ temperature: 0,
5975
+ maxTokens: 100
5976
+ }
5977
+ );
5978
+ const cleaned = (response || "").replace(/^["']|["']$/g, "").trim();
5979
+ if (cleaned && cleaned.length > 3 && !cleaned.toLowerCase().includes("i cannot")) {
5980
+ return {
5981
+ optimizedPrompt: cleaned,
5982
+ originalPrompt: rawPrompt,
5983
+ wasModified: cleaned !== rawPrompt,
5984
+ transformations: [...fastResult.transformations, "llm_deep_reformulation"]
5985
+ };
5986
+ }
5987
+ } catch (err) {
5988
+ console.warn("[PromptOptimizer] LLM query optimization fallback to heuristic:", err);
5989
+ }
5990
+ return fastResult;
5991
+ }
5992
+ };
5993
+ PromptOptimizer.COMMON_ABBREVIATIONS = {
5994
+ "\\bpls\\b": "please",
5995
+ "\\bplz\\b": "please",
5996
+ "\\bthx\\b": "thanks",
5997
+ "\\bty\\b": "thank you",
5998
+ "\\bapprox\\b": "approximately",
5999
+ "\\bw/o\\b": "without",
6000
+ "\\bw/\\b": "with",
6001
+ "\\bdiff\\b": "difference",
6002
+ "\\binfo\\b": "information",
6003
+ "\\bconfig\\b": "configuration",
6004
+ "\\bdoc\\b": "document",
6005
+ "\\bdocs\\b": "documentation",
6006
+ "\\bfeat\\b": "feature",
6007
+ "\\bfeats\\b": "features",
6008
+ "\\bvs\\b": "versus",
6009
+ "\\bu\\b": "you",
6010
+ "\\bur\\b": "your",
6011
+ "\\br\\b": "are"
6012
+ };
6013
+ PromptOptimizer.TOPIC_EXPANSIONS = {
6014
+ "pricing": "What are the available pricing plans, tiers, and cost structures?",
6015
+ "price": "What is the pricing and subscription cost?",
6016
+ "features": "What are the core features, capabilities, and tools provided?",
6017
+ "security": "What are the security, privacy, and compliance standards?",
6018
+ "privacy": "What is the privacy policy and how is data protected?",
6019
+ "refund": "What is the refund and cancellation policy?",
6020
+ "refunds": "What is the refund and cancellation policy?",
6021
+ "contact": "How can I contact customer support or sales?",
6022
+ "support": "What support channels and assistance are available?",
6023
+ "docs": "Where can I find the official documentation and setup guides?",
6024
+ "installation": "How do I install and configure the SDK or service?",
6025
+ "install": "What are the step-by-step installation instructions?",
6026
+ "faq": "What are the frequently asked questions and key answers?"
6027
+ };
6028
+
5809
6029
  // src/core/mcp.ts
5810
6030
  import { spawn } from "child_process";
5811
6031
  var MCPClient = class {
@@ -7023,6 +7243,26 @@ var IntentClassifier = class {
7023
7243
  static extractDataSignals(context) {
7024
7244
  const docs = context.retrievedDocuments || [];
7025
7245
  const rowCount = docs.length;
7246
+ const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
7247
+ "filetype",
7248
+ "mimetype",
7249
+ "filename",
7250
+ "docid",
7251
+ "uploadedat",
7252
+ "dimension",
7253
+ "chunkindex",
7254
+ "totalchunks",
7255
+ "charactercount",
7256
+ "workspaceid",
7257
+ "pineconenamespace",
7258
+ "tier",
7259
+ "custommetadata",
7260
+ "csvheaders",
7261
+ "headers",
7262
+ "filesize",
7263
+ "content",
7264
+ "id"
7265
+ ]);
7026
7266
  let hasNumericFields = false;
7027
7267
  let hasDateFields = false;
7028
7268
  let hasCategoricalFields = false;
@@ -7038,6 +7278,7 @@ var IntentClassifier = class {
7038
7278
  docs.forEach((doc) => {
7039
7279
  const meta = doc.metadata || {};
7040
7280
  Object.entries(meta).forEach(([key, val]) => {
7281
+ if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
7041
7282
  keys.add(key);
7042
7283
  const kLower = key.toLowerCase();
7043
7284
  if (["price", "image", "brand", "in_stock", "rating", "sku", "product"].some((p) => kLower.includes(p))) {
@@ -7046,7 +7287,7 @@ var IntentClassifier = class {
7046
7287
  if (typeof val === "number") {
7047
7288
  numericKeys.add(key);
7048
7289
  } else if (typeof val === "string") {
7049
- if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5) {
7290
+ if (/^\d{4}-\d{2}-\d{2}/.test(val) || !isNaN(Date.parse(val)) && val.length > 5 && !val.includes(" ")) {
7050
7291
  dateKeys.add(key);
7051
7292
  } else if (val.length < 50 && !val.includes("\n")) {
7052
7293
  catKeys.add(key);
@@ -7059,10 +7300,7 @@ var IntentClassifier = class {
7059
7300
  hasCategoricalFields = catKeys.size > 0;
7060
7301
  numericFieldCount = numericKeys.size;
7061
7302
  categoricalFieldCount = catKeys.size;
7062
- if (productKeyMatches >= 2 || docs.some((d) => {
7063
- var _a2, _b;
7064
- return ((_a2 = d == null ? void 0 : d.content) != null ? _a2 : "").toLowerCase().includes("price") || ((_b = d == null ? void 0 : d.content) != null ? _b : "").toLowerCase().includes("brand");
7065
- })) {
7303
+ if (productKeyMatches >= 2) {
7066
7304
  isProductLike = true;
7067
7305
  }
7068
7306
  }
@@ -7078,12 +7316,13 @@ var IntentClassifier = class {
7078
7316
  }
7079
7317
  // ─── Intent Heuristic Checkers ─────────────────────────────────────────────
7080
7318
  static isInformationLookup(query) {
7081
- if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(query)) {
7082
- if (!/\b(compare|versus|vs\.?|trend|monthly|revenue|top 10|top selling|best|chart|graph|plot|breakdown|analytics)\b/i.test(query)) {
7319
+ const q = query.toLowerCase().trim();
7320
+ if (/^(what|who|explain|explore|define|where|when|why|how|how does|can|could|does|do|is|are|has|have|will|should|would|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q)) {
7321
+ if (!/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q)) {
7083
7322
  return true;
7084
7323
  }
7085
7324
  }
7086
- if (/^(what is the )?(price|cost|fee|rate) of\b/i.test(query)) {
7325
+ if (/^(what is the )?(price|cost|fee|rate|ratio|value|amount) of\b/i.test(q)) {
7087
7326
  return true;
7088
7327
  }
7089
7328
  return false;
@@ -7098,13 +7337,13 @@ var IntentClassifier = class {
7098
7337
  return /\b(recommend|suggest|what should i (buy|get|choose)|best options|top picks)\b/i.test(query);
7099
7338
  }
7100
7339
  static isTrendQuery(query) {
7101
- return /\b(trend|trends|growth|over time|historical|timeline|monthly|yearly|weekly|daily|revenue growth|sales growth)\b/i.test(query);
7340
+ return /\b(trend|trends|growth over time|historical trend|monthly trend|yearly trend|trend line)\b/i.test(query);
7102
7341
  }
7103
7342
  static isRankingQuery(query) {
7104
7343
  return /\b(highest|lowest|top\s*\d+|bottom\s*\d+|rank|ranking|top performing|best performing)\b/i.test(query);
7105
7344
  }
7106
7345
  static isAnalyticsQuery(query) {
7107
- return /\b(sales|revenue|analytics|insights|distribution|percentage|share|breakdown|breakup|scatter|correlation|metrics|performance|by region|by category)\b/i.test(query);
7346
+ return /\b(analytics|insights|distribution of|percentage breakdown|market share|breakdown by|breakup of|scatter plot|correlation between|by region|by category|chart of|graph of)\b/i.test(query);
7108
7347
  }
7109
7348
  };
7110
7349
 
@@ -7116,8 +7355,8 @@ var Rule1SpecificInfoRule = class {
7116
7355
  }
7117
7356
  evaluate(context, intent) {
7118
7357
  const q = (context.userQuery || "").toLowerCase().trim();
7119
- const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of)\b/i.test(q);
7120
- const isNonVisual = !/\b(compare|versus|vs\.?|trend|monthly|revenue|best|top 10|chart|table|list|grid)\b/i.test(q);
7358
+ const isFactQuery = intent === "information_lookup" || /^(what|who|explain|explore|define|where|when|why|how|meaning of|tell me about|show details of|overview of|summarize)\s+/i.test(q) || /^(what is the price of|price of|cost of|ratio of|rate of|value of)\b/i.test(q);
7359
+ const isNonVisual = !/\b(chart|graph|plot|visualize|pie chart|bar chart|line chart|scatter plot|draw a)\b/i.test(q);
7121
7360
  if (isFactQuery && isNonVisual) {
7122
7361
  return {
7123
7362
  renderType: "text",
@@ -7182,11 +7421,11 @@ var Rule4AnalyticalRule = class {
7182
7421
  const isAnalytical = intent === "analytics" || intent === "trend_analysis" || intent === "ranking" || /\b(monthly sales|revenue growth|top performing|population trends|sales by|growth|distribution|share|percentage|breakdown|correlation|scatter|trend)\b/i.test(q);
7183
7422
  if (!isAnalytical) return null;
7184
7423
  let chartType = "bar";
7185
- if (intent === "trend_analysis" || /\b(monthly|yearly|weekly|growth|over time|trend|timeline|historical)\b/i.test(q)) {
7424
+ if (intent === "trend_analysis" || /\b(monthly trend|yearly trend|growth over time|trend line|timeline)\b/i.test(q)) {
7186
7425
  chartType = "line";
7187
- } else if (/\b(share|percentage|percent|breakup|breakdown|composition|pie|donut|proportion)\b/i.test(q)) {
7426
+ } else if (/\b(market share|share of|percentage breakdown|breakup of|composition of|pie chart|donut chart|proportion of)\b/i.test(q)) {
7188
7427
  chartType = "pie";
7189
- } else if (/\b(correlation|scatter|relationship|relation|impact of|depends on)\b/i.test(q)) {
7428
+ } else if (/\b(correlation between|scatter plot|relationship between)\b/i.test(q)) {
7190
7429
  chartType = "scatter";
7191
7430
  } else {
7192
7431
  chartType = "bar";
@@ -7949,12 +8188,20 @@ ${schemaProfileText}` : ""}`;
7949
8188
  var _a3;
7950
8189
  return String((_a3 = record.fields[dimension.key]) != null ? _a3 : "");
7951
8190
  }).filter(Boolean))) : this.detectCategories(data);
7952
- if (categories.length === 0) return null;
8191
+ if (categories.length <= 1) return null;
7953
8192
  const categoryData = dimension && profile ? this.aggregateProfileByDimension(profile, dimension.key) : this.aggregateByCategory(data, categories);
8193
+ const hasInventoryData = data.some((item) => {
8194
+ const meta = item.metadata || {};
8195
+ return Object.keys(meta).some((k) => ["stock", "in_stock", "inventory", "quantity", "closing_stock", "units_sold"].includes(k.toLowerCase()));
8196
+ });
7954
8197
  const pieData = Object.entries(categoryData).map(([label, count]) => {
8198
+ if (!hasInventoryData) {
8199
+ return { label, value: count };
8200
+ }
7955
8201
  const { inStockCount, outOfStockCount } = this.calculateStockCounts(label, data);
7956
8202
  return { label, value: count, inStockCount, outOfStockCount };
7957
8203
  });
8204
+ if (pieData.length <= 1) return null;
7958
8205
  return {
7959
8206
  type: "pie_chart",
7960
8207
  title: `Distribution by ${(_a2 = dimension == null ? void 0 : dimension.label) != null ? _a2 : "Category"}`,
@@ -8261,41 +8508,34 @@ ${schemaProfileText}` : ""}`;
8261
8508
  }
8262
8509
  static _productQueryTest(query) {
8263
8510
  const q = query.toLowerCase();
8264
- const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|price|prices|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
8511
+ const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|tax|depreciation|interest rate|cashflow|cash flow|fcfe|fcff|wacc|beta|working capital|cogs|capital spending|valuation|leverage)\b/;
8512
+ if (financialTerms.test(q)) {
8513
+ return false;
8514
+ }
8515
+ const productTerms = /\b(product|products|item|items|sku|catalog|catalogue|brand|model|stock|inventory|in stock|out of stock|buy|shop)\b/.test(q);
8265
8516
  const productAction = /\b(recommend|suggest|describe|description|detail|details|about|show|find|search|browse|list|explore|exploring)\b/.test(q);
8266
8517
  const nonProductEntityTerms = /\b(organization|organisations?|organizations?|company|companies|employee|employees|staff|headcount|workforce)\b/.test(q);
8267
8518
  return productTerms && productAction && !nonProductEntityTerms;
8268
8519
  }
8269
8520
  static isProductData(item) {
8270
8521
  if (!item) return false;
8271
- const content = (item.content || "").toLowerCase();
8272
- const productKeywords = [
8273
- "product",
8274
- "price",
8275
- "stock",
8276
- "sku",
8277
- "brand",
8278
- "msrp",
8279
- "inventory",
8280
- "buy",
8281
- "shop",
8282
- "beauty",
8283
- "cosmetic",
8284
- "facial",
8285
- "cream",
8286
- "serum",
8287
- "mask",
8288
- "makeup",
8289
- "fragrance"
8290
- ];
8291
- const hasKeywords = productKeywords.some((kw) => content.includes(kw));
8292
8522
  const metadata = item.metadata || {};
8293
- const hasMetadataKey = Object.keys(metadata).some((k) => {
8294
- const val = metadata[k];
8295
- return ["price", "product", "sku", "brand", "cost"].includes(k.toLowerCase()) && val !== null && val !== void 0 && val !== "";
8296
- });
8297
- const hasPricePattern = /\$\s*\d+\.\d{2}/.test(content);
8298
- return hasKeywords || hasMetadataKey || hasPricePattern;
8523
+ const domain = String(metadata.domain || "").toLowerCase();
8524
+ if (domain === "finance" || domain === "legal" || domain === "healthcare" || domain === "timeseries") {
8525
+ return false;
8526
+ }
8527
+ if (domain === "ecommerce" || metadata.type === "product" || metadata.isCatalog === true) {
8528
+ return true;
8529
+ }
8530
+ const content = (item.content || "").toLowerCase();
8531
+ const financialTerms = /\b(price per share|share price|total cost of the deal|cost of the deal|debt|equity|ratio|revenue|ebit|ebitda|fcfe|fcff|wacc|beta|working capital|cogs|valuation|leverage)\b/;
8532
+ if (financialTerms.test(content)) {
8533
+ return false;
8534
+ }
8535
+ const hasSku = Boolean(resolveMetadataValue(metadata, "sku") || resolveMetadataValue(metadata, "product_id"));
8536
+ const hasProductIdentity = Boolean(resolveMetadataValue(metadata, "product_name") || resolveMetadataValue(metadata, "brand") && resolveMetadataValue(metadata, "name"));
8537
+ const hasPrice = Boolean(resolveMetadataValue(metadata, "retail_price") || resolveMetadataValue(metadata, "unit_price") || resolveMetadataValue(metadata, "msrp"));
8538
+ return hasSku || hasProductIdentity && hasPrice;
8299
8539
  }
8300
8540
  static isTimeSeriesData(item) {
8301
8541
  const metadata = item.metadata || {};
@@ -8314,10 +8554,31 @@ ${schemaProfileText}` : ""}`;
8314
8554
  return fieldCount.size > 2;
8315
8555
  }
8316
8556
  static profileData(data) {
8557
+ const IGNORED_SYSTEM_KEYS = /* @__PURE__ */ new Set([
8558
+ "filetype",
8559
+ "mimetype",
8560
+ "filename",
8561
+ "docid",
8562
+ "uploadedat",
8563
+ "dimension",
8564
+ "chunkindex",
8565
+ "totalchunks",
8566
+ "charactercount",
8567
+ "workspaceid",
8568
+ "pineconenamespace",
8569
+ "tier",
8570
+ "custommetadata",
8571
+ "csvheaders",
8572
+ "headers",
8573
+ "filesize",
8574
+ "content",
8575
+ "id"
8576
+ ]);
8317
8577
  const records = (data || []).filter((item) => Boolean(item && typeof item === "object")).map((item) => {
8318
8578
  var _a2, _b;
8319
8579
  const fields2 = {};
8320
8580
  Object.entries(item.metadata || {}).forEach(([key, value]) => {
8581
+ if (IGNORED_SYSTEM_KEYS.has(key.toLowerCase())) return;
8321
8582
  const primitive = this.toPrimitive(value);
8322
8583
  if (primitive !== null) fields2[key] = primitive;
8323
8584
  });
@@ -9271,27 +9532,74 @@ ${m.content}`).join("\n\n---\n\n");
9271
9532
  * Ingest documents with automatic chunking, embedding, and batch upsert.
9272
9533
  */
9273
9534
  async ingest(documents, namespace) {
9535
+ var _a2;
9274
9536
  await this.initialize();
9275
9537
  const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
9276
9538
  const results = [];
9539
+ if (!documents || documents.length === 0) return results;
9540
+ const docChunksMap = [];
9541
+ const allChunks = [];
9277
9542
  for (const doc of documents) {
9278
9543
  try {
9279
9544
  const chunks = await this.prepareChunks(doc);
9280
- const vectors = await this.processEmbeddings(chunks);
9545
+ docChunksMap.push({ doc, chunks });
9546
+ allChunks.push(...chunks);
9547
+ } catch (error) {
9548
+ Logger.error(`[Pipeline] Failed to prepare chunks for document ${doc.docId}`, error);
9549
+ results.push({ docId: doc.docId, chunksIngested: 0 });
9550
+ }
9551
+ }
9552
+ if (allChunks.length === 0) {
9553
+ return results;
9554
+ }
9555
+ try {
9556
+ const allVectors = await this.processEmbeddings(allChunks);
9557
+ let vectorOffset = 0;
9558
+ const allUpsertDocs = [];
9559
+ const docSuccessCount = /* @__PURE__ */ new Map();
9560
+ for (const { doc, chunks } of docChunksMap) {
9561
+ const docVectors = allVectors.slice(vectorOffset, vectorOffset + chunks.length);
9562
+ vectorOffset += chunks.length;
9281
9563
  const upsertDocs = chunks.map((chunk, i) => ({
9282
9564
  id: chunk.id,
9283
- vector: vectors[i],
9565
+ vector: docVectors[i],
9284
9566
  content: chunk.content,
9285
9567
  metadata: chunk.metadata
9286
9568
  }));
9287
- const totalProcessed = await this.processUpserts(upsertDocs, ns);
9288
- results.push({ docId: doc.docId, chunksIngested: totalProcessed });
9569
+ allUpsertDocs.push(...upsertDocs);
9570
+ docSuccessCount.set(doc.docId, chunks.length);
9571
+ }
9572
+ const totalProcessed = await this.processUpserts(allUpsertDocs, ns);
9573
+ for (const { doc, chunks } of docChunksMap) {
9574
+ results.push({ docId: doc.docId, chunksIngested: (_a2 = docSuccessCount.get(doc.docId)) != null ? _a2 : chunks.length });
9289
9575
  if (this.graphDB && this.entityExtractor) {
9290
- await this.processGraphIngestion(doc.docId, chunks);
9576
+ this.processGraphIngestion(doc.docId, chunks).catch((err) => {
9577
+ Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
9578
+ });
9579
+ }
9580
+ }
9581
+ } catch (batchErr) {
9582
+ Logger.error(`[Pipeline] Batch embedding/upsert failed, falling back to individual ingestion`, batchErr);
9583
+ for (const { doc, chunks } of docChunksMap) {
9584
+ try {
9585
+ const vectors = await this.processEmbeddings(chunks);
9586
+ const upsertDocs = chunks.map((chunk, i) => ({
9587
+ id: chunk.id,
9588
+ vector: vectors[i],
9589
+ content: chunk.content,
9590
+ metadata: chunk.metadata
9591
+ }));
9592
+ const totalProcessed = await this.processUpserts(upsertDocs, ns);
9593
+ results.push({ docId: doc.docId, chunksIngested: totalProcessed });
9594
+ if (this.graphDB && this.entityExtractor) {
9595
+ this.processGraphIngestion(doc.docId, chunks).catch((err) => {
9596
+ Logger.warn(`[Pipeline] Background graph ingestion failed for doc ${doc.docId}`, { error: String(err) });
9597
+ });
9598
+ }
9599
+ } catch (singleErr) {
9600
+ Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, singleErr);
9601
+ results.push({ docId: doc.docId, chunksIngested: 0 });
9291
9602
  }
9292
- } catch (error) {
9293
- Logger.error(`[Pipeline] Failed to ingest document ${doc.docId}`, error);
9294
- results.push({ docId: doc.docId, chunksIngested: 0 });
9295
9603
  }
9296
9604
  }
9297
9605
  return results;
@@ -9311,19 +9619,40 @@ ${m.content}`).join("\n\n---\n\n");
9311
9619
  }
9312
9620
  /**
9313
9621
  * Step 2: Generate embeddings for chunks with retry logic.
9314
- * Uses batchEmbed when available for efficiency; falls back to sequential embedding.
9622
+ * Uses batchEmbed when available for efficiency; falls back to concurrent embedding.
9315
9623
  */
9316
9624
  async processEmbeddings(chunks) {
9317
- const embedBatchOptions = {
9318
- batchSize: 50,
9319
- maxRetries: 3,
9320
- initialDelayMs: 100
9321
- };
9322
- const vectors = await BatchProcessor.mapWithRetry(
9323
- chunks.map((c) => c.content),
9625
+ if (chunks.length === 0) return [];
9626
+ const texts = chunks.map((c) => c.content);
9627
+ if (typeof this.embeddingProvider.batchEmbed === "function") {
9628
+ try {
9629
+ const batchSize = 50;
9630
+ const allVectors = [];
9631
+ for (let i = 0; i < texts.length; i += batchSize) {
9632
+ const slice = texts.slice(i, i + batchSize);
9633
+ const batchVectors = await this.embeddingProvider.batchEmbed(slice, { taskType: "document" });
9634
+ allVectors.push(...batchVectors);
9635
+ }
9636
+ if (allVectors.length === chunks.length) {
9637
+ return allVectors;
9638
+ }
9639
+ } catch (err) {
9640
+ Logger.warn("[Pipeline] batchEmbed failed, falling back to concurrent embedding:", {
9641
+ error: err instanceof Error ? err.message : String(err)
9642
+ });
9643
+ }
9644
+ }
9645
+ const concurrentResult = await BatchProcessor.processConcurrent(
9646
+ texts,
9324
9647
  (text) => this.embeddingProvider.embed(text, { taskType: "document" }),
9325
- embedBatchOptions
9648
+ 10,
9649
+ {
9650
+ maxRetries: 3,
9651
+ initialDelayMs: 100,
9652
+ throwOnPartialFailure: false
9653
+ }
9326
9654
  );
9655
+ const vectors = concurrentResult.results;
9327
9656
  if (vectors.length !== chunks.length) {
9328
9657
  throw new EmbeddingFailedException(
9329
9658
  `[Pipeline] Embedding mismatch: got ${vectors.length} vectors for ${chunks.length} chunks. Check embedding provider logs for individual chunk failures.`
@@ -9350,25 +9679,24 @@ ${m.content}`).join("\n\n---\n\n");
9350
9679
  }
9351
9680
  /** Step 4: Optional graph-based entity extraction and ingestion. */
9352
9681
  async processGraphIngestion(docId, chunks) {
9353
- const extractionOptions = {
9354
- batchSize: 2,
9355
- maxRetries: 1,
9356
- initialDelayMs: 500
9357
- };
9358
- await BatchProcessor.processBatch(
9682
+ const maxConcurrency = Math.min(3, chunks.length);
9683
+ await BatchProcessor.processConcurrent(
9359
9684
  chunks,
9360
- async (batch) => {
9361
- for (const chunk of batch) {
9362
- try {
9363
- const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
9364
- if (nodes.length > 0) await this.graphDB.addNodes(nodes);
9365
- if (edges.length > 0) await this.graphDB.addEdges(edges);
9366
- } catch (err) {
9367
- Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
9368
- }
9685
+ async (chunk) => {
9686
+ try {
9687
+ const { nodes, edges } = await this.entityExtractor.extract(chunk.content);
9688
+ if (nodes.length > 0) await this.graphDB.addNodes(nodes);
9689
+ if (edges.length > 0) await this.graphDB.addEdges(edges);
9690
+ } catch (err) {
9691
+ Logger.warn(`[Pipeline] Entity extraction failed for chunk in doc ${docId}`, { error: String(err) });
9369
9692
  }
9370
9693
  },
9371
- extractionOptions
9694
+ maxConcurrency,
9695
+ {
9696
+ maxRetries: 1,
9697
+ initialDelayMs: 200,
9698
+ throwOnPartialFailure: false
9699
+ }
9372
9700
  );
9373
9701
  }
9374
9702
  async runNormalQuery(question, history = [], namespace) {
@@ -9493,7 +9821,7 @@ ${m.content}`).join("\n\n---\n\n");
9493
9821
  */
9494
9822
  askStreamInternal(_0) {
9495
9823
  return __asyncGenerator(this, arguments, function* (question, history = [], namespace) {
9496
- var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A;
9824
+ var _a2, _b, _c, _d, _e, _f, _g2, _h, _i, _j, _k, _l, _m, _n, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _A, _B;
9497
9825
  yield new __await(this.initialize());
9498
9826
  const ns = formatNamespace(namespace != null ? namespace : this.config.projectId);
9499
9827
  const topK = (_b = (_a2 = this.config.rag) == null ? void 0 : _a2.topK) != null ? _b : 5;
@@ -9501,29 +9829,35 @@ ${m.content}`).join("\n\n---\n\n");
9501
9829
  const requestId = `req_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
9502
9830
  const requestStart = performance.now();
9503
9831
  try {
9504
- let searchQuery = question;
9832
+ let effectiveQuestion = question;
9833
+ let promptOptResult;
9834
+ if (((_e = this.config.rag) == null ? void 0 : _e.optimizePrompt) !== false) {
9835
+ promptOptResult = PromptOptimizer.optimize(question, history);
9836
+ effectiveQuestion = promptOptResult.optimizedPrompt;
9837
+ }
9838
+ let searchQuery = effectiveQuestion;
9505
9839
  let rewrittenQuery;
9506
- if ((_e = this.config.rag) == null ? void 0 : _e.useQueryTransformation) {
9507
- searchQuery = yield new __await(this.rewriteQuery(question, history));
9508
- rewrittenQuery = searchQuery !== question ? searchQuery : void 0;
9840
+ if ((_f = this.config.rag) == null ? void 0 : _f.useQueryTransformation) {
9841
+ searchQuery = yield new __await(this.rewriteQuery(effectiveQuestion, history));
9842
+ rewrittenQuery = searchQuery !== effectiveQuestion ? searchQuery : void 0;
9509
9843
  }
9510
- const hints = QueryProcessor.extractQueryFieldHints(question, (_f = this.config.rag) == null ? void 0 : _f.filterableFields);
9511
- const filter = QueryProcessor.buildQueryFilter(question, hints);
9512
- const numericPredicates = QueryProcessor.extractNumericPredicates(question, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
9844
+ const hints = QueryProcessor.extractQueryFieldHints(effectiveQuestion, (_g2 = this.config.rag) == null ? void 0 : _g2.filterableFields);
9845
+ const filter = QueryProcessor.buildQueryFilter(effectiveQuestion, hints);
9846
+ const numericPredicates = QueryProcessor.extractNumericPredicates(effectiveQuestion, (_h = this.config.rag) == null ? void 0 : _h.filterableFields);
9513
9847
  if (numericPredicates.length > 0) {
9514
9848
  filter.__numericPredicates = numericPredicates;
9515
9849
  }
9516
9850
  const embedStart = performance.now();
9517
9851
  const cacheKey = `${ns}::${searchQuery}`;
9518
9852
  const cachedVector = this.embeddingCache.get(cacheKey);
9519
- const arch = (_h = this.config.rag) == null ? void 0 : _h.architecture;
9520
- const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_i = this.config.rag) == null ? void 0 : _i.useGraphRetrieval);
9853
+ const arch = (_i = this.config.rag) == null ? void 0 : _i.architecture;
9854
+ const useGraph = arch !== "naive" && arch !== "retrieve-and-rerank" && this.graphDB && ((_j = this.config.rag) == null ? void 0 : _j.useGraphRetrieval);
9521
9855
  const [strategyResult, embeddedVector] = yield new __await(Promise.all([
9522
9856
  QueryProcessor.determineRetrievalStrategy(
9523
9857
  searchQuery,
9524
9858
  useGraph ? this.llmRouter.get("fast") : void 0,
9525
- (_j = this.config.rag) == null ? void 0 : _j.graphKeywords,
9526
- (_k = this.config.rag) == null ? void 0 : _k.vectorKeywords
9859
+ (_k = this.config.rag) == null ? void 0 : _k.graphKeywords,
9860
+ (_l = this.config.rag) == null ? void 0 : _l.vectorKeywords
9527
9861
  ),
9528
9862
  // Embed immediately regardless of strategy — costs nothing if cached.
9529
9863
  // If strategy turns out to be 'graph'-only we just won't use the vector.
@@ -9533,7 +9867,7 @@ ${m.content}`).join("\n\n---\n\n");
9533
9867
  if (!cachedVector && queryVector.length > 0) {
9534
9868
  this.embeddingCache.set(cacheKey, queryVector);
9535
9869
  }
9536
- const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_l = this.config.rag) == null ? void 0 : _l.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
9870
+ const graphData = (strategyResult === "graph" || strategyResult === "both") && this.graphDB && ((_m = this.config.rag) == null ? void 0 : _m.useGraphRetrieval) ? yield new __await(this.graphDB.query(searchQuery)) : void 0;
9537
9871
  const hasMetadataFilter = filter.metadata && Object.keys(filter.metadata).length > 0;
9538
9872
  const hasNumericPredicates = Array.isArray(filter.__numericPredicates) && filter.__numericPredicates.length > 0;
9539
9873
  const wantsExhaustiveList = true;
@@ -9550,7 +9884,7 @@ ${m.content}`).join("\n\n---\n\n");
9550
9884
  return ((_a3 = m == null ? void 0 : m.score) != null ? _a3 : 0) >= scoreThreshold;
9551
9885
  });
9552
9886
  const rerankLimit = Math.max(retrievalLimit, fullSources.length);
9553
- const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_m = this.config.rag) == null ? void 0 : _m.useReranking));
9887
+ const useReranking = arch !== "naive" && (arch === "retrieve-and-rerank" || ((_n = this.config.rag) == null ? void 0 : _n.useReranking));
9554
9888
  if (!hasNumericPredicates && useReranking) {
9555
9889
  fullSources = (yield new __await(this.reranker.rerank(fullSources, question, rerankLimit))).filter((s) => Boolean(s && typeof s === "object"));
9556
9890
  }
@@ -9578,7 +9912,7 @@ ${graphContext}
9578
9912
  VECTOR CONTEXT:
9579
9913
  ${context}`;
9580
9914
  }
9581
- if ((_o = (_n = this.config.rag) == null ? void 0 : _n.cag) == null ? void 0 : _o.enabled) {
9915
+ if ((_p = (_o = this.config.rag) == null ? void 0 : _o.cag) == null ? void 0 : _p.enabled) {
9582
9916
  if (!this.coldContexts.has(ns)) {
9583
9917
  yield new __await(this.loadColdContext(ns));
9584
9918
  }
@@ -9610,31 +9944,31 @@ ${context}`;
9610
9944
  SchemaMapper.train(this.llmProvider, ns, allMetadataKeys).catch(() => {
9611
9945
  });
9612
9946
  }
9613
- const isProductQ = UITransformer.isProductQueryPublic(question);
9947
+ const isProductQ = UITransformer.isProductQueryPublic(effectiveQuestion);
9614
9948
  const uiTransformationPromise = isProductQ ? Promise.resolve(
9615
9949
  UITransformer.transform(
9616
- question,
9950
+ effectiveQuestion,
9617
9951
  sources,
9618
9952
  this.config,
9619
9953
  cachedSchema,
9620
9954
  { visualizationHint: "product_browse", recommendedChart: "text", filterInStockOnly: false, wantsExplicitTable: false, isTemporal: false, isComparison: false, language: "en" }
9621
9955
  )
9622
9956
  ) : this.generateUiTransformation(
9623
- question,
9957
+ effectiveQuestion,
9624
9958
  sources,
9625
9959
  cachedSchema,
9626
9960
  hasNumericPredicates
9627
9961
  ).catch((uiError) => {
9628
9962
  Logger.warn("[Pipeline] UI transformation failed concurrently", { error: String(uiError) });
9629
- return UITransformer.transform(question, sources, this.config, cachedSchema);
9963
+ return UITransformer.transform(effectiveQuestion, sources, this.config, cachedSchema);
9630
9964
  });
9631
9965
  let hallucinationScoringPromise = Promise.resolve(void 0);
9632
9966
  const restrictionSuffix = "\n\n(IMPORTANT: Format your response beautifully using rich Markdown! Use bolding for emphasis, headings (##) for structure, bullet points for lists, and proper line breaks between paragraphs to make it highly readable. However, NEVER generate Markdown tables, HTML figures, or text charts (the UI renders requested charts separately, so NEVER say you cannot create, display, draw, render, or provide a chart/visualization). If listing products, use simple markdown bullet points or comma-separated names. NEVER use plus signs (+) to separate product names, category names, or list items. For product description/detail questions, provide customer-facing prose only and do NOT list internal catalog fields such as Handle, Body (HTML), Vendor, Type, Tags, Published, Option, Variant, SKU, or raw metadata labels. You CAN use numbers for years/counts like 2006 or 5800, but NEVER put a dollar sign ($) before them.)";
9633
9967
  const isClaude37 = this.config.llm.model.includes("claude-3-7-sonnet");
9634
- const isNativeThinking = isClaude37 && (((_p = this.config.llm.options) == null ? void 0 : _p.thinking) === true || ((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === "enabled" || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) !== false);
9968
+ const isNativeThinking = isClaude37 && (((_q = this.config.llm.options) == null ? void 0 : _q.thinking) === true || ((_r = this.config.llm.options) == null ? void 0 : _r.thinking) === "enabled" || ((_s = this.config.llm.options) == null ? void 0 : _s.thinking) !== false);
9635
9969
  const modelNameLower = this.config.llm.model.toLowerCase();
9636
9970
  const isReasoningModel = modelNameLower.includes("-r1") || modelNameLower.includes("deepseek-r1") || modelNameLower.includes("reasoning") || modelNameLower.includes("thinking");
9637
- const isSimulatedThinking = !isNativeThinking && (((_s = this.config.llm.options) == null ? void 0 : _s.thinking) === true || ((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === "enabled" || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) !== false);
9971
+ const isSimulatedThinking = !isNativeThinking && (((_t = this.config.llm.options) == null ? void 0 : _t.thinking) === true || ((_u = this.config.llm.options) == null ? void 0 : _u.thinking) === "enabled" || ((_v = this.config.llm.options) == null ? void 0 : _v.thinking) !== false);
9638
9972
  let finalRestrictionSuffix = restrictionSuffix;
9639
9973
  if (isSimulatedThinking) {
9640
9974
  finalRestrictionSuffix += "\n\n(IMPORTANT: You must think step-by-step before answering. Write your thinking process inside a `<think>...</think>` block. Write the `<think>` block first, then follow it with your final response. Keep the thinking process thorough and detailed. Example: `<think>Thinking details here...</think>Final answer text here.`)";
@@ -9643,9 +9977,9 @@ ${context}`;
9643
9977
  role: m.role || "user",
9644
9978
  content: typeof m.content === "string" ? m.content : m.content != null ? String(m.content) : ""
9645
9979
  }));
9646
- const userQuestion = { role: "user", content: question + finalRestrictionSuffix };
9980
+ const userQuestion = { role: "user", content: effectiveQuestion + finalRestrictionSuffix };
9647
9981
  const messages = [...hardenedHistory, userQuestion];
9648
- const systemPrompt = (_v = this.config.llm.systemPrompt) != null ? _v : "";
9982
+ const systemPrompt = (_w = this.config.llm.systemPrompt) != null ? _w : "";
9649
9983
  const userPrompt = messages.map((m) => `${m.role}: ${m.content}`).join("\n");
9650
9984
  let fullReply = "";
9651
9985
  let thinkingText = "";
@@ -9756,7 +10090,7 @@ ${context}`;
9756
10090
  }
9757
10091
  yield fullReply;
9758
10092
  }
9759
- const runHallucination = ((_w = this.config.llm.options) == null ? void 0 : _w.hallucinationScoring) === true;
10093
+ const runHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.hallucinationScoring) === true;
9760
10094
  if (runHallucination) {
9761
10095
  hallucinationScoringPromise = scoreHallucination(this.llmProvider, fullReply, context).catch(() => void 0);
9762
10096
  }
@@ -9778,7 +10112,7 @@ ${context}`;
9778
10112
  totalTokens: promptTokens + completionTokens,
9779
10113
  estimatedCostUsd: estimateCostUsd(promptTokens, completionTokens, this.config.llm.model)
9780
10114
  };
9781
- const awaitHallucination = ((_x = this.config.llm.options) == null ? void 0 : _x.awaitHallucination) === true;
10115
+ const awaitHallucination = ((_y = this.config.llm.options) == null ? void 0 : _y.awaitHallucination) === true;
9782
10116
  const [uiTransformation, hallucinationResult] = yield new __await(Promise.all([
9783
10117
  uiTransformationPromise,
9784
10118
  awaitHallucination ? hallucinationScoringPromise : Promise.resolve(void 0)
@@ -9786,9 +10120,9 @@ ${context}`;
9786
10120
  const buildTrace = (hScore) => __spreadValues({
9787
10121
  requestId,
9788
10122
  query: question,
9789
- rewrittenQuery,
10123
+ rewrittenQuery: rewrittenQuery || ((promptOptResult == null ? void 0 : promptOptResult.wasModified) ? promptOptResult.optimizedPrompt : void 0),
9790
10124
  systemPrompt,
9791
- userPrompt: question + finalRestrictionSuffix,
10125
+ userPrompt: effectiveQuestion + finalRestrictionSuffix,
9792
10126
  chunks: (sources || []).filter(Boolean).map((s) => {
9793
10127
  var _a3, _b2;
9794
10128
  return {
@@ -9807,10 +10141,10 @@ ${context}`;
9807
10141
  hallucinationReason: hScore.reason
9808
10142
  } : {});
9809
10143
  const trace = buildTrace(hallucinationResult);
9810
- const isTelemetryActive = (_z = (_y = this.config.telemetry) == null ? void 0 : _y.enabled) != null ? _z : Boolean(this.config.licenseKey);
10144
+ const isTelemetryActive = (_A = (_z = this.config.telemetry) == null ? void 0 : _z.enabled) != null ? _A : Boolean(this.config.licenseKey);
9811
10145
  if (isTelemetryActive) {
9812
10146
  const defaultUrl = "https://www.retrivora.com/api/telemetry";
9813
- const telemetryUrl = ((_A = this.config.telemetry) == null ? void 0 : _A.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
10147
+ const telemetryUrl = ((_B = this.config.telemetry) == null ? void 0 : _B.url) || process.env.TELEMETRY_URL || process.env.NEXT_PUBLIC_TELEMETRY_URL || defaultUrl;
9814
10148
  const absoluteUrl = telemetryUrl.startsWith("http") ? telemetryUrl : "https://www.retrivora.com" + (telemetryUrl.startsWith("/") ? telemetryUrl : "/" + telemetryUrl);
9815
10149
  (async () => {
9816
10150
  var _a3, _b2, _c2, _d2, _e2;
@@ -10386,13 +10720,81 @@ var VectorPlugin = class {
10386
10720
 
10387
10721
  // src/utils/DocumentParser.ts
10388
10722
  var DocumentParser = class {
10723
+ /**
10724
+ * Fast Magic-Bytes sniffer for instant file format identification (<1ms).
10725
+ */
10726
+ static sniffMagicBytes(buffer) {
10727
+ if (!buffer || buffer.length < 4) return null;
10728
+ if (buffer[0] === 37 && buffer[1] === 80 && buffer[2] === 68 && buffer[3] === 70) {
10729
+ return "pdf";
10730
+ }
10731
+ if (buffer[0] === 80 && buffer[1] === 75 && buffer[2] === 3 && buffer[3] === 4) {
10732
+ return "zip";
10733
+ }
10734
+ if (buffer[0] === 208 && buffer[1] === 207 && buffer[2] === 17 && buffer[3] === 224) {
10735
+ return "ole2";
10736
+ }
10737
+ const firstNonWhitespace = buffer.subarray(0, 32).toString("utf-8").trim();
10738
+ if (firstNonWhitespace.startsWith("{") || firstNonWhitespace.startsWith("[")) {
10739
+ return "json";
10740
+ }
10741
+ return "text";
10742
+ }
10743
+ /**
10744
+ * Convert spreadsheet sheets into high-fidelity, context-preserved tabular text.
10745
+ * Every row retains its column and section context so vector embeddings achieve 100% semantic retrieval precision.
10746
+ */
10747
+ static formatSheetToContextualText(sheet, sheetName, XLSX) {
10748
+ const jsonRows = XLSX.utils.sheet_to_json(sheet, { header: 1, blankrows: false });
10749
+ if (!jsonRows || jsonRows.length === 0) {
10750
+ const csv = XLSX.utils.sheet_to_csv(sheet, { blankrows: false });
10751
+ return csv && csv.trim() ? `--- Sheet: ${sheetName} ---
10752
+ ${csv.trim()}` : "";
10753
+ }
10754
+ const lines = [`--- Sheet: ${sheetName} ---`];
10755
+ let currentHeader = [];
10756
+ for (let r = 0; r < jsonRows.length; r++) {
10757
+ const row = jsonRows[r];
10758
+ if (!row || row.length === 0) continue;
10759
+ const cells = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "").filter(Boolean);
10760
+ if (cells.length === 0) continue;
10761
+ if (cells.length === 1 || cells.length <= 2 && cells[0].toUpperCase().includes("STEP")) {
10762
+ lines.push(`
10763
+ ## ${cells.join(" ")}`);
10764
+ continue;
10765
+ }
10766
+ const isHeaderCandidate = cells.every((c) => isNaN(Number(c)) && !/^\$?[\d,.]+/.test(c));
10767
+ if (isHeaderCandidate && cells.length >= 2) {
10768
+ currentHeader = row.map((c) => c !== null && c !== void 0 ? String(c).trim() : "");
10769
+ lines.push(`| ${cells.join(" | ")} |`);
10770
+ lines.push(`| ${cells.map(() => "---").join(" | ")} |`);
10771
+ continue;
10772
+ }
10773
+ if (currentHeader.length > 0 && cells.length > 1) {
10774
+ const fieldPairs = [];
10775
+ for (let c = 0; c < row.length; c++) {
10776
+ const h = currentHeader[c];
10777
+ const val = row[c];
10778
+ if (h && val !== null && val !== void 0 && String(val).trim() !== "") {
10779
+ fieldPairs.push(`${h}: ${String(val).trim()}`);
10780
+ }
10781
+ }
10782
+ if (fieldPairs.length > 0) {
10783
+ lines.push(fieldPairs.join(" | "));
10784
+ continue;
10785
+ }
10786
+ }
10787
+ lines.push(cells.join(", "));
10788
+ }
10789
+ return lines.join("\n");
10790
+ }
10389
10791
  /**
10390
10792
  * Extract text from a File or Buffer based on its type.
10391
10793
  */
10392
- static async parse(file, fileName, mimeType) {
10794
+ static async parse(file, fileName, mimeType = "") {
10393
10795
  var _a2;
10394
10796
  const extension = ((_a2 = fileName.split(".").pop()) == null ? void 0 : _a2.toLowerCase()) || "";
10395
- if (extension === "txt" || extension === "md" || mimeType === "text/plain" || mimeType === "text/markdown") {
10797
+ if (extension === "txt" || extension === "md" || extension === "log" || mimeType === "text/plain" || mimeType === "text/markdown") {
10396
10798
  return this.readAsText(file);
10397
10799
  }
10398
10800
  if (extension === "json" || mimeType === "application/json") {
@@ -10404,38 +10806,115 @@ var DocumentParser = class {
10404
10806
  return text;
10405
10807
  }
10406
10808
  }
10407
- if (extension === "csv" || mimeType === "text/csv") {
10809
+ if (extension === "yaml" || extension === "yml" || mimeType.includes("yaml") || mimeType.includes("yml")) {
10810
+ return this.readAsText(file);
10811
+ }
10812
+ if (extension === "csv" || extension === "tsv" || mimeType === "text/csv" || mimeType === "text/tab-separated-values") {
10408
10813
  return this.readAsText(file);
10409
10814
  }
10410
- if (extension === "xlsx" || extension === "xls" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
10815
+ if (extension === "pptx" || extension === "ppt" || extension === "odp" || mimeType.includes("presentation") || mimeType.includes("powerpoint")) {
10411
10816
  try {
10412
- const ExcelJS = await import("exceljs");
10817
+ const JSZip = (await import("jszip")).default || await import("jszip");
10413
10818
  const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10414
- const workbook = new ExcelJS.Workbook();
10415
- await workbook.xlsx.load(buffer);
10416
- const sheetsText = [];
10417
- workbook.eachSheet((sheet, sheetId) => {
10418
- const csvLines = [];
10419
- sheet.eachRow((row, rowNumber) => {
10420
- const values = row.values;
10421
- if (Array.isArray(values)) {
10422
- csvLines.push(values.slice(1).join(","));
10819
+ const zip = await JSZip.loadAsync(buffer);
10820
+ const slideFiles = Object.keys(zip.files).filter((name) => /^ppt\/slides\/slide\d+\.xml$/i.test(name)).sort((a, b) => {
10821
+ var _a3, _b;
10822
+ const numA = parseInt(((_a3 = a.match(/\d+/)) == null ? void 0 : _a3[0]) || "0", 10);
10823
+ const numB = parseInt(((_b = b.match(/\d+/)) == null ? void 0 : _b[0]) || "0", 10);
10824
+ return numA - numB;
10825
+ });
10826
+ if (slideFiles.length > 0) {
10827
+ const slidesText = [];
10828
+ for (let i = 0; i < slideFiles.length; i++) {
10829
+ const slideXml = await zip.files[slideFiles[i]].async("text");
10830
+ const textMatches = slideXml.match(/<a:t[^>]*>([\s\S]*?)<\/a:t>/gi);
10831
+ if (textMatches && textMatches.length > 0) {
10832
+ const slideContent = textMatches.map((t) => t.replace(/<[^>]+>/g, "").trim()).filter(Boolean).join(" ");
10833
+ if (slideContent.trim()) {
10834
+ slidesText.push(`--- Slide ${i + 1} ---
10835
+ ${slideContent.trim()}`);
10836
+ }
10423
10837
  }
10424
- });
10425
- const csv = csvLines.join("\n");
10426
- if (csv && csv.trim()) {
10427
- sheetsText.push(`--- Sheet: ${sheet.name} ---
10428
- ${csv.trim()}`);
10429
10838
  }
10430
- });
10431
- return sheetsText.join("\n\n") || `[Empty Excel Workbook: ${fileName}]`;
10432
- } catch (e) {
10839
+ if (slidesText.length > 0) {
10840
+ return slidesText.join("\n\n");
10841
+ }
10842
+ }
10843
+ if (zip.files["content.xml"]) {
10844
+ const contentXml = await zip.files["content.xml"].async("text");
10845
+ const cleanText = this.stripXmlTags(contentXml);
10846
+ if (cleanText.trim()) {
10847
+ return cleanText.trim();
10848
+ }
10849
+ }
10850
+ const allXmlFiles = Object.keys(zip.files).filter((f) => f.endsWith(".xml") && (f.includes("slide") || f.includes("notes")));
10851
+ const extractedSegments = [];
10852
+ for (const xf of allXmlFiles) {
10853
+ const xml = await zip.files[xf].async("text");
10854
+ const stripped = this.stripXmlTags(xml);
10855
+ if (stripped.length > 10) extractedSegments.push(stripped);
10856
+ }
10857
+ if (extractedSegments.length > 0) {
10858
+ return extractedSegments.join("\n\n");
10859
+ }
10860
+ return `[PowerPoint Presentation: ${fileName}]`;
10861
+ } catch (err) {
10862
+ console.warn("[DocumentParser] PowerPoint parsing failed:", err);
10433
10863
  try {
10434
10864
  return await this.readAsText(file);
10435
- } catch (e2) {
10436
- return `[Excel File Content: ${fileName}]`;
10865
+ } catch (e) {
10866
+ return `[PowerPoint Presentation File: ${fileName}]`;
10867
+ }
10868
+ }
10869
+ }
10870
+ if (extension === "xlsx" || extension === "xls" || extension === "ods" || mimeType.includes("spreadsheet") || mimeType.includes("excel")) {
10871
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10872
+ try {
10873
+ const XLSX = await import("xlsx");
10874
+ const workbook = XLSX.read(buffer, { type: "buffer" });
10875
+ const sheetsText = [];
10876
+ for (const sheetName of workbook.SheetNames) {
10877
+ const sheet = workbook.Sheets[sheetName];
10878
+ const formatted = this.formatSheetToContextualText(sheet, sheetName, XLSX);
10879
+ if (formatted && formatted.trim()) {
10880
+ sheetsText.push(formatted.trim());
10881
+ }
10882
+ }
10883
+ if (sheetsText.length > 0) {
10884
+ return sheetsText.join("\n\n");
10437
10885
  }
10886
+ } catch (xlsxErr) {
10887
+ console.warn("[DocumentParser] SheetJS (xlsx) parsing failed, attempting ExcelJS:", xlsxErr);
10438
10888
  }
10889
+ const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
10890
+ if (isZip) {
10891
+ try {
10892
+ const ExcelJS = await import("exceljs");
10893
+ const workbook = new ExcelJS.Workbook();
10894
+ await workbook.xlsx.load(buffer);
10895
+ const sheetsText = [];
10896
+ workbook.eachSheet((sheet) => {
10897
+ const csvLines = [];
10898
+ sheet.eachRow((row) => {
10899
+ const values = row.values;
10900
+ if (Array.isArray(values)) {
10901
+ csvLines.push(values.slice(1).join(","));
10902
+ }
10903
+ });
10904
+ const csv = csvLines.join("\n");
10905
+ if (csv && csv.trim()) {
10906
+ sheetsText.push(`--- Sheet: ${sheet.name} ---
10907
+ ${csv.trim()}`);
10908
+ }
10909
+ });
10910
+ if (sheetsText.length > 0) {
10911
+ return sheetsText.join("\n\n");
10912
+ }
10913
+ } catch (err) {
10914
+ console.warn("[DocumentParser] ExcelJS OpenXML parsing failed, using legacy fallback:", err);
10915
+ }
10916
+ }
10917
+ return this.parseLegacyExcelOrFallback(buffer, fileName);
10439
10918
  }
10440
10919
  if (extension === "pdf" || mimeType === "application/pdf") {
10441
10920
  try {
@@ -10449,15 +10928,48 @@ ${csv.trim()}`);
10449
10928
  return `[PDF Parsing Error for ${fileName}]`;
10450
10929
  }
10451
10930
  }
10452
- if (extension === "docx" || extension === "doc" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword") {
10931
+ if (extension === "docx" || extension === "doc" || extension === "odt" || extension === "rtf" || mimeType === "application/vnd.openxmlformats-officedocument.wordprocessingml.document" || mimeType === "application/msword" || mimeType === "application/rtf") {
10453
10932
  try {
10454
10933
  const mammoth = await import("mammoth");
10455
10934
  const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10456
10935
  const result = await mammoth.extractRawText({ buffer });
10457
- return result.value;
10936
+ if (result.value && result.value.trim()) {
10937
+ return result.value;
10938
+ }
10939
+ } catch (err) {
10940
+ console.warn("[DocumentParser] Word parsing failed via mammoth, attempting fallback text extraction.", err);
10941
+ }
10942
+ try {
10943
+ const raw = await this.readAsText(file);
10944
+ if (extension === "rtf" || raw.startsWith("{\\rtf")) {
10945
+ return raw.replace(/\\par[d]?/g, "\n").replace(/\\line/g, "\n").replace(/\\tab/g, " ").replace(/\\'[0-9a-fA-F]{2}/g, "").replace(/\\[a-zA-Z0-9\-]+/g, "").replace(/[{}]/g, "").replace(/\n\s*\n/g, "\n\n").trim();
10946
+ }
10947
+ return raw;
10948
+ } catch (e) {
10949
+ return `[Word Document File: ${fileName}]`;
10950
+ }
10951
+ }
10952
+ if (extension === "html" || extension === "htm" || extension === "xml" || mimeType.includes("html") || mimeType.includes("xml")) {
10953
+ const raw = await this.readAsText(file);
10954
+ return this.stripHtmlAndXml(raw);
10955
+ }
10956
+ if (extension === "epub" || mimeType.includes("epub")) {
10957
+ try {
10958
+ const JSZip = (await import("jszip")).default || await import("jszip");
10959
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
10960
+ const zip = await JSZip.loadAsync(buffer);
10961
+ const htmlFiles = Object.keys(zip.files).filter((name) => /\.(html|xhtml|xml)$/i.test(name));
10962
+ const chapters = [];
10963
+ for (const hf of htmlFiles) {
10964
+ const content = await zip.files[hf].async("text");
10965
+ const clean = this.stripHtmlAndXml(content);
10966
+ if (clean.length > 50) chapters.push(clean);
10967
+ }
10968
+ if (chapters.length > 0) {
10969
+ return chapters.join("\n\n---\n\n");
10970
+ }
10458
10971
  } catch (err) {
10459
- console.warn('[DocumentParser] Word parsing failed. Make sure "mammoth" is installed.', err);
10460
- return `[Word Parsing Error for ${fileName}]`;
10972
+ console.warn("[DocumentParser] EPUB parsing failed:", err);
10461
10973
  }
10462
10974
  }
10463
10975
  try {
@@ -10466,6 +10978,237 @@ ${csv.trim()}`);
10466
10978
  throw new Error(`[DocumentParser] Unsupported file format: ${fileName} (${mimeType})`);
10467
10979
  }
10468
10980
  }
10981
+ /**
10982
+ * Helper to strip XML tags and normalize inner text content.
10983
+ */
10984
+ static stripXmlTags(xml) {
10985
+ return xml.replace(/<[^>]+>/g, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/&apos;/g, "'").replace(/\s+/g, " ").trim();
10986
+ }
10987
+ /**
10988
+ * Helper to strip HTML / XML tags while removing script and style blocks completely.
10989
+ */
10990
+ static stripHtmlAndXml(html) {
10991
+ return html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, "").replace(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/gi, "").replace(/<head\b[^<]*(?:(?!<\/head>)<[^<]*)*<\/head>/gi, "").replace(/<br\s*[\/]?>/gi, "\n").replace(/<\/(p|div|h1|h2|h3|h4|h5|h6|li|tr)>/gi, "\n").replace(/<[^>]+>/g, " ").replace(/&nbsp;/gi, " ").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&").replace(/&quot;/g, '"').replace(/\n\s*\n+/g, "\n\n").trim();
10992
+ }
10993
+ /**
10994
+ * Safe text extraction from non-OpenXML / legacy Excel spreadsheets (.xls, BIFF8, HTML tables, XML Spreadsheet 2003).
10995
+ */
10996
+ static parseLegacyExcelOrFallback(buffer, fileName) {
10997
+ const rawStr = buffer.toString("utf-8");
10998
+ if (/<html|<table|<tr|<tbody/i.test(rawStr)) {
10999
+ const rows = [];
11000
+ const rowMatches = rawStr.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi);
11001
+ if (rowMatches && rowMatches.length > 0) {
11002
+ for (const rowHtml of rowMatches) {
11003
+ const cells = rowHtml.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi);
11004
+ if (cells && cells.length > 0) {
11005
+ const rowText = cells.map((c) => this.stripHtmlAndXml(c).trim()).filter(Boolean).join(", ");
11006
+ if (rowText) rows.push(rowText);
11007
+ }
11008
+ }
11009
+ }
11010
+ if (rows.length > 0) {
11011
+ return rows.join("\n");
11012
+ }
11013
+ const clean = this.stripHtmlAndXml(rawStr);
11014
+ if (clean.length > 10) return clean;
11015
+ }
11016
+ if (/<(?:ss:)?Workbook|<\?xml/i.test(rawStr) && /<Row/i.test(rawStr)) {
11017
+ const rows = [];
11018
+ const rowMatches = rawStr.match(/<Row[^>]*>([\s\S]*?)<\/Row>/gi);
11019
+ if (rowMatches && rowMatches.length > 0) {
11020
+ for (const rowXml of rowMatches) {
11021
+ const dataMatches = rowXml.match(/<Data[^>]*>([\s\S]*?)<\/Data>/gi);
11022
+ if (dataMatches && dataMatches.length > 0) {
11023
+ const rowText = dataMatches.map((d) => this.stripXmlTags(d).trim()).filter(Boolean).join(", ");
11024
+ if (rowText) rows.push(rowText);
11025
+ }
11026
+ }
11027
+ }
11028
+ if (rows.length > 0) {
11029
+ return rows.join("\n");
11030
+ }
11031
+ }
11032
+ const extractedStrings = [];
11033
+ const ignoredOleMarkers = /* @__PURE__ */ new Set([
11034
+ "root entry",
11035
+ "workbook",
11036
+ "compobj",
11037
+ "summaryinformation",
11038
+ "documentsummaryinformation",
11039
+ "microsoft excel",
11040
+ "worksheet",
11041
+ "biff8",
11042
+ "calcpr"
11043
+ ]);
11044
+ let currentUtf16 = "";
11045
+ for (let i = 0; i < buffer.length - 1; i += 2) {
11046
+ const charCode = buffer[i];
11047
+ const zeroByte = buffer[i + 1];
11048
+ if (zeroByte === 0 && charCode >= 32 && charCode <= 126) {
11049
+ currentUtf16 += String.fromCharCode(charCode);
11050
+ } else {
11051
+ if (currentUtf16.length >= 2) {
11052
+ const trimmed = currentUtf16.trim();
11053
+ if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
11054
+ extractedStrings.push(trimmed);
11055
+ }
11056
+ }
11057
+ currentUtf16 = "";
11058
+ }
11059
+ }
11060
+ let currentAscii = "";
11061
+ for (let i = 0; i < buffer.length; i++) {
11062
+ const byte = buffer[i];
11063
+ if (byte >= 32 && byte <= 126 || byte === 9 || byte === 10 || byte === 13) {
11064
+ currentAscii += String.fromCharCode(byte);
11065
+ } else {
11066
+ if (currentAscii.length >= 2) {
11067
+ const trimmed = currentAscii.trim();
11068
+ if (trimmed.length >= 2 && !ignoredOleMarkers.has(trimmed.toLowerCase())) {
11069
+ extractedStrings.push(trimmed);
11070
+ }
11071
+ }
11072
+ currentAscii = "";
11073
+ }
11074
+ }
11075
+ const uniqueStrings = [];
11076
+ for (const str of extractedStrings) {
11077
+ if (uniqueStrings.length === 0 || uniqueStrings[uniqueStrings.length - 1] !== str) {
11078
+ uniqueStrings.push(str);
11079
+ }
11080
+ }
11081
+ if (uniqueStrings.length > 0) {
11082
+ return uniqueStrings.join(" | ");
11083
+ }
11084
+ const fallbackText = rawStr.replace(/[^\x20-\x7E\t\n\r]/g, " ").replace(/\s+/g, " ").trim();
11085
+ return fallbackText.length > 5 ? fallbackText : `[Excel Spreadsheet Content: ${fileName}]`;
11086
+ }
11087
+ /**
11088
+ * Dynamically infer document domain and schema from extracted text, filename, and format.
11089
+ */
11090
+ static inferDocumentDomainAndSchema(text, fileName = "", mimeType = "") {
11091
+ const lowerText = text.toLowerCase();
11092
+ const lowerName = fileName.toLowerCase();
11093
+ const financialTerms = [
11094
+ "price per share",
11095
+ "share price",
11096
+ "total cost of the deal",
11097
+ "cost of the deal",
11098
+ "lbo",
11099
+ "merger",
11100
+ "valuation",
11101
+ "d/e ratio",
11102
+ "ebit",
11103
+ "ebitda",
11104
+ "fcfe",
11105
+ "fcff",
11106
+ "wacc",
11107
+ "beta",
11108
+ "working capital",
11109
+ "cogs",
11110
+ "cap spending",
11111
+ "depreciation",
11112
+ "cashflow",
11113
+ "cash flow",
11114
+ "net income",
11115
+ "balance sheet",
11116
+ "income statement",
11117
+ "interest rate",
11118
+ "terminal year",
11119
+ "equity",
11120
+ "debt outstanding",
11121
+ "gross margin",
11122
+ "fiscal year",
11123
+ "fy20",
11124
+ "fy21",
11125
+ "fy22",
11126
+ "fy23",
11127
+ "fy24",
11128
+ "fy25",
11129
+ "q1",
11130
+ "q2",
11131
+ "q3",
11132
+ "q4"
11133
+ ];
11134
+ let financialScore = 0;
11135
+ for (const term of financialTerms) {
11136
+ if (lowerText.includes(term) || lowerName.includes(term)) {
11137
+ financialScore += 1;
11138
+ }
11139
+ }
11140
+ const ecommerceTerms = [
11141
+ "sku",
11142
+ "product_id",
11143
+ "product id",
11144
+ "item_id",
11145
+ "add to cart",
11146
+ "buy now",
11147
+ "in stock",
11148
+ "out of stock",
11149
+ "inventory count",
11150
+ "retail price",
11151
+ "unit price",
11152
+ "msrp",
11153
+ "cart",
11154
+ "shop",
11155
+ "catalog",
11156
+ "product_name",
11157
+ "product name",
11158
+ "brand name"
11159
+ ];
11160
+ let ecommerceScore = 0;
11161
+ for (const term of ecommerceTerms) {
11162
+ if (lowerText.includes(term) || lowerName.includes(term)) {
11163
+ ecommerceScore += 1;
11164
+ }
11165
+ }
11166
+ const legalTerms = ["agreement", "confidentiality", "indemnification", "jurisdiction", "party of the first part", "clauses", "arbitration", "terms and conditions"];
11167
+ let legalScore = 0;
11168
+ for (const term of legalTerms) {
11169
+ if (lowerText.includes(term)) legalScore += 1;
11170
+ }
11171
+ const healthTerms = ["patient", "dosage", "diagnosis", "treatment", "clinical trial", "prescription", "symptoms", "physician", "hospital"];
11172
+ let healthScore = 0;
11173
+ for (const term of healthTerms) {
11174
+ if (lowerText.includes(term)) healthScore += 1;
11175
+ }
11176
+ let schema;
11177
+ const lines = text.split("\n").map((l) => l.trim()).filter(Boolean);
11178
+ if (lines.length > 1 && (lines[0].includes(",") || lines[0].includes(" ") || lines[0].includes("|"))) {
11179
+ const delimiter = lines[0].includes(" ") ? " " : lines[0].includes("|") ? "|" : ",";
11180
+ const headers = lines[0].split(delimiter).map((h) => h.trim().replace(/^['"]|['"]$/g, "")).filter(Boolean);
11181
+ if (headers.length > 1 && lines.length > 1) {
11182
+ const sampleRow = lines[1].split(delimiter).map((c) => c.trim());
11183
+ schema = {};
11184
+ headers.forEach((h, idx) => {
11185
+ const val = sampleRow[idx] || "";
11186
+ if (/^-?\d+(?:\.\d+)?%?$/.test(val) || /^\$?[\d,]+(?:\.\d+)?$/.test(val)) {
11187
+ schema[h] = val.includes("%") ? "percentage" : val.includes("$") ? "currency" : "number";
11188
+ } else if (/^\d{4}[-/]\d{2}[-/]\d{2}/.test(val)) {
11189
+ schema[h] = "date";
11190
+ } else {
11191
+ schema[h] = "string";
11192
+ }
11193
+ });
11194
+ }
11195
+ }
11196
+ const isFinancial = financialScore >= 2 || financialScore >= 1 && (lowerName.includes("val") || lowerName.includes("fin") || lowerName.includes("lbo") || lowerName.includes("deal"));
11197
+ const isCatalog = ecommerceScore >= 2 && !isFinancial;
11198
+ let domain = "general";
11199
+ if (isFinancial) {
11200
+ domain = "finance";
11201
+ } else if (isCatalog) {
11202
+ domain = "ecommerce";
11203
+ } else if (legalScore >= 2) {
11204
+ domain = "legal";
11205
+ } else if (healthScore >= 2) {
11206
+ domain = "healthcare";
11207
+ } else if (schema && Object.keys(schema).length > 0) {
11208
+ domain = "tabular";
11209
+ }
11210
+ return { domain, schema, isCatalog, isFinancial };
11211
+ }
10469
11212
  static async readAsText(file) {
10470
11213
  if (Buffer.isBuffer(file)) {
10471
11214
  return file.toString("utf-8");
@@ -11112,18 +11855,22 @@ async function enforceLicense(req, config, bodyLicenseKey) {
11112
11855
  }
11113
11856
  }
11114
11857
  function getAllowedNamespaces(projectId) {
11115
- const trimmed = projectId.trim();
11858
+ const trimmed = (projectId || "default").trim();
11116
11859
  const withoutPrefix = trimmed.startsWith("retrivora-") ? trimmed.replace(/^retrivora-/, "") : trimmed;
11117
11860
  return new Set([
11118
11861
  trimmed,
11119
11862
  withoutPrefix,
11120
- `retrivora-${withoutPrefix}`
11863
+ `retrivora-${withoutPrefix}`,
11864
+ "default",
11865
+ "retrivora-default"
11121
11866
  ].filter(Boolean));
11122
11867
  }
11123
11868
  function resolveLicensedNamespace(payload, requestedNamespace) {
11869
+ var _a2;
11124
11870
  const allowed = getAllowedNamespaces(payload.projectId);
11125
11871
  const requested = requestedNamespace == null ? void 0 : requestedNamespace.trim();
11126
- if (requested && !allowed.has(requested)) {
11872
+ const isDevOrFree = process.env.NODE_ENV !== "production" || isFreeTier(payload.tier);
11873
+ if (requested && !isDevOrFree && !allowed.has(requested)) {
11127
11874
  return {
11128
11875
  ok: false,
11129
11876
  response: new Response(JSON.stringify({
@@ -11137,7 +11884,7 @@ function resolveLicensedNamespace(payload, requestedNamespace) {
11137
11884
  })
11138
11885
  };
11139
11886
  }
11140
- const defaultNamespace = payload.projectId.startsWith("retrivora-") ? payload.projectId : `retrivora-${payload.projectId}`;
11887
+ const defaultNamespace = ((_a2 = payload.projectId) == null ? void 0 : _a2.startsWith("retrivora-")) ? payload.projectId : `retrivora-${payload.projectId || "default"}`;
11141
11888
  return { ok: true, namespace: requested || defaultNamespace };
11142
11889
  }
11143
11890
  var MAX_MESSAGE_LENGTH = 8e3;
@@ -11581,7 +12328,7 @@ function createIngestHandler(configOrPlugin, options) {
11581
12328
  function createHealthHandler(configOrPlugin, options) {
11582
12329
  const plugin = getOrCreatePlugin(configOrPlugin);
11583
12330
  const onAuthorize = options == null ? void 0 : options.onAuthorize;
11584
- return async function GET(req) {
12331
+ return async function GET(req, context) {
11585
12332
  if (req) {
11586
12333
  const authResult = await checkAuth(req, onAuthorize);
11587
12334
  if (authResult) return authResult;
@@ -11692,9 +12439,10 @@ function createUploadHandler(configOrPlugin, options) {
11692
12439
  }
11693
12440
  const documents = [];
11694
12441
  for (const file of files) {
12442
+ const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
11695
12443
  const isExcel = file.name.toLowerCase().endsWith(".xlsx") || file.name.toLowerCase().endsWith(".xls") || file.type.includes("spreadsheet") || file.type.includes("excel");
11696
12444
  if (file.name.toLowerCase().endsWith(".csv") || file.type === "text/csv") {
11697
- const text = await file.text();
12445
+ const text = buffer.toString("utf-8");
11698
12446
  const Papa = await import("papaparse");
11699
12447
  const parsed = Papa.default.parse(text, { header: true, skipEmptyLines: true });
11700
12448
  if (parsed.data && parsed.data.length > 0) {
@@ -11734,49 +12482,70 @@ function createUploadHandler(configOrPlugin, options) {
11734
12482
  }
11735
12483
  }
11736
12484
  } else if (isExcel) {
11737
- const ExcelJS = await import("exceljs");
11738
- const buffer = Buffer.isBuffer(file) ? file : Buffer.from(await file.arrayBuffer());
11739
- const workbook = new ExcelJS.Workbook();
11740
- await workbook.xlsx.load(buffer);
11741
- workbook.eachSheet((sheet, sheetId) => {
11742
- const sheetName = sheet.name;
11743
- let headers = [];
11744
- let i = 0;
11745
- sheet.eachRow((row, rowNumber) => {
11746
- const values = Array.isArray(row.values) ? row.values.slice(1) : [];
11747
- if (rowNumber === 1) {
11748
- headers = values.map((v) => String(v));
11749
- return;
11750
- }
11751
- i++;
11752
- const contentParts = [];
11753
- const rowData = {};
11754
- values.forEach((val, index) => {
11755
- const key = headers[index] || `Column${index + 1}`;
11756
- rowData[key] = val;
11757
- if (key && val !== void 0 && val !== null && String(val).trim()) {
11758
- contentParts.push(`${key}: ${val}`);
11759
- }
11760
- });
11761
- if (contentParts.length > 0) {
11762
- documents.push({
11763
- docId: `${file.name}-${sheetName}-row-${i}`,
11764
- content: contentParts.join(", "),
11765
- metadata: __spreadValues(__spreadProps(__spreadValues({
11766
- fileName: file.name,
11767
- sheetName,
11768
- fileSize: file.size,
11769
- fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
11770
- uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
11771
- }, dimension ? { dimension } : {}), {
11772
- headers
11773
- }), rowData)
12485
+ let excelLoaded = false;
12486
+ try {
12487
+ const isZip = buffer.length >= 4 && buffer[0] === 80 && buffer[1] === 75;
12488
+ if (isZip) {
12489
+ const ExcelJS = await import("exceljs");
12490
+ const workbook = new ExcelJS.Workbook();
12491
+ await workbook.xlsx.load(buffer);
12492
+ workbook.eachSheet((sheet, sheetId) => {
12493
+ const sheetName = sheet.name;
12494
+ let headers = [];
12495
+ let i = 0;
12496
+ sheet.eachRow((row, rowNumber) => {
12497
+ const values = Array.isArray(row.values) ? row.values.slice(1) : [];
12498
+ if (rowNumber === 1) {
12499
+ headers = values.map((v) => String(v));
12500
+ return;
12501
+ }
12502
+ i++;
12503
+ const contentParts = [];
12504
+ const rowData = {};
12505
+ values.forEach((val, index) => {
12506
+ const key = headers[index] || `Column${index + 1}`;
12507
+ rowData[key] = val;
12508
+ if (key && val !== void 0 && val !== null && String(val).trim()) {
12509
+ contentParts.push(`${key}: ${val}`);
12510
+ }
12511
+ });
12512
+ if (contentParts.length > 0) {
12513
+ documents.push({
12514
+ docId: `${file.name}-${sheetName}-row-${i}`,
12515
+ content: contentParts.join(", "),
12516
+ metadata: __spreadValues(__spreadProps(__spreadValues({
12517
+ fileName: file.name,
12518
+ sheetName,
12519
+ fileSize: file.size,
12520
+ fileType: file.type || "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
12521
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12522
+ }, dimension ? { dimension } : {}), {
12523
+ headers
12524
+ }), rowData)
12525
+ });
12526
+ }
11774
12527
  });
11775
- }
12528
+ });
12529
+ excelLoaded = documents.length > 0;
12530
+ }
12531
+ } catch (excelErr) {
12532
+ excelLoaded = false;
12533
+ }
12534
+ if (!excelLoaded) {
12535
+ const content = await DocumentParser.parse(buffer, file.name, file.type);
12536
+ documents.push({
12537
+ docId: file.name,
12538
+ content,
12539
+ metadata: __spreadValues({
12540
+ fileName: file.name,
12541
+ fileSize: file.size,
12542
+ fileType: file.type || "application/vnd.ms-excel",
12543
+ uploadedAt: (/* @__PURE__ */ new Date()).toISOString()
12544
+ }, dimension ? { dimension } : {})
11776
12545
  });
11777
- });
12546
+ }
11778
12547
  } else {
11779
- const content = await DocumentParser.parse(file, file.name, file.type);
12548
+ const content = await DocumentParser.parse(buffer, file.name, file.type);
11780
12549
  documents.push({
11781
12550
  docId: file.name,
11782
12551
  content,
@@ -11935,7 +12704,7 @@ function createSessionsHandler(configOrPlugin, options) {
11935
12704
  const storage = new DatabaseStorage(plugin.getConfig());
11936
12705
  const onAuthorize = options == null ? void 0 : options.onAuthorize;
11937
12706
  const onResolveScope = options == null ? void 0 : options.onResolveScope;
11938
- return async function GET(req) {
12707
+ return async function GET(req, context) {
11939
12708
  if (req) {
11940
12709
  const authResult = await checkAuth(req, onAuthorize);
11941
12710
  if (authResult) return authResult;