@utilix-tech/sdk 0.1.4 → 0.3.0

package/dist/tools/ai_agent.cjs

@@ -0,0 +1,949 @@
+'use strict';
+
+// src/tools/ai_agent.ts
+var MODEL_PRICING = [
+  { model: "gpt-4o", label: "GPT-4o", inputPer1M: 5, outputPer1M: 15 },
+  { model: "gpt-4o-mini", label: "GPT-4o mini", inputPer1M: 0.15, outputPer1M: 0.6 },
+  { model: "gpt-4-turbo", label: "GPT-4 Turbo", inputPer1M: 10, outputPer1M: 30 },
+  { model: "claude-sonnet-4", label: "Claude Sonnet 4", inputPer1M: 3, outputPer1M: 15 },
+  { model: "claude-haiku-4", label: "Claude Haiku 4", inputPer1M: 0.8, outputPer1M: 4 },
+  { model: "claude-opus-4", label: "Claude Opus 4", inputPer1M: 15, outputPer1M: 75 },
+  { model: "gemini-1.5-flash", label: "Gemini 1.5 Flash", inputPer1M: 0.075, outputPer1M: 0.3 },
+  { model: "gemini-1.5-pro", label: "Gemini 1.5 Pro", inputPer1M: 3.5, outputPer1M: 10.5 },
+  { model: "llama-3.1-70b", label: "Llama 3.1 70B", inputPer1M: 0.59, outputPer1M: 0.79 }
+];
+function estimateTokens(text) {
+  if (!text) return { tokens: 0, chars: 0, words: 0, method: "approximate" };
+  const chars = text.length;
+  const words = text.trim() === "" ? 0 : text.trim().split(/\s+/).length;
+  const byChars = Math.ceil(chars / 4);
+  const tokens = Math.ceil(byChars * 0.7 + words * 0.3);
+  return { tokens, chars, words, method: "approximate" };
+}
+function estimateCost(inputTokens, outputTokens, model) {
+  const outTokens = outputTokens ?? inputTokens;
+  const pricing = model ? MODEL_PRICING.filter((m) => m.model === model) : MODEL_PRICING;
+  const costs = pricing.map((m) => {
+    const inputCost = inputTokens / 1e6 * m.inputPer1M;
+    const outputCost = outTokens / 1e6 * m.outputPer1M;
+    return { model: m.model, label: m.label, inputCost, outputCost, totalCost: inputCost + outputCost };
+  });
+  return {
+    estimate: estimateTokens(""),
+    // placeholder when called directly
+    costs,
+    model
+  };
+}
+var PII_PATTERNS = [
+  {
+    type: "email",
+    label: "Email address",
+    re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g,
+    mask: (v) => v.replace(/^(.{2}).*(@.*)$/, "$1***$2")
+  },
+  {
+    type: "phone",
+    label: "Phone number",
+    re: /(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g,
+    mask: (v) => v.replace(/\d(?=\d{4})/g, "*")
+  },
+  {
+    type: "ssn",
+    label: "Social Security Number",
+    re: /\b\d{3}[-\s]?\d{2}[-\s]?\d{4}\b/g,
+    mask: (v) => v.replace(/\d/g, "*").replace(/^\*+/, "***")
+  },
+  {
+    type: "credit_card",
+    label: "Credit card number",
+    re: /\b(?:4\d{12}(?:\d{3})?|5[1-5]\d{14}|3[47]\d{13}|6(?:011|5\d{2})\d{12}|(?:2131|1800|35\d{3})\d{11})\b/g,
+    mask: (v) => v.slice(0, 4) + " **** **** " + v.slice(-4)
+  },
+  {
+    type: "ip_address",
+    label: "IP address",
+    re: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g,
+    mask: (v) => v.replace(/\.\d+$/, ".*")
+  },
+  {
+    type: "iban",
+    label: "IBAN",
+    re: /\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}(?:[A-Z0-9]?){0,16}\b/g,
+    mask: (v) => v.slice(0, 4) + "****" + v.slice(-4)
+  },
+  {
+    type: "crypto_wallet",
+    label: "Crypto wallet address",
+    re: /\b(?:0x[a-fA-F0-9]{40}|[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-z0-9]{39,59})\b/g,
+    mask: (v) => v.slice(0, 6) + "..." + v.slice(-4)
+  }
+];
+function detectPii(text) {
+  const findings = [];
+  for (const p of PII_PATTERNS) {
+    p.re.lastIndex = 0;
+    let m;
+    while ((m = p.re.exec(text)) !== null) {
+      findings.push({
+        type: p.type,
+        label: p.label,
+        value: m[0],
+        masked: p.mask(m[0]),
+        start: m.index,
+        end: m.index + m[0].length
+      });
+    }
+  }
+  findings.sort((a, b) => a.start - b.start);
+  return { found: findings.length > 0, count: findings.length, findings };
+}
+function redactPii(text, replacement = "[REDACTED]") {
+  const result = detectPii(text);
+  let redacted = text;
+  for (const f of [...result.findings].reverse()) {
+    redacted = redacted.slice(0, f.start) + replacement + redacted.slice(f.end);
+  }
+  return { ...result, redacted };
+}
+var maskSecret = (v) => v.slice(0, 6) + "*".repeat(Math.max(0, v.length - 10)) + v.slice(-4);
+var SECRET_PATTERNS = [
+  { type: "openai_key", label: "OpenAI API key", re: /\bsk-[a-zA-Z0-9]{20,}\b/g, risk: "critical" },
+  { type: "anthropic_key", label: "Anthropic API key", re: /\bsk-ant-[a-zA-Z0-9\-]{20,}\b/g, risk: "critical" },
+  { type: "aws_key", label: "AWS access key", re: /\bAKIA[0-9A-Z]{16}\b/g, risk: "critical" },
+  { type: "github_token", label: "GitHub token", re: /\b(?:ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36,}\b/g, risk: "critical" },
+  { type: "github_token", label: "GitHub fine-grained PAT", re: /\bgithub_pat_[a-zA-Z0-9_]{59,}\b/g, risk: "critical" },
+  { type: "stripe_key", label: "Stripe key", re: /\b(?:sk|pk)_(?:live|test)_[a-zA-Z0-9]{24,}\b/g, risk: "critical" },
+  { type: "google_api_key", label: "Google API key", re: /\bAIza[0-9A-Za-z\-_]{35}\b/g, risk: "high" },
+  { type: "slack_token", label: "Slack token", re: /\bxox[boas]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,}\b/g, risk: "high" },
+  { type: "sendgrid_key", label: "SendGrid API key", re: /\bSG\.[a-zA-Z0-9\-_]{22}\.[a-zA-Z0-9\-_]{43}\b/g, risk: "high" },
+  { type: "npm_token", label: "npm token", re: /\bnpm_[a-zA-Z0-9]{36}\b/g, risk: "high" },
+  { type: "jwt_token", label: "JWT token", re: /\beyJ[a-zA-Z0-9_\-]+\.eyJ[a-zA-Z0-9_\-]+\.[a-zA-Z0-9_\-]+\b/g, risk: "medium" },
+  { type: "private_key", label: "Private key block", re: /-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----/g, risk: "critical" },
+  { type: "generic_api_key", label: "Generic API key pattern", re: /\b(?:api[_-]?key|apikey|api[_-]?secret|access[_-]?token)\s*[:=]\s*['"]?[a-zA-Z0-9\-_]{20,}['"]?/gi, risk: "low" }
+];
+function getLineCol(text, index) {
+  const before = text.slice(0, index);
+  const line = (before.match(/\n/g) || []).length + 1;
+  const column = index - before.lastIndexOf("\n");
+  return { line, column };
+}
+function detectSecrets(text) {
+  const findings = [];
+  let maxRisk = "none";
+  const riskOrder = { none: 0, low: 1, medium: 2, high: 3, critical: 4 };
+  for (const p of SECRET_PATTERNS) {
+    p.re.lastIndex = 0;
+    let m;
+    while ((m = p.re.exec(text)) !== null) {
+      const { line, column } = getLineCol(text, m.index);
+      findings.push({ type: p.type, label: p.label, value: m[0], masked: maskSecret(m[0]), line, column });
+      if (riskOrder[p.risk] > riskOrder[maxRisk]) maxRisk = p.risk;
+    }
+  }
+  return { found: findings.length > 0, count: findings.length, findings, riskLevel: maxRisk };
+}
+var INJECTION_PATTERNS = [
+  { pattern: "Ignore previous instructions", re: /ignore\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+instructions/gi, severity: "high", weight: 0.9 },
+  { pattern: "Disregard instructions", re: /disregard\s+(?:all\s+)?(?:previous|prior|above|your)\s+instructions/gi, severity: "high", weight: 0.9 },
+  { pattern: "New instructions override", re: /(?:new|actual|real)\s+instructions?[:：]/gi, severity: "high", weight: 0.8 },
+  { pattern: "System prompt reveal", re: /(?:reveal|show|print|output|tell me)\s+(?:your\s+)?(?:system\s+prompt|instructions|prompt)/gi, severity: "high", weight: 0.85 },
+  { pattern: "Role confusion (act as)", re: /\bact\s+as\s+(?:a\s+)?(?:different|new|another|an?\s+ai|dan|jailbreak)/gi, severity: "high", weight: 0.8 },
+  { pattern: "DAN jailbreak", re: /\bDAN\b|do\s+anything\s+now/gi, severity: "high", weight: 0.95 },
+  { pattern: "Jailbreak keyword", re: /\b(?:jailbreak|jailbroken|unrestricted\s+mode|developer\s+mode)\b/gi, severity: "high", weight: 0.9 },
+  { pattern: "System role injection", re: /^system\s*[:：]/gim, severity: "high", weight: 0.85 },
+  { pattern: "Assistant role injection", re: /^assistant\s*[:：]/gim, severity: "high", weight: 0.85 },
+  { pattern: "Prompt injection tag", re: /<(?:system|assistant|user|prompt|instruction)[^>]*>/gi, severity: "medium", weight: 0.7 },
+  { pattern: "Forget instructions", re: /\b(?:forget|discard|drop)\s+(?:all\s+)?(?:previous\s+)?instructions/gi, severity: "high", weight: 0.85 },
+  { pattern: "Override safety", re: /(?:override|bypass|circumvent|disable)\s+(?:your\s+)?(?:safety|filters?|restrictions?|guidelines?)/gi, severity: "high", weight: 0.9 },
+  { pattern: "Hypothetical framing", re: /(?:hypothetically|in\s+a\s+fictional|pretend\s+that|imagine\s+you\s+are)\s+.*(?:no\s+restrictions?|unrestricted)/gi, severity: "medium", weight: 0.6 },
+  { pattern: "Translate to bypass", re: /translate.*(?:ignore|disregard|bypass)/gi, severity: "medium", weight: 0.5 },
+  { pattern: "Markdown injection", re: /\[.*\]\(javascript:/gi, severity: "low", weight: 0.4 }
+];
+function detectPromptInjection(text) {
+  const findings = [];
+  let totalWeight = 0;
+  for (const p of INJECTION_PATTERNS) {
+    p.re.lastIndex = 0;
+    let m;
+    while ((m = p.re.exec(text)) !== null) {
+      if (!findings.find((f) => f.pattern === p.pattern)) {
+        findings.push({ pattern: p.pattern, matched: m[0], severity: p.severity });
+        totalWeight = Math.min(1, totalWeight + p.weight * (1 - totalWeight));
+      }
+    }
+  }
+  const score = Math.round(totalWeight * 100) / 100;
+  const severity = score === 0 ? "none" : score < 0.4 ? "low" : score < 0.7 ? "medium" : "high";
+  return { isInjection: score > 0.3, score, severity, findings };
+}
+function repairJson(input) {
+  const original = input;
+  const fixes = [];
+  let s = input.trim();
+  if (!s) return { ok: false, repaired: "", original, fixes: [], error: "Empty input" };
+  try {
+    JSON.parse(s);
+    return { ok: true, repaired: s, original, fixes: [] };
+  } catch {
+  }
+  const withoutLineComments = s.replace(/\/\/[^\n]*/g, "");
+  const withoutBlockComments = withoutLineComments.replace(/\/\*[\s\S]*?\*\//g, "");
+  if (withoutBlockComments !== s) {
+    s = withoutBlockComments;
+    fixes.push("Removed JS comments");
+  }
+  const singleToDouble = s.replace(/'([^'\\]*(\\.[^'\\]*)*)'/g, (_, inner) => `"${inner.replace(/"/g, '\\"')}"`);
+  if (singleToDouble !== s) {
+    s = singleToDouble;
+    fixes.push("Replaced single quotes with double quotes");
+  }
+  const quotedKeys = s.replace(/([{,]\s*)([a-zA-Z_$][a-zA-Z0-9_$]*)\s*:/g, '$1"$2":');
+  if (quotedKeys !== s) {
+    s = quotedKeys;
+    fixes.push("Quoted unquoted object keys");
+  }
+  const noTrailingCommas = s.replace(/,(\s*[}\]])/g, "$1");
+  if (noTrailingCommas !== s) {
+    s = noTrailingCommas;
+    fixes.push("Removed trailing commas");
+  }
+  const noUndefined = s.replace(/\bundefined\b/g, "null");
+  if (noUndefined !== s) {
+    s = noUndefined;
+    fixes.push("Replaced undefined with null");
+  }
+  const noNaN = s.replace(/\bNaN\b/g, "null").replace(/\b(?:Infinity|-Infinity)\b/g, "null");
+  if (noNaN !== s) {
+    s = noNaN;
+    fixes.push("Replaced NaN/Infinity with null");
+  }
+  const opens = [];
+  let inStr = false;
+  let escape = false;
+  for (const ch of s) {
+    if (escape) {
+      escape = false;
+      continue;
+    }
+    if (ch === "\\" && inStr) {
+      escape = true;
+      continue;
+    }
+    if (ch === '"') {
+      inStr = !inStr;
+      continue;
+    }
+    if (inStr) continue;
+    if (ch === "{") opens.push("}");
+    else if (ch === "[") opens.push("]");
+    else if (ch === "}" || ch === "]") opens.pop();
+  }
+  if (opens.length > 0) {
+    s += opens.reverse().join("");
+    fixes.push(`Closed ${opens.length} unclosed bracket(s)`);
+  }
+  try {
+    JSON.parse(s);
+    return { ok: true, repaired: s, original, fixes };
+  } catch (e) {
+    return { ok: false, repaired: s, original, fixes, error: e.message };
+  }
+}
+function _estimateTokens(text) {
+  const chars = text.length;
+  const words = text.trim() === "" ? 0 : text.trim().split(/\s+/).length;
+  return Math.ceil(Math.ceil(chars / 4) * 0.7 + words * 0.3);
+}
+function trimToTokens(text, maxTokens, strategy = "end") {
+  const originalTokens = _estimateTokens(text);
+  if (originalTokens <= maxTokens) return { trimmed: text, originalTokens, trimmedTokens: originalTokens, truncated: false, removedChars: 0 };
+  const fit = (s) => _estimateTokens(s) <= maxTokens;
+  if (strategy === "end") {
+    let lo2 = 0, hi2 = text.length;
+    while (lo2 < hi2) {
+      const mid = lo2 + hi2 + 1 >> 1;
+      fit(text.slice(0, mid)) ? lo2 = mid : hi2 = mid - 1;
+    }
+    const trimmed2 = text.slice(0, lo2).trimEnd();
+    return { trimmed: trimmed2, originalTokens, trimmedTokens: _estimateTokens(trimmed2), truncated: true, removedChars: text.length - lo2 };
+  }
+  if (strategy === "start") {
+    let lo2 = 0, hi2 = text.length;
+    while (lo2 < hi2) {
+      const mid = lo2 + hi2 >> 1;
+      fit(text.slice(mid)) ? hi2 = mid : lo2 = mid + 1;
+    }
+    const trimmed2 = text.slice(lo2).trimStart();
+    return { trimmed: trimmed2, originalTokens, trimmedTokens: _estimateTokens(trimmed2), truncated: true, removedChars: lo2 };
+  }
+  let lo = 0, hi = text.length;
+  while (lo < hi) {
+    const mid = lo + hi + 1 >> 1;
+    const h2 = mid >> 1;
+    fit(text.slice(0, h2) + "\n...\n" + text.slice(text.length - h2)) ? lo = mid : hi = mid - 1;
+  }
+  const h = lo >> 1;
+  const trimmed = text.slice(0, h).trimEnd() + "\n...\n" + text.slice(text.length - h).trimStart();
+  return { trimmed, originalTokens, trimmedTokens: _estimateTokens(trimmed), truncated: true, removedChars: text.length - lo };
+}
+function chunkText(text, chunkSize = 200, overlap = 20, strategy = "paragraph") {
+  let units;
+  if (strategy === "paragraph") units = text.split(/\n{2,}/).filter((s) => s.trim());
+  else if (strategy === "sentence") units = text.match(/[^.!?]+[.!?]+(\s|$)|[^.!?]+$/g)?.map((s) => s.trim()).filter(Boolean) ?? [text];
+  else {
+    const words = text.split(/\s+/);
+    const us = [];
+    let buf = "";
+    for (const w of words) {
+      if (buf && (buf + " " + w).length > 200) {
+        us.push(buf);
+        buf = w;
+      } else buf = buf ? buf + " " + w : w;
+    }
+    if (buf) us.push(buf);
+    units = us;
+  }
+  const chunks = [];
+  let i = 0, charPos = 0;
+  while (i < units.length) {
+    const buf = [];
+    let tokens = 0, j = i;
+    while (j < units.length && tokens < chunkSize) {
+      buf.push(units[j]);
+      tokens = _estimateTokens(buf.join(strategy === "paragraph" ? "\n\n" : " "));
+      j++;
+    }
+    const ct = buf.join(strategy === "paragraph" ? "\n\n" : " ");
+    const sc = Math.max(0, text.indexOf(buf[0], charPos));
+    chunks.push({ index: chunks.length, text: ct, tokens: _estimateTokens(ct), startChar: sc, endChar: sc + ct.length });
+    const ov = Math.max(1, Math.floor(overlap / Math.max(1, tokens / buf.length)));
+    i = Math.max(i + 1, j - ov);
+    charPos = sc;
+  }
+  return chunks;
+}
+function extractUrls(text) {
+  const seen = /* @__PURE__ */ new Set();
+  const results = [];
+  const hrefRe = /href=["']([^"']+)["']/gi;
+  let m;
+  while ((m = hrefRe.exec(text)) !== null) {
+    const url = m[1];
+    if (url.startsWith("http") && !seen.has(url)) {
+      seen.add(url);
+      results.push(parseUrl(url, m.index));
+    }
+  }
+  const urlRe = /https?:\/\/(?:www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_+.~#?&/=]*)/g;
+  while ((m = urlRe.exec(text)) !== null) {
+    if (!seen.has(m[0])) {
+      seen.add(m[0]);
+      results.push(parseUrl(m[0], m.index));
+    }
+  }
+  results.sort((a, b) => a.position - b.position);
+  return { urls: results, count: results.length };
+}
+function parseUrl(url, position) {
+  try {
+    const u = new URL(url);
+    return { url, protocol: u.protocol.replace(":", ""), domain: u.hostname, path: u.pathname + u.search + u.hash, position };
+  } catch {
+    return { url, protocol: url.startsWith("https") ? "https" : "http", domain: "", path: "", position };
+  }
+}
+function sanitizeHtml(html, keepLinks = false) {
+  let s = html;
+  const scripts = s.match(/<script[\s\S]*?<\/script>/gi) ?? [];
+  s = s.replace(/<script[\s\S]*?<\/script>/gi, "");
+  const styles = s.match(/<style[\s\S]*?<\/style>/gi) ?? [];
+  s = s.replace(/<style[\s\S]*?<\/style>/gi, "");
+  s = s.replace(/<!--[\s\S]*?-->/g, "").replace(/\son\w+\s*=\s*["'][^"']*["']/gi, "").replace(/\son\w+\s*=\s*[^\s>]*/gi, "");
+  if (keepLinks) s = s.replace(/<a[^>]+href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, "$2 [$1]");
+  s = s.replace(/<\/?(p|div|h[1-6]|li|tr|td|th|br|hr|blockquote|section|article|header|footer|main|nav)[^>]*>/gi, "\n");
+  const remaining = (s.match(/<[^>]+>/g) ?? []).length;
+  s = s.replace(/<[^>]+>/g, "").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/&nbsp;/g, " ");
+  s = s.replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
+  return { text: s, removedTags: remaining + scripts.length + styles.length, removedScripts: scripts.length, removedStyles: styles.length, originalLength: html.length, cleanLength: s.length };
+}
+function flattenJson(obj, separator = ".", prefix = "") {
+  const result = {};
+  function walk(value, path) {
+    if (value === null || typeof value !== "object") {
+      result[path] = value;
+      return;
+    }
+    if (Array.isArray(value)) {
+      if (value.length === 0) {
+        result[path] = [];
+        return;
+      }
+      value.forEach((item, i) => walk(item, path ? `${path}[${i}]` : `[${i}]`));
+      return;
+    }
+    const entries = Object.entries(value);
+    if (entries.length === 0) {
+      result[path] = {};
+      return;
+    }
+    for (const [k, v] of entries) walk(v, path ? `${path}${separator}${k}` : k);
+  }
+  walk(obj, prefix);
+  return result;
+}
+function unflattenJson(flat, separator = ".") {
+  const result = {};
+  for (const [flatKey, value] of Object.entries(flat)) {
+    const parts = flatKey.split(separator).flatMap((p) => {
+      const m = p.match(/^([^\[]*)((?:\[\d+\])*)$/);
+      if (!m) return [p];
+      const ps = [];
+      if (m[1]) ps.push(m[1]);
+      for (const idx of m[2].matchAll(/\[(\d+)\]/g)) ps.push(idx[1]);
+      return ps;
+    });
+    let cur = result;
+    for (let i = 0; i < parts.length - 1; i++) {
+      const key = parts[i], nk = parts[i + 1];
+      if (!(key in cur)) cur[key] = /^\d+$/.test(nk) ? [] : {};
+      cur = cur[key];
+    }
+    cur[parts[parts.length - 1]] = value;
+  }
+  return result;
+}
+function _isObj(v) {
+  return v !== null && typeof v === "object" && !Array.isArray(v);
+}
+function _deepMerge(base, override, strategy, path, conflicts) {
+  if (_isObj(base) && _isObj(override)) {
+    const result = { ...base };
+    for (const [key, val] of Object.entries(override)) {
+      const np = path ? `${path}.${key}` : key;
+      result[key] = key in result ? _deepMerge(result[key], val, strategy, np, conflicts) : val;
+    }
+    return result;
+  }
+  if (Array.isArray(base) && Array.isArray(override)) {
+    if (strategy !== "deep") return override;
+    return [...base, ...override];
+  }
+  if (base !== void 0 && override !== void 0 && JSON.stringify(base) !== JSON.stringify(override)) conflicts.push({ path, base, override });
+  return override;
+}
+function mergeJson(base, override, strategy = "deep") {
+  const conflicts = [];
+  return { merged: _deepMerge(base, override, strategy, "", conflicts), conflicts };
+}
+function extractJson(text) {
+  const blocks = [];
+  let i = 0;
+  const cbRe = /```(?:json|JSON)?\s*\n?([\s\S]*?)```/g;
+  let m;
+  while ((m = cbRe.exec(text)) !== null) {
+    try {
+      const raw = m[1].trim();
+      const parsed = JSON.parse(raw);
+      blocks.push({ index: 0, raw, parsed, type: Array.isArray(parsed) ? "array" : typeof parsed, source: "code-block", startPos: m.index });
+    } catch {
+    }
+  }
+  const cbRanges = [...text.matchAll(/```[\s\S]*?```/g)].map((m2) => [m2.index, m2.index + m2[0].length]);
+  const inCb = (pos) => cbRanges.some(([s, e]) => pos >= s && pos < e);
+  while (i < text.length) {
+    const ch = text[i];
+    if ((ch === "{" || ch === "[") && !inCb(i)) {
+      let depth = 1, j = i + 1, inStr = false;
+      while (j < text.length && depth > 0) {
+        const c = text[j];
+        if (c === '"' && text[j - 1] !== "\\") inStr = !inStr;
+        if (!inStr) {
+          if (c === "{" || c === "[") depth++;
+          else if (c === "}" || c === "]") depth--;
+        }
+        j++;
+      }
+      if (depth === 0) {
+        const raw = text.slice(i, j);
+        try {
+          const parsed = JSON.parse(raw);
+          if (!blocks.some((b) => b.raw.trim() === raw.trim()) && raw.length > 2) blocks.push({ index: 0, raw, parsed, type: Array.isArray(parsed) ? "array" : typeof parsed, source: "inline", startPos: i });
+          i = j;
+          continue;
+        } catch {
+        }
+      }
+    }
+    i++;
+  }
+  blocks.sort((a, b) => a.startPos - b.startPos).forEach((b, idx) => {
+    b.index = idx;
+  });
+  return { blocks, count: blocks.length };
+}
+function deduplicateLines(text, strategy = "exact") {
+  const raw = text.split("\n");
+  const norm = (s) => strategy === "exact" ? s : strategy === "case-insensitive" ? s.toLowerCase() : strategy === "trimmed" ? s.trim() : s.trim().replace(/\s+/g, " ").toLowerCase();
+  const seen = /* @__PURE__ */ new Map();
+  const result = [];
+  for (let i = 0; i < raw.length; i++) {
+    const key = norm(raw[i]);
+    if (!seen.has(key)) {
+      seen.set(key, { count: 1, firstLine: i + 1, original: raw[i] });
+      result.push(raw[i]);
+    } else seen.get(key).count++;
+  }
+  const duplicates = [...seen.entries()].filter(([, v]) => v.count > 1).map(([, v]) => ({ line: v.original, count: v.count, firstLine: v.firstLine })).sort((a, b) => b.count - a.count);
+  return { lines: result, originalCount: raw.length, uniqueCount: result.length, removedCount: raw.length - result.length, duplicates };
+}
+var _STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "is", "was", "are", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "must", "shall", "not", "no", "nor", "so", "yet", "both", "either", "neither", "as", "if", "then", "than", "that", "this", "these", "those", "it", "its", "i", "we", "you", "he", "she", "they", "their", "there", "here", "what", "which", "who", "when", "where", "how", "why", "all", "any", "each", "every", "more", "most", "other", "some", "such", "into", "over", "also", "just", "about", "up", "out", "can", "get", "only", "new"]);
+function extractKeywords(text, maxKeywords = 20, minWordLen = 3) {
+  const words = text.toLowerCase().match(/\b[a-z]{3,}\b/g) ?? [];
+  const freq = /* @__PURE__ */ new Map();
+  words.forEach((w, pos) => {
+    if (w.length >= minWordLen && !_STOP.has(w)) {
+      if (!freq.has(w)) freq.set(w, { count: 0, positions: [] });
+      freq.get(w).count++;
+      freq.get(w).positions.push(pos);
+    }
+  });
+  const maxFreq = Math.max(...[...freq.values()].map((v) => v.count), 1);
+  const sentences = text.split(/[.!?]+/).filter(Boolean);
+  const keywords = [...freq.entries()].map(([word, { count, positions }]) => {
+    const tf = count / words.length;
+    const df = sentences.filter((s) => s.toLowerCase().includes(word)).length || 1;
+    const idf = Math.log((sentences.length + 1) / df);
+    return { word, count, score: Math.round(tf * idf * (count / maxFreq) * 1e3) / 100, positions };
+  }).sort((a, b) => b.score - a.score || b.count - a.count).slice(0, maxKeywords);
+  return { keywords, totalWords: words.length, uniqueWords: freq.size };
+}
+function _validate(data, schema, path, errors) {
+  const typeOf = (v) => v === null ? "null" : Array.isArray(v) ? "array" : typeof v;
+  if (schema.type) {
+    const types = Array.isArray(schema.type) ? schema.type : [schema.type];
+    if (!types.some((t) => t === typeOf(data) || t === "integer" && typeOf(data) === "number" && Number.isInteger(data))) errors.push({ path, message: `Expected ${types.join("|")}, got ${typeOf(data)}`, value: data });
+  }
+  if ("enum" in schema && Array.isArray(schema.enum) && !schema.enum.some((e) => JSON.stringify(e) === JSON.stringify(data))) errors.push({ path, message: `Must be one of: ${schema.enum.map((e) => JSON.stringify(e)).join(", ")}` });
+  if (typeof data === "string") {
+    if (typeof schema.minLength === "number" && data.length < schema.minLength) errors.push({ path, message: `Too short (min ${schema.minLength})` });
+    if (typeof schema.maxLength === "number" && data.length > schema.maxLength) errors.push({ path, message: `Too long (max ${schema.maxLength})` });
+    if (typeof schema.pattern === "string") {
+      try {
+        if (!new RegExp(schema.pattern).test(data)) errors.push({ path, message: `Does not match pattern ${schema.pattern}` });
+      } catch {
+      }
+    }
+  }
+  if (typeof data === "number") {
+    if (typeof schema.minimum === "number" && data < schema.minimum) errors.push({ path, message: `Must be >= ${schema.minimum}` });
+    if (typeof schema.maximum === "number" && data > schema.maximum) errors.push({ path, message: `Must be <= ${schema.maximum}` });
+  }
+  if (Array.isArray(data)) {
+    if (typeof schema.minItems === "number" && data.length < schema.minItems) errors.push({ path, message: `Too few items (min ${schema.minItems})` });
+    if (schema.items && typeof schema.items === "object") data.forEach((item, i) => _validate(item, schema.items, `${path}[${i}]`, errors));
+  }
+  if (data !== null && typeof data === "object" && !Array.isArray(data)) {
+    const obj = data;
+    if (Array.isArray(schema.required)) {
+      for (const k of schema.required) if (!(k in obj)) errors.push({ path: path ? `${path}.${k}` : k, message: "Required property missing" });
+    }
+    if (schema.properties) {
+      for (const [k, sub] of Object.entries(schema.properties)) if (k in obj) _validate(obj[k], sub, path ? `${path}.${k}` : k, errors);
+    }
+  }
+  if (Array.isArray(schema.allOf)) for (const sub of schema.allOf) _validate(data, sub, path, errors);
+  if (Array.isArray(schema.anyOf)) {
+    const any = schema.anyOf.map((sub) => {
+      const e = [];
+      _validate(data, sub, path, e);
+      return e;
+    });
+    if (any.every((e) => e.length > 0)) errors.push({ path, message: "Must match at least one schema in anyOf" });
+  }
+}
+function validateJsonSchema(data, schema) {
+  const errors = [];
+  _validate(data, schema, "", errors);
+  return { valid: errors.length === 0, errors };
+}
+function _isObjD(v) {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function _deepEq(a, b) {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (Array.isArray(a) && Array.isArray(b)) return a.length === b.length && a.every((v, i) => _deepEq(v, b[i]));
+  if (_isObjD(a) && _isObjD(b)) {
+    const ka = Object.keys(a), kb = Object.keys(b);
+    return ka.length === kb.length && ka.every((k) => _deepEq(a[k], b[k]));
+  }
+  return false;
+}
+function _walkDiff(a, b, path, entries) {
+  if (_isObjD(a) && _isObjD(b)) {
+    for (const k of /* @__PURE__ */ new Set([...Object.keys(a), ...Object.keys(b)])) {
+      const p = path ? `${path}.${k}` : k;
+      if (!(k in a)) entries.push({ path: p, op: "added", newValue: b[k] });
+      else if (!(k in b)) entries.push({ path: p, op: "removed", oldValue: a[k] });
+      else _walkDiff(a[k], b[k], p, entries);
+    }
+  } else if (Array.isArray(a) && Array.isArray(b)) {
+    const len = Math.max(a.length, b.length);
+    for (let i = 0; i < len; i++) {
+      const p = `${path}[${i}]`;
+      if (i >= a.length) entries.push({ path: p, op: "added", newValue: b[i] });
+      else if (i >= b.length) entries.push({ path: p, op: "removed", oldValue: a[i] });
+      else _walkDiff(a[i], b[i], p, entries);
+    }
+  } else {
+    if (_deepEq(a, b)) entries.push({ path, op: "unchanged", oldValue: a, newValue: b });
+    else entries.push({ path, op: "changed", oldValue: a, newValue: b });
+  }
+}
+function diffJson(a, b) {
+  const entries = [];
+  _walkDiff(a, b, "", entries);
+  return { entries, added: entries.filter((e) => e.op === "added").length, removed: entries.filter((e) => e.op === "removed").length, changed: entries.filter((e) => e.op === "changed").length, unchanged: entries.filter((e) => e.op === "unchanged").length, totalPaths: entries.length };
+}
+function _stripTags(h) {
+  return h.replace(/<[^>]+>/g, "").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&nbsp;/g, " ").replace(/&quot;/g, '"').trim();
+}
+function _cells(rowHtml, tag) {
+  const c = [], re = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, "gi");
+  let m;
+  while ((m = re.exec(rowHtml)) !== null) c.push(_stripTags(m[1]));
+  return c;
+}
+function extractTables(html) {
+  const tables = [];
+  const tr = /<table[^>]*>([\s\S]*?)<\/table>/gi;
+  let tm;
+  let idx = 0;
+  while ((tm = tr.exec(html)) !== null) {
+    const body = tm[1], capM = body.match(/<caption[^>]*>([\s\S]*?)<\/caption>/i), caption = capM ? _stripTags(capM[1]) : "";
+    let headers = [];
+    const theadM = body.match(/<thead[^>]*>([\s\S]*?)<\/thead>/i);
+    if (theadM) {
+      const r = theadM[1].match(/<tr[^>]*>([\s\S]*?)<\/tr>/i);
+      if (r) headers = _cells(r[1], "th").concat(_cells(r[1], "td"));
+    }
+    if (!headers.length) {
+      const f = body.match(/<tr[^>]*>([\s\S]*?)<\/tr>/i);
+      if (f) headers = _cells(f[1], "th");
+    }
+    const rows = [], tbM = body.match(/<tbody[^>]*>([\s\S]*?)<\/tbody>/i), src = tbM ? tbM[1] : body;
+    const rr = /<tr[^>]*>([\s\S]*?)<\/tr>/gi;
+    let rm;
+    while ((rm = rr.exec(src)) !== null) {
+      const c = _cells(rm[1], "td");
+      if (c.length) rows.push(c);
+    }
+    tables.push({ index: idx++, caption, headers, rows, rowCount: rows.length, colCount: Math.max(headers.length, ...rows.map((r) => r.length)) });
+  }
+  return { tables, count: tables.length };
+}
+var _ENT_PATTERNS = [
+  { type: "email", re: /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g },
+  { type: "phone", re: /(?<!\d)(\+?1[\s.-]?)?(\(?\d{3}\)?[\s.\-]?\d{3}[\s.\-]?\d{4})(?!\d)/g },
+  { type: "date", re: /\b(\d{4}[-\/]\d{2}[-\/]\d{2}|\d{2}[-\/]\d{2}[-\/]\d{4}|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4})\b/gi },
+  { type: "ip", re: /\b(?:(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\.){3}(?:25[0-5]|2[0-4]\d|[01]?\d\d?)\b/g },
+  { type: "creditCard", re: /\b(?:\d{4}[\s\-]?){3}\d{4}\b/g },
+  { type: "ssn", re: /\b\d{3}-\d{2}-\d{4}\b/g },
+  { type: "iban", re: /\b[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z0-9]?){0,16}\b/g },
+  { type: "hashtag", re: /#[a-zA-Z]\w*/g },
+  { type: "mention", re: /@[a-zA-Z]\w*/g },
+  { type: "currency", re: /(?:USD|EUR|GBP|JPY|CAD|AUD|CHF|CNY|INR)\s*[\d,]+(?:\.\d{1,2})?|[\$£€¥₹]\s*[\d,]+(?:\.\d{1,2})?/g }
+];
+function extractEntities(text) {
+  const entities = [], seen = /* @__PURE__ */ new Set();
+  for (const { type, re } of _ENT_PATTERNS) {
+    re.lastIndex = 0;
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      const k = `${type}:${m[0]}`;
+      if (!seen.has(k)) {
+        seen.add(k);
+        entities.push({ type, value: m[0].trim(), position: m.index });
+      }
+    }
+  }
+  entities.sort((a, b) => a.position - b.position);
+  const byType = {};
+  for (const e of entities) {
+    if (!byType[e.type]) byType[e.type] = [];
+    byType[e.type].push(e);
+  }
+  return { entities, byType, count: entities.length };
+}
+function compressHtml(html, opts = {}) {
+  const { removeComments = true, removeScripts = true, removeStyles = true, collapseWhitespace = true } = opts;
+  let s = html;
+  if (removeScripts) s = s.replace(/<script[\s\S]*?<\/script>/gi, "");
+  if (removeStyles) s = s.replace(/<style[\s\S]*?<\/style>/gi, "");
+  if (removeComments) s = s.replace(/<!--[\s\S]*?-->/g, "");
+  s = s.replace(/\s+data-[a-z\-]+="[^"]*"/gi, "").replace(/\s+class="[^"]*"/gi, "").replace(/\s+style="[^"]*"/gi, "").replace(/\s+id="[^"]*"/gi, "");
+  if (collapseWhitespace) {
+    s = s.replace(/>\s+</g, "><").replace(/\s{2,}/g, " ").trim();
+  }
+  const enc = (t) => Buffer.byteLength(t, "utf8");
+  const originalBytes = enc(html), compressedBytes = enc(s), originalTokens = Math.ceil(html.length / 4), compressedTokens = Math.ceil(s.length / 4);
+  return { compressed: s, originalBytes, compressedBytes, originalTokens, compressedTokens, savings: originalBytes - compressedBytes, savingsPct: originalBytes > 0 ? Math.round((originalBytes - compressedBytes) / originalBytes * 100) : 0 };
+}
+function compressMarkdown(md, opts = {}) {
+  const { collapseBlankLines = true, removeComments = true, stripFrontmatter = false, trimLines = true, removeHorizontalRules = false } = opts;
+  const changes = [];
+  let s = md;
+  if (stripFrontmatter) {
+    const m = s.match(/^---\n[\s\S]*?\n---\n/);
+    if (m) {
+      s = s.slice(m[0].length);
+      changes.push("Stripped frontmatter");
+    }
+  }
+  if (removeComments) {
+    const b = s.length;
+    s = s.replace(/<!--[\s\S]*?-->/g, "");
+    if (s.length < b) changes.push("Removed HTML comments");
+  }
+  if (trimLines) {
+    const b = s;
+    s = s.split("\n").map((l) => l.trimEnd()).join("\n");
+    if (s !== b) changes.push("Trimmed trailing whitespace");
+  }
+  if (collapseBlankLines) {
+    const b = s;
+    s = s.replace(/\n{3,}/g, "\n\n");
+    if (s !== b) changes.push("Collapsed blank lines");
+  }
+  if (removeHorizontalRules) {
+    const b = s;
+    s = s.replace(/^[-*_]{3,}\s*$/gm, "");
+    if (s !== b) changes.push("Removed horizontal rules");
+  }
+  const lines = s.split("\n");
+  const result = [];
+  let inCode = false;
+  for (const line of lines) {
+    if (line.startsWith("```")) inCode = !inCode;
+    if (!inCode && !line.startsWith("    ") && !line.startsWith("	")) result.push(line.replace(/ {2,}/g, " "));
+    else result.push(line);
+  }
+  s = result.join("\n").trim();
+  const ot = Math.ceil(md.length / 4), ct = Math.ceil(s.length / 4);
+  return { compressed: s, originalTokens: ot, compressedTokens: ct, savings: ot - ct, savingsPct: ot > 0 ? Math.round((ot - ct) / ot * 100) : 0, changes };
+}
+function _compressVal(val, opts, s) {
+  if (val === null) return val;
+  if (Array.isArray(val)) {
+    return val.map((v) => _compressVal(v, opts, s)).filter((v) => {
+      if (opts.removeNulls && v === null) {
+        s.nulls++;
+        return false;
+      }
+      if (opts.removeEmptyArrays && Array.isArray(v) && v.length === 0) {
+        s.empArr++;
+        return false;
+      }
+      if (opts.removeEmptyObjects && typeof v === "object" && v !== null && !Array.isArray(v) && Object.keys(v).length === 0) {
+        s.empObj++;
+        return false;
+      }
+      return true;
+    });
+  }
+  if (typeof val === "object") {
+    const obj = val;
+    const keys = opts.sortKeys ? Object.keys(obj).sort() : Object.keys(obj);
+    const r = {};
+    for (const k of keys) {
+      const v = _compressVal(obj[k], opts, s);
+      if (opts.removeNulls && v === null) {
+        s.nulls++;
+        continue;
+      }
+      if (opts.removeEmptyArrays && Array.isArray(v) && v.length === 0) {
+        s.empArr++;
+        continue;
+      }
+      if (opts.removeEmptyObjects && typeof v === "object" && v !== null && !Array.isArray(v) && Object.keys(v).length === 0) {
+        s.empObj++;
+        continue;
+      }
+      r[k] = v;
+    }
+    return r;
+  }
+  return val;
+}
+function compressJson(json, opts = {}) {
+  const { removeNulls = true, removeEmptyArrays = true, removeEmptyObjects = true, sortKeys = false } = opts;
+  const origStr = JSON.stringify(json);
+  const s = { nulls: 0, empArr: 0, empObj: 0 };
+  const compressed = _compressVal(json, { removeNulls, removeEmptyArrays, removeEmptyObjects, sortKeys }, s);
+  const compStr = JSON.stringify(compressed);
+  const ot = Math.ceil(origStr.length / 4), ct = Math.ceil(compStr.length / 4);
+  return { compressed: compStr, originalTokens: ot, compressedTokens: ct, savings: ot - ct, savingsPct: ot > 0 ? Math.round((ot - ct) / ot * 100) : 0, removedNulls: s.nulls, removedEmptyArrays: s.empArr, removedEmptyObjects: s.empObj };
+}
+var _RR_STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "is", "are", "was", "were", "be", "been", "have", "has", "had", "it", "this", "that", "these", "those", "i", "we", "you", "he", "she", "they"]);
+function _tok(text) {
+  return text.toLowerCase().match(/\b[a-z]{2,}\b/g)?.filter((w) => !_RR_STOP.has(w)) ?? [];
+}
+function rerankChunks(query, chunks, topK) {
+  const qt = _tok(query), tc = chunks.map((c) => _tok(c));
+  const idf = /* @__PURE__ */ new Map();
+  for (const t of qt) {
+    const df = tc.filter((c) => c.includes(t)).length;
+    idf.set(t, Math.log((chunks.length + 1) / (df + 1)) + 1);
+  }
+  const ranked = chunks.map((text, index) => {
+    const counts = /* @__PURE__ */ new Map();
+    for (const t of tc[index]) counts.set(t, (counts.get(t) ?? 0) + 1);
+    const max = Math.max(1, ...counts.values());
+    let score = 0;
+    const matched = [];
+    for (const t of qt) {
+      const s = (counts.get(t) ?? 0) / max * (idf.get(t) ?? 1);
+      if (s > 0) {
+        score += s;
+        matched.push(t);
+      }
+    }
+    return { index, text, score: Math.round(score * 1e3) / 1e3, matchedTerms: [...new Set(matched)] };
+  });
+  ranked.sort((a, b) => b.score - a.score);
+  return { ranked: topK ? ranked.slice(0, topK) : ranked, query, totalChunks: chunks.length };
+}
+function scoreRelevance(query, passage) {
+  const qt = [...new Set(_tok(query))], pt = _tok(passage);
+  const counts = /* @__PURE__ */ new Map();
+  for (const t of pt) counts.set(t, (counts.get(t) ?? 0) + 1);
+  const max = Math.max(1, ...counts.values());
+  let score = 0;
+  const matched = [];
+  for (const t of qt) {
+    const c = counts.get(t) ?? 0;
+    if (c > 0) {
+      score += c / max * (1 / Math.log2(pt.length + 2));
+      matched.push(t);
+    }
+  }
+  const coverage = qt.length > 0 ? matched.length / qt.length : 0;
+  const norm = Math.min(1, score * qt.length);
+  const fs = Math.round((norm * 0.6 + coverage * 0.4) * 100) / 100;
+  const grade = fs >= 0.6 ? "high" : fs >= 0.35 ? "medium" : fs > 0 ? "low" : "none";
+  return { score: fs, grade, matchedTerms: matched, coverage: Math.round(coverage * 100) / 100 };
+}
+var _SYN = { fast: ["quick", "rapid"], slow: ["sluggish", "delayed"], error: ["bug", "fault", "exception"], fix: ["repair", "resolve", "patch"], search: ["query", "find", "retrieve"], document: ["file", "text", "content"], user: ["customer", "client"], data: ["information", "records"], api: ["endpoint", "service"], build: ["compile", "create"], delete: ["remove", "drop", "erase"], update: ["modify", "edit", "change"], create: ["add", "insert", "generate"], get: ["fetch", "retrieve", "obtain"], list: ["enumerate", "show", "index"], send: ["post", "submit", "transmit"], check: ["verify", "validate", "inspect"], large: ["big", "huge", "massive"], small: ["tiny", "minimal", "compact"], improve: ["enhance", "optimize", "boost"], analyze: ["examine", "evaluate", "assess"], model: ["llm", "ai", "classifier"], token: ["word", "subword", "unit"], chunk: ["segment", "fragment", "block"], embed: ["encode", "vectorize", "represent"], score: ["rank", "rate", "evaluate"], similar: ["related", "relevant", "matching"], agent: ["bot", "assistant", "worker"], memory: ["context", "history", "state"], cost: ["price", "expense", "fee"], deploy: ["release", "ship", "launch"] };
+var _QE_STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "is", "are", "was"]);
+function expandQuery(query, maxSynonyms = 2) {
+  const words = query.toLowerCase().match(/\b[a-z]{2,}\b/g) ?? [];
+  const added = [];
+  const terms = [...words];
+  for (const w of words) {
+    if (_QE_STOP.has(w)) continue;
+    const s = _SYN[w];
+    if (s) {
+      for (const syn of s.slice(0, maxSynonyms)) if (!terms.includes(syn)) {
+        terms.push(syn);
+        added.push(syn);
+      }
+    }
+  }
+  for (const w of words) {
+    let stem = "";
+    if (w.endsWith("ing")) stem = w.slice(0, -3);
+    else if (w.endsWith("ed")) stem = w.slice(0, -2);
+    else if (w.endsWith("s") && !w.endsWith("ss")) stem = w.slice(0, -1);
+    else if (w.endsWith("er")) stem = w.slice(0, -2);
+    if (stem.length >= 3 && !terms.includes(stem)) {
+      terms.push(stem);
+      added.push(stem);
+    }
+  }
+  return { original: query, expanded: terms.join(" "), terms, synonymsAdded: added };
+}
+var _SUM_STOP = /* @__PURE__ */ new Set(["a", "an", "the", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with", "by", "from", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall", "can", "not", "no", "it", "its", "this", "that", "these", "those", "i", "we", "you", "he", "she", "they", "their", "our", "your", "his", "her"]);
+function _sumTok(text) {
+  return text.toLowerCase().match(/\b[a-z]{3,}\b/g)?.filter((w) => !_SUM_STOP.has(w)) ?? [];
+}
+function _splitSents(text) {
+  return text.match(/[^.!?\n]+[.!?\n]+(\s|$)|[^.!?\n]+$/g)?.map((s) => s.trim()).filter(Boolean) ?? [text];
+}
+function summarizeForLlm(text, maxTokens = 200, strategy = "extractive") {
+  const sents = _splitSents(text);
+  const ot = Math.ceil(text.length / 4);
+  const tok = (s) => Math.ceil(s.length / 4);
+  if (strategy === "first") {
+    let sum = "";
+    for (const s of sents) {
+      const c = sum ? sum + " " + s : s;
+      if (tok(c) > maxTokens) break;
+      sum = c;
+    }
+    const st2 = tok(sum);
+    return { summary: sum, originalTokens: ot, summaryTokens: st2, compressionRatio: Math.round(st2 / Math.max(1, ot) * 100) / 100, sentencesKept: sum.split(/[.!?]/).filter(Boolean).length, sentencesTotal: sents.length };
+  }
+  if (strategy === "last") {
+    const rev = [...sents].reverse();
+    let sum = "";
+    for (const s of rev) {
+      const c = s + (sum ? " " + sum : "");
+      if (tok(c) > maxTokens) break;
+      sum = c;
+    }
+    const st2 = tok(sum);
+    return { summary: sum, originalTokens: ot, summaryTokens: st2, compressionRatio: Math.round(st2 / Math.max(1, ot) * 100) / 100, sentencesKept: sum.split(/[.!?]/).filter(Boolean).length, sentencesTotal: sents.length };
+  }
+  const allTok = _sumTok(text);
+  const freq = /* @__PURE__ */ new Map();
+  for (const t of allTok) freq.set(t, (freq.get(t) ?? 0) + 1);
+  const maxF = Math.max(1, ...freq.values());
+  const scored = sents.map((s, i) => {
+    const ts = _sumTok(s);
+    const sc = ts.reduce((sum, t) => sum + (freq.get(t) ?? 0) / maxF, 0) / Math.max(1, ts.length);
+    return { s, i, score: sc + (i === 0 || i === sents.length - 1 ? 0.1 : 0) };
+  });
+  const picked = [];
+  let used = 0;
+  for (const { s, i } of [...scored].sort((a, b) => b.score - a.score)) {
+    const t = tok(s);
+    if (used + t > maxTokens) continue;
+    picked.push({ s, i });
+    used += t;
+  }
+  picked.sort((a, b) => a.i - b.i);
+  const summary = picked.map((p) => p.s).join(" ");
+  const st = tok(summary);
+  return { summary, originalTokens: ot, summaryTokens: st, compressionRatio: Math.round(st / Math.max(1, ot) * 100) / 100, sentencesKept: picked.length, sentencesTotal: sents.length };
+}
+
+exports.MODEL_PRICING = MODEL_PRICING;
+exports.chunkText = chunkText;
+exports.compressHtml = compressHtml;
+exports.compressJson = compressJson;
+exports.compressMarkdown = compressMarkdown;
+exports.deduplicateLines = deduplicateLines;
+exports.detectPii = detectPii;
+exports.detectPromptInjection = detectPromptInjection;
+exports.detectSecrets = detectSecrets;
+exports.diffJson = diffJson;
+exports.estimateCost = estimateCost;
+exports.estimateTokens = estimateTokens;
+exports.expandQuery = expandQuery;
+exports.extractEntities = extractEntities;
+exports.extractJson = extractJson;
+exports.extractKeywords = extractKeywords;
+exports.extractTables = extractTables;
+exports.extractUrls = extractUrls;
+exports.flattenJson = flattenJson;
+exports.mergeJson = mergeJson;
+exports.redactPii = redactPii;
+exports.repairJson = repairJson;
+exports.rerankChunks = rerankChunks;
+exports.sanitizeHtml = sanitizeHtml;
+exports.scoreRelevance = scoreRelevance;
+exports.summarizeForLlm = summarizeForLlm;
+exports.trimToTokens = trimToTokens;
+exports.unflattenJson = unflattenJson;
+exports.validateJsonSchema = validateJsonSchema;