illuma-agents 1.0.15 → 1.0.17

package/dist/esm/utils/toonFormat.mjs

@@ -0,0 +1,351 @@
+/**
+ * TOON Format Utility
+ *
+ * Provides JSON to TOON conversion for token-efficient encoding.
+ * TOON format can reduce token count by 30-60% for JSON data.
+ *
+ * Uses the official @toon-format/toon library when available.
+ */
+// Dynamic import holder for the TOON library (ESM)
+let toonEncode = null;
+let toonLoadAttempted = false;
+let toonLoadPromise = null;
+/**
+ * Lazily loads the TOON library
+ * Tries both ESM dynamic import and CommonJS require for compatibility
+ */
+async function loadToonLibrary() {
+    if (toonLoadAttempted)
+        return toonEncode !== null;
+    toonLoadAttempted = true;
+    // Try ESM dynamic import first
+    try {
+        const mod = await import('@toon-format/toon');
+        toonEncode = mod.encode;
+        return true;
+    }
+    catch {
+        // ESM import failed, try CommonJS require
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const mod = require('@toon-format/toon');
+            toonEncode = mod.encode;
+            return true;
+        }
+        catch {
+            // Library not available - that's OK, we'll return original content
+            return false;
+        }
+    }
+}
+/**
+ * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.
+ */
+async function ensureToonLoaded() {
+    if (!toonLoadPromise) {
+        toonLoadPromise = loadToonLibrary();
+    }
+    return toonLoadPromise;
+}
+// Start loading immediately
+ensureToonLoaded();
+/**
+ * Check if a string appears to be in TOON format.
+ * TOON format characteristics:
+ * - Uses key: value syntax (like YAML)
+ * - Uses array notation like [3] or {fields}
+ * - Does NOT start with { or [
+ * - Often has patterns like "name[N]:" or "name{fields}:"
+ *
+ * @param str - String to check
+ * @returns true if string appears to be TOON format
+ */
+function isToonFormat(str) {
+    if (typeof str !== 'string' || str.length === 0)
+        return false;
+    const trimmed = str.trim();
+    // TOON doesn't start with JSON brackets
+    if (trimmed.startsWith('{') || trimmed.startsWith('['))
+        return false;
+    // Check for TOON-specific patterns:
+    // 1. Key-value with colon (but not URL-like patterns)
+    // 2. Array notation like "items[3]:" or "data[10]:"
+    // 3. Object schema notation like "items{id,name}:"
+    const toonPatterns = [
+        /^\w+:$/m, // Simple key: at start of line
+        /^\w+\[\d+\]:/m, // Array notation: items[3]:
+        /^\w+\{[^}]+\}:/m, // Schema notation: items{id,name}:
+        /^\w+\[\d+\]\{[^}]+\}:/m, // Combined: items[3]{id,name}:
+    ];
+    // Must match at least one TOON pattern
+    const hasToonPattern = toonPatterns.some(pattern => pattern.test(trimmed));
+    // Additional check: TOON typically has multiple lines with consistent indentation
+    const lines = trimmed.split('\n').filter(l => l.trim());
+    const hasMultipleKeyValueLines = lines.filter(l => /^\s*\w+.*:/.test(l)).length >= 2;
+    return hasToonPattern || hasMultipleKeyValueLines;
+}
+/**
+ * Extract the first valid JSON object or array from a string.
+ * Handles pure JSON or JSON embedded in text (e.g., tool responses).
+ *
+ * @param str - The string to extract JSON from
+ * @returns Object with found status, parsed JSON, and indices
+ */
+function extractFirstJson(str) {
+    if (typeof str !== 'string' || str.length === 0) {
+        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+    }
+    // Find the first { or [ character
+    const objStart = str.indexOf('{');
+    const arrStart = str.indexOf('[');
+    if (objStart === -1 && arrStart === -1) {
+        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+    }
+    let startIndex;
+    if (objStart === -1) {
+        startIndex = arrStart;
+    }
+    else if (arrStart === -1) {
+        startIndex = objStart;
+    }
+    else {
+        startIndex = Math.min(objStart, arrStart);
+    }
+    // Find the matching closing bracket using a stack-based approach
+    // Properly handle strings to avoid counting brackets inside strings
+    let depth = 0;
+    let inString = false;
+    let escapeNext = false;
+    for (let i = startIndex; i < str.length; i++) {
+        const char = str[i];
+        if (escapeNext) {
+            escapeNext = false;
+            continue;
+        }
+        if (char === '\\' && inString) {
+            escapeNext = true;
+            continue;
+        }
+        if (char === '"') {
+            inString = !inString;
+            continue;
+        }
+        if (inString)
+            continue;
+        if (char === '{' || char === '[')
+            depth++;
+        if (char === '}' || char === ']')
+            depth--;
+        if (depth === 0) {
+            // Found matching bracket, try to parse
+            const jsonStr = str.substring(startIndex, i + 1);
+            try {
+                const parsed = JSON.parse(jsonStr);
+                return { found: true, parsed, startIndex, endIndex: i };
+            }
+            catch {
+                // Not valid JSON at this bracket level, continue searching
+                // Reset and look for the next JSON start after this position
+                const nextResult = extractFirstJson(str.substring(i + 1));
+                if (nextResult.found) {
+                    return {
+                        found: true,
+                        parsed: nextResult.parsed,
+                        startIndex: i + 1 + nextResult.startIndex,
+                        endIndex: i + 1 + nextResult.endIndex,
+                    };
+                }
+                return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+            }
+        }
+    }
+    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+}
+/**
+ * Convert JSON content to TOON format for token efficiency.
+ * Extracts JSON from string if embedded, converts to TOON.
+ * Returns original string if:
+ * - Already in TOON format
+ * - Not JSON
+ * - TOON conversion fails or is larger
+ *
+ * @param str - The string containing JSON to convert
+ * @returns Object with conversion status, result, and reduction percentage
+ */
+function jsonToToon(str) {
+    if (!toonEncode || typeof str !== 'string') {
+        return { converted: false, result: str, reduction: 0, error: !toonEncode ? 'TOON library not loaded' : undefined };
+    }
+    // Check if already in TOON format - skip conversion
+    if (isToonFormat(str)) {
+        return { converted: false, result: str, reduction: 0, alreadyToon: true };
+    }
+    try {
+        const { found, parsed, startIndex, endIndex } = extractFirstJson(str);
+        if (!found || !parsed) {
+            return { converted: false, result: str, reduction: 0, error: 'No JSON found in content' };
+        }
+        // Check if this JSON structure would benefit from TOON
+        // Text-heavy content (emails, documents) won't compress well
+        if (!isToonBeneficial(parsed)) {
+            return { converted: false, result: str, reduction: 0, error: 'Content is text-heavy (>70% string values), TOON not beneficial' };
+        }
+        const toonResult = toonEncode(parsed);
+        // Preserve text before and after the JSON block
+        const textBefore = str.substring(0, startIndex);
+        const textAfter = str.substring(endIndex + 1);
+        // Build the full result: textBefore + TOON + textAfter
+        const fullResult = textBefore + toonResult + textAfter;
+        if (fullResult.length < str.length) {
+            const reduction = Math.round(((str.length - fullResult.length) / str.length) * 100);
+            return { converted: true, result: fullResult, reduction };
+        }
+        else {
+            // TOON output was larger or same size - not beneficial
+            return { converted: false, result: str, reduction: 0, error: `TOON output not smaller (${fullResult.length} >= ${str.length})` };
+        }
+    }
+    catch (err) {
+        // TOON encoding or extraction failed - log error for debugging
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        return { converted: false, result: str, reduction: 0, error: `TOON conversion error: ${errorMsg}` };
+    }
+}
+/**
+ * Check if TOON library is loaded and available
+ */
+function isToonAvailable() {
+    return toonEncode !== null;
+}
+/**
+ * Analyze JSON structure to determine if TOON conversion would be beneficial.
+ * Text-heavy content (emails, documents) doesn't compress well with TOON.
+ * Structured data (API responses, metadata) compresses much better.
+ *
+ * @param parsed - Parsed JSON object
+ * @returns true if TOON conversion is likely beneficial
+ */
+function isToonBeneficial(parsed) {
+    if (!parsed || typeof parsed !== 'object')
+        return false;
+    const jsonStr = JSON.stringify(parsed);
+    const totalLength = jsonStr.length;
+    // Count characters in string values (text content)
+    let textContentLength = 0;
+    function countTextContent(obj) {
+        if (typeof obj === 'string') {
+            // Count string length (this is text content that TOON can't compress)
+            textContentLength += obj.length;
+        }
+        else if (Array.isArray(obj)) {
+            obj.forEach(countTextContent);
+        }
+        else if (obj && typeof obj === 'object') {
+            Object.values(obj).forEach(countTextContent);
+        }
+    }
+    countTextContent(parsed);
+    // Calculate ratio of text content to total JSON
+    const textRatio = textContentLength / totalLength;
+    // If more than 70% is text content, TOON won't help much
+    // TOON compresses structure (field names, brackets), not text
+    return textRatio < 0.7;
+}
+/**
+ * Process tool output: apply TOON conversion if beneficial, then truncate if needed.
+ * This is the main entry point for processing any tool output (regular or MCP).
+ *
+ * Flow:
+ * 1. Check if already TOON format → skip conversion
+ * 2. Try TOON conversion if content is JSON and large enough
+ * 3. Truncate if still exceeds maxLength (with smart break points)
+ *
+ * @param content - The tool output content
+ * @param options - Processing options
+ * @returns Processed content and metadata
+ */
+function processToolOutput(content, options = {}) {
+    const { maxLength = 100000, enableToon = true, minSizeForToon = 1000, minReductionPercent = 10, // Increased from 5% - only apply TOON when clearly beneficial
+     } = options;
+    const originalLength = content.length;
+    const originalTokens = Math.ceil(originalLength / 4);
+    let result = content;
+    let toonConverted = false;
+    let alreadyToon = false;
+    let reduction = 0;
+    let truncated = false;
+    let toonError;
+    // Step 1: Check if already TOON format
+    if (isToonFormat(content)) {
+        alreadyToon = true;
+    }
+    // Step 2: Apply TOON conversion if enabled and content is large enough
+    else if (enableToon && content.length > minSizeForToon) {
+        const toonResult = jsonToToon(content);
+        if (toonResult.alreadyToon) {
+            alreadyToon = true;
+        }
+        else if (toonResult.converted && toonResult.reduction >= minReductionPercent) {
+            result = toonResult.result;
+            toonConverted = true;
+            reduction = toonResult.reduction;
+        }
+        else if (toonResult.error) {
+            // Track error for debugging
+            toonError = toonResult.error;
+        }
+        else if (toonResult.converted && toonResult.reduction < minReductionPercent) {
+            // TOON converted but reduction was too small to be worth it
+            toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;
+        }
+        else if (!toonResult.converted) {
+            // Conversion failed without explicit error - investigate
+            toonError = toonResult.error || 'TOON conversion returned false with no error';
+        }
+    }
+    // Step 3: Truncate if still too long (with smart break points)
+    if (result.length > maxLength) {
+        let truncatedStr = result.substring(0, maxLength);
+        // Try to find a clean break point
+        if (toonConverted || alreadyToon) {
+            // For TOON format, break at newline for cleaner output
+            const lastNewline = truncatedStr.lastIndexOf('\n');
+            if (lastNewline > maxLength * 0.7) {
+                truncatedStr = truncatedStr.substring(0, lastNewline);
+            }
+        }
+        else {
+            // For JSON, try to find a clean JSON break point
+            const lastCompleteItem = truncatedStr.lastIndexOf('},');
+            const lastArrayItem = truncatedStr.lastIndexOf('],');
+            const breakPoint = Math.max(lastCompleteItem, lastArrayItem);
+            if (breakPoint > maxLength * 0.5) {
+                truncatedStr = truncatedStr.substring(0, breakPoint + 1);
+            }
+        }
+        // Build truncation message
+        const truncationInfo = toonConverted
+            ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +
+                `Still exceeds ${maxLength.toLocaleString()} char limit.`
+            : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;
+        result = truncatedStr + `\n\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;
+        truncated = true;
+    }
+    const finalLength = result.length;
+    const finalTokens = Math.ceil(finalLength / 4);
+    return {
+        content: result,
+        toonConverted,
+        truncated,
+        reduction,
+        alreadyToon,
+        originalLength,
+        originalTokens,
+        finalLength,
+        finalTokens,
+        toonError,
+    };
+}
+
+export { ensureToonLoaded, extractFirstJson, isToonAvailable, isToonFormat, jsonToToon, processToolOutput };
+//# sourceMappingURL=toonFormat.mjs.map

package/dist/esm/utils/toonFormat.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"toonFormat.mjs","sources":["../../../src/utils/toonFormat.ts"],"sourcesContent":["/**\r\n * TOON Format Utility\r\n * \r\n * Provides JSON to TOON conversion for token-efficient encoding.\r\n * TOON format can reduce token count by 30-60% for JSON data.\r\n * \r\n * Uses the official @toon-format/toon library when available.\r\n */\r\n\r\n// Dynamic import holder for the TOON library (ESM)\r\nlet toonEncode: ((data: unknown) => string) | null = null;\r\nlet toonLoadAttempted = false;\r\nlet toonLoadPromise: Promise<boolean> | null = null;\r\n\r\n/**\r\n * Lazily loads the TOON library\r\n * Tries both ESM dynamic import and CommonJS require for compatibility\r\n */\r\nasync function loadToonLibrary(): Promise<boolean> {\r\n  if (toonLoadAttempted) return toonEncode !== null;\r\n  toonLoadAttempted = true;\r\n\r\n  // Try ESM dynamic import first\r\n  try {\r\n    const mod = await import('@toon-format/toon');\r\n    toonEncode = mod.encode;\r\n    return true;\r\n  } catch {\r\n    // ESM import failed, try CommonJS require\r\n    try {\r\n      // eslint-disable-next-line @typescript-eslint/no-require-imports\r\n      const mod = require('@toon-format/toon');\r\n      toonEncode = mod.encode;\r\n      return true;\r\n    } catch {\r\n      // Library not available - that's OK, we'll return original content\r\n      return false;\r\n    }\r\n  }\r\n}\r\n\r\n/**\r\n * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.\r\n */\r\nexport async function ensureToonLoaded(): Promise<boolean> {\r\n  if (!toonLoadPromise) {\r\n    toonLoadPromise = loadToonLibrary();\r\n  }\r\n  return toonLoadPromise;\r\n}\r\n\r\n// Start loading immediately\r\nensureToonLoaded();\r\n\r\n/**\r\n * Check if a string appears to be in TOON format.\r\n * TOON format characteristics:\r\n * - Uses key: value syntax (like YAML)\r\n * - Uses array notation like [3] or {fields}\r\n * - Does NOT start with { or [\r\n * - Often has patterns like \"name[N]:\" or \"name{fields}:\"\r\n * \r\n * @param str - String to check\r\n * @returns true if string appears to be TOON format\r\n */\r\nexport function isToonFormat(str: string): boolean {\r\n  if (typeof str !== 'string' || str.length === 0) return false;\r\n  \r\n  const trimmed = str.trim();\r\n  \r\n  // TOON doesn't start with JSON brackets\r\n  if (trimmed.startsWith('{') || trimmed.startsWith('[')) return false;\r\n  \r\n  // Check for TOON-specific patterns:\r\n  // 1. Key-value with colon (but not URL-like patterns)\r\n  // 2. Array notation like \"items[3]:\" or \"data[10]:\"\r\n  // 3. Object schema notation like \"items{id,name}:\"\r\n  const toonPatterns = [\r\n    /^\\w+:$/m,                      // Simple key: at start of line\r\n    /^\\w+\\[\\d+\\]:/m,                // Array notation: items[3]:\r\n    /^\\w+\\{[^}]+\\}:/m,              // Schema notation: items{id,name}:\r\n    /^\\w+\\[\\d+\\]\\{[^}]+\\}:/m,       // Combined: items[3]{id,name}:\r\n  ];\r\n  \r\n  // Must match at least one TOON pattern\r\n  const hasToonPattern = toonPatterns.some(pattern => pattern.test(trimmed));\r\n  \r\n  // Additional check: TOON typically has multiple lines with consistent indentation\r\n  const lines = trimmed.split('\\n').filter(l => l.trim());\r\n  const hasMultipleKeyValueLines = lines.filter(l => /^\\s*\\w+.*:/.test(l)).length >= 2;\r\n  \r\n  return hasToonPattern || hasMultipleKeyValueLines;\r\n}\r\n\r\n/**\r\n * Extract the first valid JSON object or array from a string.\r\n * Handles pure JSON or JSON embedded in text (e.g., tool responses).\r\n * \r\n * @param str - The string to extract JSON from\r\n * @returns Object with found status, parsed JSON, and indices\r\n */\r\nexport function extractFirstJson(str: string): {\r\n  found: boolean;\r\n  parsed: unknown;\r\n  startIndex: number;\r\n  endIndex: number;\r\n} {\r\n  if (typeof str !== 'string' || str.length === 0) {\r\n    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\r\n  }\r\n\r\n  // Find the first { or [ character\r\n  const objStart = str.indexOf('{');\r\n  const arrStart = str.indexOf('[');\r\n\r\n  if (objStart === -1 && arrStart === -1) {\r\n    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\r\n  }\r\n\r\n  let startIndex: number;\r\n  if (objStart === -1) {\r\n    startIndex = arrStart;\r\n  } else if (arrStart === -1) {\r\n    startIndex = objStart;\r\n  } else {\r\n    startIndex = Math.min(objStart, arrStart);\r\n  }\r\n\r\n  // Find the matching closing bracket using a stack-based approach\r\n  // Properly handle strings to avoid counting brackets inside strings\r\n  let depth = 0;\r\n  let inString = false;\r\n  let escapeNext = false;\r\n\r\n  for (let i = startIndex; i < str.length; i++) {\r\n    const char = str[i];\r\n\r\n    if (escapeNext) {\r\n      escapeNext = false;\r\n      continue;\r\n    }\r\n\r\n    if (char === '\\\\' && inString) {\r\n      escapeNext = true;\r\n      continue;\r\n    }\r\n\r\n    if (char === '\"') {\r\n      inString = !inString;\r\n      continue;\r\n    }\r\n\r\n    if (inString) continue;\r\n\r\n    if (char === '{' || char === '[') depth++;\r\n    if (char === '}' || char === ']') depth--;\r\n\r\n    if (depth === 0) {\r\n      // Found matching bracket, try to parse\r\n      const jsonStr = str.substring(startIndex, i + 1);\r\n      try {\r\n        const parsed = JSON.parse(jsonStr);\r\n        return { found: true, parsed, startIndex, endIndex: i };\r\n      } catch {\r\n        // Not valid JSON at this bracket level, continue searching\r\n        // Reset and look for the next JSON start after this position\r\n        const nextResult = extractFirstJson(str.substring(i + 1));\r\n        if (nextResult.found) {\r\n          return {\r\n            found: true,\r\n            parsed: nextResult.parsed,\r\n            startIndex: i + 1 + nextResult.startIndex,\r\n            endIndex: i + 1 + nextResult.endIndex,\r\n          };\r\n        }\r\n        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\r\n      }\r\n    }\r\n  }\r\n\r\n  return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\r\n}\r\n\r\n/**\r\n * Convert JSON content to TOON format for token efficiency.\r\n * Extracts JSON from string if embedded, converts to TOON.\r\n * Returns original string if:\r\n * - Already in TOON format\r\n * - Not JSON\r\n * - TOON conversion fails or is larger\r\n * \r\n * @param str - The string containing JSON to convert\r\n * @returns Object with conversion status, result, and reduction percentage\r\n */\r\nexport function jsonToToon(str: string): {\r\n  converted: boolean;\r\n  result: string;\r\n  reduction: number;\r\n  alreadyToon?: boolean;\r\n  error?: string;\r\n} {\r\n  if (!toonEncode || typeof str !== 'string') {\r\n    return { converted: false, result: str, reduction: 0, error: !toonEncode ? 'TOON library not loaded' : undefined };\r\n  }\r\n\r\n  // Check if already in TOON format - skip conversion\r\n  if (isToonFormat(str)) {\r\n    return { converted: false, result: str, reduction: 0, alreadyToon: true };\r\n  }\r\n\r\n  try {\r\n    const { found, parsed, startIndex, endIndex } = extractFirstJson(str);\r\n\r\n    if (!found || !parsed) {\r\n      return { converted: false, result: str, reduction: 0, error: 'No JSON found in content' };\r\n    }\r\n\r\n    // Check if this JSON structure would benefit from TOON\r\n    // Text-heavy content (emails, documents) won't compress well\r\n    if (!isToonBeneficial(parsed)) {\r\n      return { converted: false, result: str, reduction: 0, error: 'Content is text-heavy (>70% string values), TOON not beneficial' };\r\n    }\r\n\r\n    const toonResult = toonEncode(parsed);\r\n    \r\n    // Preserve text before and after the JSON block\r\n    const textBefore = str.substring(0, startIndex);\r\n    const textAfter = str.substring(endIndex + 1);\r\n    \r\n    // Build the full result: textBefore + TOON + textAfter\r\n    const fullResult = textBefore + toonResult + textAfter;\r\n\r\n    if (fullResult.length < str.length) {\r\n      const reduction = Math.round(((str.length - fullResult.length) / str.length) * 100);\r\n      return { converted: true, result: fullResult, reduction };\r\n    } else {\r\n      // TOON output was larger or same size - not beneficial\r\n      return { converted: false, result: str, reduction: 0, error: `TOON output not smaller (${fullResult.length} >= ${str.length})` };\r\n    }\r\n  } catch (err) {\r\n    // TOON encoding or extraction failed - log error for debugging\r\n    const errorMsg = err instanceof Error ? err.message : String(err);\r\n    return { converted: false, result: str, reduction: 0, error: `TOON conversion error: ${errorMsg}` };\r\n  }\r\n}\r\n\r\n/**\r\n * Check if TOON library is loaded and available\r\n */\r\nexport function isToonAvailable(): boolean {\r\n  return toonEncode !== null;\r\n}\r\n\r\n/**\r\n * Result of processing tool output\r\n */\r\nexport interface ProcessToolOutputResult {\r\n  /** The processed content string */\r\n  content: string;\r\n  /** Whether TOON conversion was applied */\r\n  toonConverted: boolean;\r\n  /** Whether content was truncated */\r\n  truncated: boolean;\r\n  /** Percentage reduction from TOON (0 if not converted) */\r\n  reduction: number;\r\n  /** Whether input was already in TOON format */\r\n  alreadyToon: boolean;\r\n  /** Original content length */\r\n  originalLength: number;\r\n  /** Estimated original tokens (~4 chars per token) */\r\n  originalTokens: number;\r\n  /** Final content length */\r\n  finalLength: number;\r\n  /** Estimated final tokens */\r\n  finalTokens: number;\r\n  /** Error message if TOON conversion failed (for debugging) */\r\n  toonError?: string;\r\n}\r\n\r\n/**\r\n * Options for processing tool output\r\n */\r\nexport interface ProcessToolOutputOptions {\r\n  /** Maximum output length in characters (default: 100000) */\r\n  maxLength?: number;\r\n  /** Whether to apply TOON conversion (default: true) */\r\n  enableToon?: boolean;\r\n  /** Minimum content size to attempt TOON conversion (default: 1000) */\r\n  minSizeForToon?: number;\r\n  /** Minimum reduction % to accept TOON result (default: 10) */\r\n  minReductionPercent?: number;\r\n}\r\n\r\n/**\r\n * Analyze JSON structure to determine if TOON conversion would be beneficial.\r\n * Text-heavy content (emails, documents) doesn't compress well with TOON.\r\n * Structured data (API responses, metadata) compresses much better.\r\n * \r\n * @param parsed - Parsed JSON object\r\n * @returns true if TOON conversion is likely beneficial\r\n */\r\nfunction isToonBeneficial(parsed: unknown): boolean {\r\n  if (!parsed || typeof parsed !== 'object') return false;\r\n  \r\n  const jsonStr = JSON.stringify(parsed);\r\n  const totalLength = jsonStr.length;\r\n  \r\n  // Count characters in string values (text content)\r\n  let textContentLength = 0;\r\n  \r\n  function countTextContent(obj: unknown): void {\r\n    if (typeof obj === 'string') {\r\n      // Count string length (this is text content that TOON can't compress)\r\n      textContentLength += obj.length;\r\n    } else if (Array.isArray(obj)) {\r\n      obj.forEach(countTextContent);\r\n    } else if (obj && typeof obj === 'object') {\r\n      Object.values(obj).forEach(countTextContent);\r\n    }\r\n  }\r\n  \r\n  countTextContent(parsed);\r\n  \r\n  // Calculate ratio of text content to total JSON\r\n  const textRatio = textContentLength / totalLength;\r\n  \r\n  // If more than 70% is text content, TOON won't help much\r\n  // TOON compresses structure (field names, brackets), not text\r\n  return textRatio < 0.7;\r\n}\r\n\r\n/**\r\n * Process tool output: apply TOON conversion if beneficial, then truncate if needed.\r\n * This is the main entry point for processing any tool output (regular or MCP).\r\n * \r\n * Flow:\r\n * 1. Check if already TOON format → skip conversion\r\n * 2. Try TOON conversion if content is JSON and large enough\r\n * 3. Truncate if still exceeds maxLength (with smart break points)\r\n * \r\n * @param content - The tool output content\r\n * @param options - Processing options\r\n * @returns Processed content and metadata\r\n */\r\nexport function processToolOutput(\r\n  content: string,\r\n  options: ProcessToolOutputOptions = {}\r\n): ProcessToolOutputResult {\r\n  const { \r\n    maxLength = 100000, \r\n    enableToon = true,\r\n    minSizeForToon = 1000,\r\n    minReductionPercent = 10,  // Increased from 5% - only apply TOON when clearly beneficial\r\n  } = options;\r\n\r\n  const originalLength = content.length;\r\n  const originalTokens = Math.ceil(originalLength / 4);\r\n  \r\n  let result = content;\r\n  let toonConverted = false;\r\n  let alreadyToon = false;\r\n  let reduction = 0;\r\n  let truncated = false;\r\n  let toonError: string | undefined;\r\n\r\n  // Step 1: Check if already TOON format\r\n  if (isToonFormat(content)) {\r\n    alreadyToon = true;\r\n  }\r\n  // Step 2: Apply TOON conversion if enabled and content is large enough\r\n  else if (enableToon && content.length > minSizeForToon) {\r\n    const toonResult = jsonToToon(content);\r\n    if (toonResult.alreadyToon) {\r\n      alreadyToon = true;\r\n    } else if (toonResult.converted && toonResult.reduction >= minReductionPercent) {\r\n      result = toonResult.result;\r\n      toonConverted = true;\r\n      reduction = toonResult.reduction;\r\n    } else if (toonResult.error) {\r\n      // Track error for debugging\r\n      toonError = toonResult.error;\r\n    } else if (toonResult.converted && toonResult.reduction < minReductionPercent) {\r\n      // TOON converted but reduction was too small to be worth it\r\n      toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;\r\n    } else if (!toonResult.converted) {\r\n      // Conversion failed without explicit error - investigate\r\n      toonError = toonResult.error || 'TOON conversion returned false with no error';\r\n    }\r\n  }\r\n\r\n  // Step 3: Truncate if still too long (with smart break points)\r\n  if (result.length > maxLength) {\r\n    let truncatedStr = result.substring(0, maxLength);\r\n    \r\n    // Try to find a clean break point\r\n    if (toonConverted || alreadyToon) {\r\n      // For TOON format, break at newline for cleaner output\r\n      const lastNewline = truncatedStr.lastIndexOf('\\n');\r\n      if (lastNewline > maxLength * 0.7) {\r\n        truncatedStr = truncatedStr.substring(0, lastNewline);\r\n      }\r\n    } else {\r\n      // For JSON, try to find a clean JSON break point\r\n      const lastCompleteItem = truncatedStr.lastIndexOf('},');\r\n      const lastArrayItem = truncatedStr.lastIndexOf('],');\r\n      const breakPoint = Math.max(lastCompleteItem, lastArrayItem);\r\n      if (breakPoint > maxLength * 0.5) {\r\n        truncatedStr = truncatedStr.substring(0, breakPoint + 1);\r\n      }\r\n    }\r\n    \r\n    // Build truncation message\r\n    const truncationInfo = toonConverted\r\n      ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +\r\n        `Still exceeds ${maxLength.toLocaleString()} char limit.`\r\n      : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;\r\n    \r\n    result = truncatedStr + `\\n\\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;\r\n    truncated = true;\r\n  }\r\n\r\n  const finalLength = result.length;\r\n  const finalTokens = Math.ceil(finalLength / 4);\r\n\r\n  return { \r\n    content: result, \r\n    toonConverted, \r\n    truncated, \r\n    reduction, \r\n    alreadyToon,\r\n    originalLength,\r\n    originalTokens,\r\n    finalLength,\r\n    finalTokens,\r\n    toonError,\r\n  };\r\n}\r\n"],"names":[],"mappings":"AAAA;;;;;;;AAOG;AAEH;AACA,IAAI,UAAU,GAAuC,IAAI;AACzD,IAAI,iBAAiB,GAAG,KAAK;AAC7B,IAAI,eAAe,GAA4B,IAAI;AAEnD;;;AAGG;AACH,eAAe,eAAe,GAAA;AAC5B,IAAA,IAAI,iBAAiB;QAAE,OAAO,UAAU,KAAK,IAAI;IACjD,iBAAiB,GAAG,IAAI;;AAGxB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,OAAO,mBAAmB,CAAC;AAC7C,QAAA,UAAU,GAAG,GAAG,CAAC,MAAM;AACvB,QAAA,OAAO,IAAI;;AACX,IAAA,MAAM;;AAEN,QAAA,IAAI;;AAEF,YAAA,MAAM,GAAG,GAAG,OAAO,CAAC,mBAAmB,CAAC;AACxC,YAAA,UAAU,GAAG,GAAG,CAAC,MAAM;AACvB,YAAA,OAAO,IAAI;;AACX,QAAA,MAAM;;AAEN,YAAA,OAAO,KAAK;;;AAGlB;AAEA;;AAEG;AACI,eAAe,gBAAgB,GAAA;IACpC,IAAI,CAAC,eAAe,EAAE;QACpB,eAAe,GAAG,eAAe,EAAE;;AAErC,IAAA,OAAO,eAAe;AACxB;AAEA;AACA,gBAAgB,EAAE;AAElB;;;;;;;;;;AAUG;AACG,SAAU,YAAY,CAAC,GAAW,EAAA;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,KAAK;AAE7D,IAAA,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE;;AAG1B,IAAA,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;AAAE,QAAA,OAAO,KAAK;;;;;AAMpE,IAAA,MAAM,YAAY,GAAG;AACnB,QAAA,SAAS;AACT,QAAA,eAAe;AACf,QAAA,iBAAiB;AACjB,QAAA,wBAAwB;KACzB;;AAGD,IAAA,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;IAG1E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACvD,MAAM,wBAAwB,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC;IAEpF,OAAO,cAAc,IAAI,wBAAwB;AACnD;AAEA;;;;;;AAMG;AACG,SAAU,gBAAgB,CAAC,GAAW,EAAA;IAM1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;AAC/C,QAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;;;IAIrE,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IAEjC,IAAI,QAAQ,KAAK,EAAE,IAAI,QAAQ,KAAK,EAAE,EAAE;AACtC,QAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;;AAGrE,IAAA,IAAI,UAAkB;AACtB,IAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;QACnB,UAAU,GAAG,QAAQ;;AAChB,SAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;QAC1B,UAAU,GAAG,QAAQ;;SAChB;QACL,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC;;;;IAK3C,IAAI,KAAK,GAAG,CAAC;IACb,IAAI,QAAQ,GAAG,KAAK;IACpB,IAAI,UAAU,GAAG,KAAK;AAEtB,IAAA,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC5C,QAAA,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QAEnB,IAAI,UAAU,EAAE;YACd,UAAU,GAAG,KAAK;YAClB;;AAGF,QAAA,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,EAAE;YAC7B,UAAU,GAAG,IAAI;YACjB;;AAGF,QAAA,IAAI,IAAI,KAAK,GAAG,EAAE;YAChB,QAAQ,GAAG,CAAC,QAAQ;YACpB;;AAGF,QAAA,IAAI,QAAQ;YAAE;AAEd,QAAA,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;AAAE,YAAA,KAAK,EAAE;AACzC,QAAA,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;AAAE,YAAA,KAAK,EAAE;AAEzC,QAAA,IAAI,KAAK,KAAK,CAAC,EAAE;;AAEf,YAAA,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;AAChD,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;AAClC,gBAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE;;AACvD,YAAA,MAAM;;;AAGN,gBAAA,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACzD,gBAAA,IAAI,UAAU,CAAC,KAAK,EAAE;oBACpB,OAAO;AACL,wBAAA,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,UAAU,CAAC,MAAM;AACzB,wBAAA,UAAU,EAAE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU;AACzC,wBAAA,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,QAAQ;qBACtC;;AAEH,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;;;;AAKzE,IAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;AACrE;AAEA;;;;;;;;;;AAUG;AACG,SAAU,UAAU,CAAC,GAAW,EAAA;IAOpC,IAAI,CAAC,UAAU,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;QAC1C,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,UAAU,GAAG,yBAAyB,GAAG,SAAS,EAAE;;;AAIpH,IAAA,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE;AACrB,QAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG3E,IAAA,IAAI;AACF,QAAA,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC;AAErE,QAAA,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE;AACrB,YAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,0BAA0B,EAAE;;;;AAK3F,QAAA,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE;AAC7B,YAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,iEAAiE,EAAE;;AAGlI,QAAA,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;;QAGrC,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC;QAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC;;AAG7C,QAAA,MAAM,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS;QAEtD,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC;YACnF,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE;;aACpD;;YAEL,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,CAA4B,yBAAA,EAAA,UAAU,CAAC,MAAM,CAAO,IAAA,EAAA,GAAG,CAAC,MAAM,CAAG,CAAA,CAAA,EAAE;;;IAElI,OAAO,GAAG,EAAE;;AAEZ,QAAA,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,GAAG,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC;AACjE,QAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,KAAK,EAAE,0BAA0B,QAAQ,CAAA,CAAE,EAAE;;AAEvG;AAEA;;AAEG;SACa,eAAe,GAAA;IAC7B,OAAO,UAAU,KAAK,IAAI;AAC5B;AA0CA;;;;;;;AAOG;AACH,SAAS,gBAAgB,CAAC,MAAe,EAAA;AACvC,IAAA,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;AAAE,QAAA,OAAO,KAAK;IAEvD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACtC,IAAA,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM;;IAGlC,IAAI,iBAAiB,GAAG,CAAC;IAEzB,SAAS,gBAAgB,CAAC,GAAY,EAAA;AACpC,QAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;;AAE3B,YAAA,iBAAiB,IAAI,GAAG,CAAC,MAAM;;AAC1B,aAAA,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;AAC7B,YAAA,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC;;AACxB,aAAA,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YACzC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;;;IAIhD,gBAAgB,CAAC,MAAM,CAAC;;AAGxB,IAAA,MAAM,SAAS,GAAG,iBAAiB,GAAG,WAAW;;;IAIjD,OAAO,SAAS,GAAG,GAAG;AACxB;AAEA;;;;;;;;;;;;AAYG;SACa,iBAAiB,CAC/B,OAAe,EACf,UAAoC,EAAE,EAAA;AAEtC,IAAA,MAAM,EACJ,SAAS,GAAG,MAAM,EAClB,UAAU,GAAG,IAAI,EACjB,cAAc,GAAG,IAAI,EACrB,mBAAmB,GAAG,EAAE;AACzB,MAAA,GAAG,OAAO;AAEX,IAAA,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM;IACrC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAEpD,IAAI,MAAM,GAAG,OAAO;IACpB,IAAI,aAAa,GAAG,KAAK;IACzB,IAAI,WAAW,GAAG,KAAK;IACvB,IAAI,SAAS,GAAG,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK;AACrB,IAAA,IAAI,SAA6B;;AAGjC,IAAA,IAAI,YAAY,CAAC,OAAO,CAAC,EAAE;QACzB,WAAW,GAAG,IAAI;;;SAGf,IAAI,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE;AACtD,QAAA,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;AACtC,QAAA,IAAI,UAAU,CAAC,WAAW,EAAE;YAC1B,WAAW,GAAG,IAAI;;aACb,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,IAAI,mBAAmB,EAAE;AAC9E,YAAA,MAAM,GAAG,UAAU,CAAC,MAAM;YAC1B,aAAa,GAAG,IAAI;AACpB,YAAA,SAAS,GAAG,UAAU,CAAC,SAAS;;AAC3B,aAAA,IAAI,UAAU,CAAC,KAAK,EAAE;;AAE3B,YAAA,SAAS,GAAG,UAAU,CAAC,KAAK;;aACvB,IAAI,UAAU,CAAC,SAAS,IAAI,UAAU,CAAC,SAAS,GAAG,mBAAmB,EAAE;;YAE7E,SAAS,GAAG,kBAAkB,UAAU,CAAC,SAAS,CAAqB,kBAAA,EAAA,mBAAmB,GAAG;;AACxF,aAAA,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;;AAEhC,YAAA,SAAS,GAAG,UAAU,CAAC,KAAK,IAAI,8CAA8C;;;;AAKlF,IAAA,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE;QAC7B,IAAI,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC;;AAGjD,QAAA,IAAI,aAAa,IAAI,WAAW,EAAE;;YAEhC,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;AAClD,YAAA,IAAI,WAAW,GAAG,SAAS,GAAG,GAAG,EAAE;gBACjC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,WAAW,CAAC;;;aAElD;;YAEL,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;YACvD,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;YACpD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,aAAa,CAAC;AAC5D,YAAA,IAAI,UAAU,GAAG,SAAS,GAAG,GAAG,EAAE;gBAChC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC;;;;QAK5D,MAAM,cAAc,GAAG;AACrB,cAAE,CAAA,SAAA,EAAY,cAAc,CAAC,cAAc,EAAE,CAAA,cAAA,EAAiB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA,QAAA,EAAW,SAAS,CAAY,UAAA,CAAA;AAC1H,gBAAA,CAAA,cAAA,EAAiB,SAAS,CAAC,cAAc,EAAE,CAAc,YAAA;AAC3D,cAAE,CAAA,oBAAA,EAAuB,cAAc,CAAC,cAAc,EAAE,CAAiB,cAAA,EAAA,cAAc,CAAC,cAAc,EAAE,CAAA,SAAA,CAAW;AAErH,QAAA,MAAM,GAAG,YAAY,GAAG,CAA0B,uBAAA,EAAA,cAAc,4DAA4D;QAC5H,SAAS,GAAG,IAAI;;AAGlB,IAAA,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM;IACjC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IAE9C,OAAO;AACL,QAAA,OAAO,EAAE,MAAM;QACf,aAAa;QACb,SAAS;QACT,SAAS;QACT,WAAW;QACX,cAAc;QACd,cAAc;QACd,WAAW;QACX,WAAW;QACX,SAAS;KACV;AACH;;;;"}

package/dist/types/common/enum.d.ts

@@ -17,6 +17,8 @@ export declare enum GraphEvents {
     ON_MESSAGE_DELTA = "on_message_delta",
     /** [Custom] Reasoning Delta events for messages */
     ON_REASONING_DELTA = "on_reasoning_delta",
+    /** [Custom] Context analytics event for traces */
+    ON_CONTEXT_ANALYTICS = "on_context_analytics",
     /** Custom event, emitted by system */
     ON_CUSTOM_EVENT = "on_custom_event",
     /** Emitted when a chat model starts processing. */
@@ -129,3 +131,18 @@ export declare enum EnvVar {
     CODE_API_KEY = "CODE_EXECUTOR_API_KEY",
     CODE_BASEURL = "CODE_EXECUTOR_BASEURL"
 }
+/**
+ * Message type identifiers used by LangChain's BaseMessage.getType().
+ * Use these constants instead of instanceof checks to avoid module mismatch issues
+ * when different copies of @langchain/core/messages are loaded.
+ */
+export declare enum MessageTypes {
+    HUMAN = "human",
+    AI = "ai",
+    SYSTEM = "system",
+    TOOL = "tool",
+    FUNCTION = "function",
+    GENERIC = "generic",
+    DEVELOPER = "developer",
+    REMOVE = "remove"
+}

package/dist/types/graphs/Graph.d.ts

@@ -4,6 +4,7 @@ import type { UsageMetadata, BaseMessage } from '@langchain/core/messages';
 import type { ToolCall } from '@langchain/core/messages/tool';
 import type * as t from '@/types';
 import { Providers } from '@/common';
+import { type ContextAnalytics } from '@/utils/contextAnalytics';
 import { ToolNode as CustomToolNode } from '@/tools/ToolNode';
 import { AgentContext } from '@/agents/AgentContext';
 import { HandlerRegistry } from '@/events';
@@ -113,6 +114,13 @@ export declare class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode>
             tokens: number;
         }>;
     } | null;
+    /**
+     * Get the latest context analytics from the graph.
+     * Returns metrics like utilization %, TOON stats, message breakdown.
+     */
+    getContextAnalytics(): ContextAnalytics | null;
+    /** Store the latest context analytics for retrieval after run */
+    private lastContextAnalytics;
     createSystemRunnable({ provider, clientOptions, instructions, additional_instructions, }: {
         provider?: Providers;
         clientOptions?: t.ClientOptions;

package/dist/types/utils/contextAnalytics.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Context Analytics Utility
+ *
+ * Provides context analytics data for observability/traces.
+ * No console logging - just data structures for event emission.
+ */
+import type { BaseMessage } from '@langchain/core/messages';
+import type { TokenCounter } from '@/types/run';
+/**
+ * Context analytics data for traces
+ */
+export interface ContextAnalytics {
+    /** Total messages in context */
+    messageCount: number;
+    /** Total tokens in context */
+    totalTokens: number;
+    /** Maximum allowed context tokens */
+    maxContextTokens?: number;
+    /** Instruction/system tokens */
+    instructionTokens?: number;
+    /** Context utilization percentage (0-100) */
+    utilizationPercent?: number;
+    /** Breakdown by message type */
+    breakdown?: Record<string, {
+        tokens: number;
+        percent: number;
+    }>;
+}
+/**
+ * Build context analytics for traces (no logging)
+ */
+export declare function buildContextAnalytics(messages: BaseMessage[], options: {
+    tokenCounter?: TokenCounter;
+    maxContextTokens?: number;
+    instructionTokens?: number;
+    indexTokenCountMap?: Record<string, number | undefined>;
+}): ContextAnalytics;

package/dist/types/utils/index.d.ts

@@ -4,3 +4,5 @@ export * from './misc';
 export * from './handlers';
 export * from './run';
 export * from './tokens';
+export * from './toonFormat';
+export * from './contextAnalytics';

package/dist/types/utils/toonFormat.d.ts

@@ -0,0 +1,111 @@
+/**
+ * TOON Format Utility
+ *
+ * Provides JSON to TOON conversion for token-efficient encoding.
+ * TOON format can reduce token count by 30-60% for JSON data.
+ *
+ * Uses the official @toon-format/toon library when available.
+ */
+/**
+ * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.
+ */
+export declare function ensureToonLoaded(): Promise<boolean>;
+/**
+ * Check if a string appears to be in TOON format.
+ * TOON format characteristics:
+ * - Uses key: value syntax (like YAML)
+ * - Uses array notation like [3] or {fields}
+ * - Does NOT start with { or [
+ * - Often has patterns like "name[N]:" or "name{fields}:"
+ *
+ * @param str - String to check
+ * @returns true if string appears to be TOON format
+ */
+export declare function isToonFormat(str: string): boolean;
+/**
+ * Extract the first valid JSON object or array from a string.
+ * Handles pure JSON or JSON embedded in text (e.g., tool responses).
+ *
+ * @param str - The string to extract JSON from
+ * @returns Object with found status, parsed JSON, and indices
+ */
+export declare function extractFirstJson(str: string): {
+    found: boolean;
+    parsed: unknown;
+    startIndex: number;
+    endIndex: number;
+};
+/**
+ * Convert JSON content to TOON format for token efficiency.
+ * Extracts JSON from string if embedded, converts to TOON.
+ * Returns original string if:
+ * - Already in TOON format
+ * - Not JSON
+ * - TOON conversion fails or is larger
+ *
+ * @param str - The string containing JSON to convert
+ * @returns Object with conversion status, result, and reduction percentage
+ */
+export declare function jsonToToon(str: string): {
+    converted: boolean;
+    result: string;
+    reduction: number;
+    alreadyToon?: boolean;
+    error?: string;
+};
+/**
+ * Check if TOON library is loaded and available
+ */
+export declare function isToonAvailable(): boolean;
+/**
+ * Result of processing tool output
+ */
+export interface ProcessToolOutputResult {
+    /** The processed content string */
+    content: string;
+    /** Whether TOON conversion was applied */
+    toonConverted: boolean;
+    /** Whether content was truncated */
+    truncated: boolean;
+    /** Percentage reduction from TOON (0 if not converted) */
+    reduction: number;
+    /** Whether input was already in TOON format */
+    alreadyToon: boolean;
+    /** Original content length */
+    originalLength: number;
+    /** Estimated original tokens (~4 chars per token) */
+    originalTokens: number;
+    /** Final content length */
+    finalLength: number;
+    /** Estimated final tokens */
+    finalTokens: number;
+    /** Error message if TOON conversion failed (for debugging) */
+    toonError?: string;
+}
+/**
+ * Options for processing tool output
+ */
+export interface ProcessToolOutputOptions {
+    /** Maximum output length in characters (default: 100000) */
+    maxLength?: number;
+    /** Whether to apply TOON conversion (default: true) */
+    enableToon?: boolean;
+    /** Minimum content size to attempt TOON conversion (default: 1000) */
+    minSizeForToon?: number;
+    /** Minimum reduction % to accept TOON result (default: 10) */
+    minReductionPercent?: number;
+}
+/**
+ * Process tool output: apply TOON conversion if beneficial, then truncate if needed.
+ * This is the main entry point for processing any tool output (regular or MCP).
+ *
+ * Flow:
+ * 1. Check if already TOON format → skip conversion
+ * 2. Try TOON conversion if content is JSON and large enough
+ * 3. Truncate if still exceeds maxLength (with smart break points)
+ *
+ * @param content - The tool output content
+ * @param options - Processing options
+ * @returns Processed content and metadata
+ */
+export declare function processToolOutput(content: string, options?: ProcessToolOutputOptions): ProcessToolOutputResult;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.15",
+  "version": "1.0.17",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",
@@ -111,6 +111,7 @@
     "@langfuse/otel": "^4.3.0",
     "@langfuse/tracing": "^4.3.0",
     "@opentelemetry/sdk-node": "^0.207.0",
+    "@toon-format/toon": "^2.1.0",
     "axios": "^1.12.1",
     "cheerio": "^1.0.0",
     "dotenv": "^16.4.7",

package/src/common/enum.ts

@@ -19,6 +19,8 @@ export enum GraphEvents {
   ON_MESSAGE_DELTA = 'on_message_delta',
   /** [Custom] Reasoning Delta events for messages */
   ON_REASONING_DELTA = 'on_reasoning_delta',
+  /** [Custom] Context analytics event for traces */
+  ON_CONTEXT_ANALYTICS = 'on_context_analytics',
 
   /* Official Events */
 
@@ -176,3 +178,19 @@ export enum EnvVar {
   CODE_API_KEY = 'CODE_EXECUTOR_API_KEY',
   CODE_BASEURL = 'CODE_EXECUTOR_BASEURL',
 }
+
+/**
+ * Message type identifiers used by LangChain's BaseMessage.getType().
+ * Use these constants instead of instanceof checks to avoid module mismatch issues
+ * when different copies of @langchain/core/messages are loaded.
+ */
+export enum MessageTypes {
+  HUMAN = 'human',
+  AI = 'ai',
+  SYSTEM = 'system',
+  TOOL = 'tool',
+  FUNCTION = 'function',
+  GENERIC = 'generic',
+  DEVELOPER = 'developer',
+  REMOVE = 'remove',
+}