illuma-agents 1.0.15 → 1.0.17

package/src/utils/index.ts

@@ -4,3 +4,5 @@ export * from './misc';
 export * from './handlers';
 export * from './run';
 export * from './tokens';
+export * from './toonFormat';
+export * from './contextAnalytics';

package/src/utils/toonFormat.ts

@@ -0,0 +1,437 @@
+/**
+ * TOON Format Utility
+ * 
+ * Provides JSON to TOON conversion for token-efficient encoding.
+ * TOON format can reduce token count by 30-60% for JSON data.
+ * 
+ * Uses the official @toon-format/toon library when available.
+ */
+
+// Dynamic import holder for the TOON library (ESM)
+let toonEncode: ((data: unknown) => string) | null = null;
+let toonLoadAttempted = false;
+let toonLoadPromise: Promise<boolean> | null = null;
+
+/**
+ * Lazily loads the TOON library
+ * Tries both ESM dynamic import and CommonJS require for compatibility
+ */
+async function loadToonLibrary(): Promise<boolean> {
+  if (toonLoadAttempted) return toonEncode !== null;
+  toonLoadAttempted = true;
+
+  // Try ESM dynamic import first
+  try {
+    const mod = await import('@toon-format/toon');
+    toonEncode = mod.encode;
+    return true;
+  } catch {
+    // ESM import failed, try CommonJS require
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const mod = require('@toon-format/toon');
+      toonEncode = mod.encode;
+      return true;
+    } catch {
+      // Library not available - that's OK, we'll return original content
+      return false;
+    }
+  }
+}
+
+/**
+ * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.
+ */
+export async function ensureToonLoaded(): Promise<boolean> {
+  if (!toonLoadPromise) {
+    toonLoadPromise = loadToonLibrary();
+  }
+  return toonLoadPromise;
+}
+
+// Start loading immediately
+ensureToonLoaded();
+
+/**
+ * Check if a string appears to be in TOON format.
+ * TOON format characteristics:
+ * - Uses key: value syntax (like YAML)
+ * - Uses array notation like [3] or {fields}
+ * - Does NOT start with { or [
+ * - Often has patterns like "name[N]:" or "name{fields}:"
+ * 
+ * @param str - String to check
+ * @returns true if string appears to be TOON format
+ */
+export function isToonFormat(str: string): boolean {
+  if (typeof str !== 'string' || str.length === 0) return false;
+  
+  const trimmed = str.trim();
+  
+  // TOON doesn't start with JSON brackets
+  if (trimmed.startsWith('{') || trimmed.startsWith('[')) return false;
+  
+  // Check for TOON-specific patterns:
+  // 1. Key-value with colon (but not URL-like patterns)
+  // 2. Array notation like "items[3]:" or "data[10]:"
+  // 3. Object schema notation like "items{id,name}:"
+  const toonPatterns = [
+    /^\w+:$/m,                      // Simple key: at start of line
+    /^\w+\[\d+\]:/m,                // Array notation: items[3]:
+    /^\w+\{[^}]+\}:/m,              // Schema notation: items{id,name}:
+    /^\w+\[\d+\]\{[^}]+\}:/m,       // Combined: items[3]{id,name}:
+  ];
+  
+  // Must match at least one TOON pattern
+  const hasToonPattern = toonPatterns.some(pattern => pattern.test(trimmed));
+  
+  // Additional check: TOON typically has multiple lines with consistent indentation
+  const lines = trimmed.split('\n').filter(l => l.trim());
+  const hasMultipleKeyValueLines = lines.filter(l => /^\s*\w+.*:/.test(l)).length >= 2;
+  
+  return hasToonPattern || hasMultipleKeyValueLines;
+}
+
+/**
+ * Extract the first valid JSON object or array from a string.
+ * Handles pure JSON or JSON embedded in text (e.g., tool responses).
+ * 
+ * @param str - The string to extract JSON from
+ * @returns Object with found status, parsed JSON, and indices
+ */
+export function extractFirstJson(str: string): {
+  found: boolean;
+  parsed: unknown;
+  startIndex: number;
+  endIndex: number;
+} {
+  if (typeof str !== 'string' || str.length === 0) {
+    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+  }
+
+  // Find the first { or [ character
+  const objStart = str.indexOf('{');
+  const arrStart = str.indexOf('[');
+
+  if (objStart === -1 && arrStart === -1) {
+    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+  }
+
+  let startIndex: number;
+  if (objStart === -1) {
+    startIndex = arrStart;
+  } else if (arrStart === -1) {
+    startIndex = objStart;
+  } else {
+    startIndex = Math.min(objStart, arrStart);
+  }
+
+  // Find the matching closing bracket using a stack-based approach
+  // Properly handle strings to avoid counting brackets inside strings
+  let depth = 0;
+  let inString = false;
+  let escapeNext = false;
+
+  for (let i = startIndex; i < str.length; i++) {
+    const char = str[i];
+
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+
+    if (char === '\\' && inString) {
+      escapeNext = true;
+      continue;
+    }
+
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+
+    if (inString) continue;
+
+    if (char === '{' || char === '[') depth++;
+    if (char === '}' || char === ']') depth--;
+
+    if (depth === 0) {
+      // Found matching bracket, try to parse
+      const jsonStr = str.substring(startIndex, i + 1);
+      try {
+        const parsed = JSON.parse(jsonStr);
+        return { found: true, parsed, startIndex, endIndex: i };
+      } catch {
+        // Not valid JSON at this bracket level, continue searching
+        // Reset and look for the next JSON start after this position
+        const nextResult = extractFirstJson(str.substring(i + 1));
+        if (nextResult.found) {
+          return {
+            found: true,
+            parsed: nextResult.parsed,
+            startIndex: i + 1 + nextResult.startIndex,
+            endIndex: i + 1 + nextResult.endIndex,
+          };
+        }
+        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+      }
+    }
+  }
+
+  return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+}
+
+/**
+ * Convert JSON content to TOON format for token efficiency.
+ * Extracts JSON from string if embedded, converts to TOON.
+ * Returns original string if:
+ * - Already in TOON format
+ * - Not JSON
+ * - TOON conversion fails or is larger
+ * 
+ * @param str - The string containing JSON to convert
+ * @returns Object with conversion status, result, and reduction percentage
+ */
+export function jsonToToon(str: string): {
+  converted: boolean;
+  result: string;
+  reduction: number;
+  alreadyToon?: boolean;
+  error?: string;
+} {
+  if (!toonEncode || typeof str !== 'string') {
+    return { converted: false, result: str, reduction: 0, error: !toonEncode ? 'TOON library not loaded' : undefined };
+  }
+
+  // Check if already in TOON format - skip conversion
+  if (isToonFormat(str)) {
+    return { converted: false, result: str, reduction: 0, alreadyToon: true };
+  }
+
+  try {
+    const { found, parsed, startIndex, endIndex } = extractFirstJson(str);
+
+    if (!found || !parsed) {
+      return { converted: false, result: str, reduction: 0, error: 'No JSON found in content' };
+    }
+
+    // Check if this JSON structure would benefit from TOON
+    // Text-heavy content (emails, documents) won't compress well
+    if (!isToonBeneficial(parsed)) {
+      return { converted: false, result: str, reduction: 0, error: 'Content is text-heavy (>70% string values), TOON not beneficial' };
+    }
+
+    const toonResult = toonEncode(parsed);
+    
+    // Preserve text before and after the JSON block
+    const textBefore = str.substring(0, startIndex);
+    const textAfter = str.substring(endIndex + 1);
+    
+    // Build the full result: textBefore + TOON + textAfter
+    const fullResult = textBefore + toonResult + textAfter;
+
+    if (fullResult.length < str.length) {
+      const reduction = Math.round(((str.length - fullResult.length) / str.length) * 100);
+      return { converted: true, result: fullResult, reduction };
+    } else {
+      // TOON output was larger or same size - not beneficial
+      return { converted: false, result: str, reduction: 0, error: `TOON output not smaller (${fullResult.length} >= ${str.length})` };
+    }
+  } catch (err) {
+    // TOON encoding or extraction failed - log error for debugging
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    return { converted: false, result: str, reduction: 0, error: `TOON conversion error: ${errorMsg}` };
+  }
+}
+
+/**
+ * Check if TOON library is loaded and available
+ */
+export function isToonAvailable(): boolean {
+  return toonEncode !== null;
+}
+
+/**
+ * Result of processing tool output
+ */
+export interface ProcessToolOutputResult {
+  /** The processed content string */
+  content: string;
+  /** Whether TOON conversion was applied */
+  toonConverted: boolean;
+  /** Whether content was truncated */
+  truncated: boolean;
+  /** Percentage reduction from TOON (0 if not converted) */
+  reduction: number;
+  /** Whether input was already in TOON format */
+  alreadyToon: boolean;
+  /** Original content length */
+  originalLength: number;
+  /** Estimated original tokens (~4 chars per token) */
+  originalTokens: number;
+  /** Final content length */
+  finalLength: number;
+  /** Estimated final tokens */
+  finalTokens: number;
+  /** Error message if TOON conversion failed (for debugging) */
+  toonError?: string;
+}
+
+/**
+ * Options for processing tool output
+ */
+export interface ProcessToolOutputOptions {
+  /** Maximum output length in characters (default: 100000) */
+  maxLength?: number;
+  /** Whether to apply TOON conversion (default: true) */
+  enableToon?: boolean;
+  /** Minimum content size to attempt TOON conversion (default: 1000) */
+  minSizeForToon?: number;
+  /** Minimum reduction % to accept TOON result (default: 10) */
+  minReductionPercent?: number;
+}
+
+/**
+ * Analyze JSON structure to determine if TOON conversion would be beneficial.
+ * Text-heavy content (emails, documents) doesn't compress well with TOON.
+ * Structured data (API responses, metadata) compresses much better.
+ * 
+ * @param parsed - Parsed JSON object
+ * @returns true if TOON conversion is likely beneficial
+ */
+function isToonBeneficial(parsed: unknown): boolean {
+  if (!parsed || typeof parsed !== 'object') return false;
+  
+  const jsonStr = JSON.stringify(parsed);
+  const totalLength = jsonStr.length;
+  
+  // Count characters in string values (text content)
+  let textContentLength = 0;
+  
+  function countTextContent(obj: unknown): void {
+    if (typeof obj === 'string') {
+      // Count string length (this is text content that TOON can't compress)
+      textContentLength += obj.length;
+    } else if (Array.isArray(obj)) {
+      obj.forEach(countTextContent);
+    } else if (obj && typeof obj === 'object') {
+      Object.values(obj).forEach(countTextContent);
+    }
+  }
+  
+  countTextContent(parsed);
+  
+  // Calculate ratio of text content to total JSON
+  const textRatio = textContentLength / totalLength;
+  
+  // If more than 70% is text content, TOON won't help much
+  // TOON compresses structure (field names, brackets), not text
+  return textRatio < 0.7;
+}
+
+/**
+ * Process tool output: apply TOON conversion if beneficial, then truncate if needed.
+ * This is the main entry point for processing any tool output (regular or MCP).
+ * 
+ * Flow:
+ * 1. Check if already TOON format → skip conversion
+ * 2. Try TOON conversion if content is JSON and large enough
+ * 3. Truncate if still exceeds maxLength (with smart break points)
+ * 
+ * @param content - The tool output content
+ * @param options - Processing options
+ * @returns Processed content and metadata
+ */
+export function processToolOutput(
+  content: string,
+  options: ProcessToolOutputOptions = {}
+): ProcessToolOutputResult {
+  const { 
+    maxLength = 100000, 
+    enableToon = true,
+    minSizeForToon = 1000,
+    minReductionPercent = 10,  // Increased from 5% - only apply TOON when clearly beneficial
+  } = options;
+
+  const originalLength = content.length;
+  const originalTokens = Math.ceil(originalLength / 4);
+  
+  let result = content;
+  let toonConverted = false;
+  let alreadyToon = false;
+  let reduction = 0;
+  let truncated = false;
+  let toonError: string | undefined;
+
+  // Step 1: Check if already TOON format
+  if (isToonFormat(content)) {
+    alreadyToon = true;
+  }
+  // Step 2: Apply TOON conversion if enabled and content is large enough
+  else if (enableToon && content.length > minSizeForToon) {
+    const toonResult = jsonToToon(content);
+    if (toonResult.alreadyToon) {
+      alreadyToon = true;
+    } else if (toonResult.converted && toonResult.reduction >= minReductionPercent) {
+      result = toonResult.result;
+      toonConverted = true;
+      reduction = toonResult.reduction;
+    } else if (toonResult.error) {
+      // Track error for debugging
+      toonError = toonResult.error;
+    } else if (toonResult.converted && toonResult.reduction < minReductionPercent) {
+      // TOON converted but reduction was too small to be worth it
+      toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;
+    } else if (!toonResult.converted) {
+      // Conversion failed without explicit error - investigate
+      toonError = toonResult.error || 'TOON conversion returned false with no error';
+    }
+  }
+
+  // Step 3: Truncate if still too long (with smart break points)
+  if (result.length > maxLength) {
+    let truncatedStr = result.substring(0, maxLength);
+    
+    // Try to find a clean break point
+    if (toonConverted || alreadyToon) {
+      // For TOON format, break at newline for cleaner output
+      const lastNewline = truncatedStr.lastIndexOf('\n');
+      if (lastNewline > maxLength * 0.7) {
+        truncatedStr = truncatedStr.substring(0, lastNewline);
+      }
+    } else {
+      // For JSON, try to find a clean JSON break point
+      const lastCompleteItem = truncatedStr.lastIndexOf('},');
+      const lastArrayItem = truncatedStr.lastIndexOf('],');
+      const breakPoint = Math.max(lastCompleteItem, lastArrayItem);
+      if (breakPoint > maxLength * 0.5) {
+        truncatedStr = truncatedStr.substring(0, breakPoint + 1);
+      }
+    }
+    
+    // Build truncation message
+    const truncationInfo = toonConverted
+      ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +
+        `Still exceeds ${maxLength.toLocaleString()} char limit.`
+      : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;
+    
+    result = truncatedStr + `\n\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;
+    truncated = true;
+  }
+
+  const finalLength = result.length;
+  const finalTokens = Math.ceil(finalLength / 4);
+
+  return { 
+    content: result, 
+    toonConverted, 
+    truncated, 
+    reduction, 
+    alreadyToon,
+    originalLength,
+    originalTokens,
+    finalLength,
+    finalTokens,
+    toonError,
+  };
+}