@illuma-ai/agents 1.0.81

package/src/utils/tokens.ts

@@ -0,0 +1,142 @@
+import { getEncoding, type Tiktoken } from 'js-tiktoken';
+import type { BaseMessage } from '@langchain/core/messages';
+import { ContentTypes } from '@/common/enum';
+
+export function getTokenCountForMessage(
+  message: BaseMessage,
+  getTokenCount: (text: string) => number
+): number {
+  const tokensPerMessage = 3;
+
+  const processValue = (value: unknown): void => {
+    if (Array.isArray(value)) {
+      for (const item of value) {
+        if (
+          !item ||
+          !item.type ||
+          item.type === ContentTypes.ERROR ||
+          item.type === ContentTypes.IMAGE_URL
+        ) {
+          continue;
+        }
+
+        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {
+          const toolName = item.tool_call?.name || '';
+          if (toolName != null && toolName && typeof toolName === 'string') {
+            numTokens += getTokenCount(toolName);
+          }
+
+          const args = item.tool_call?.args || '';
+          if (args != null && args && typeof args === 'string') {
+            numTokens += getTokenCount(args);
+          }
+
+          const output = item.tool_call?.output || '';
+          if (output != null && output && typeof output === 'string') {
+            numTokens += getTokenCount(output);
+          }
+          continue;
+        }
+
+        const nestedValue = item[item.type];
+
+        if (!nestedValue) {
+          continue;
+        }
+
+        processValue(nestedValue);
+      }
+    } else if (typeof value === 'string') {
+      numTokens += getTokenCount(value);
+    } else if (typeof value === 'number') {
+      numTokens += getTokenCount(value.toString());
+    } else if (typeof value === 'boolean') {
+      numTokens += getTokenCount(value.toString());
+    }
+  };
+
+  let numTokens = tokensPerMessage;
+  processValue(message.content);
+  return numTokens;
+}
+
+let encoderPromise: Promise<Tiktoken | null> | undefined;
+let tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;
+
+/**
+ * Simple character-based token estimation (~4 chars per token).
+ * Used as fallback when tiktoken encoder fails to load.
+ */
+const estimateTokens = (text: string): number => Math.ceil(text.length / 4);
+
+async function getSharedEncoder(): Promise<Tiktoken | null> {
+  if (encoderPromise) {
+    return encoderPromise;
+  }
+  // Use bundled tokenizer data - no external network calls
+  encoderPromise = Promise.resolve().then(() => {
+    try {
+      return getEncoding('o200k_base');
+    } catch {
+      // Fallback to null - will use character estimation
+      return null;
+    }
+  });
+  return encoderPromise;
+}
+
+/**
+ * Creates a singleton token counter function that reuses the same encoder instance.
+ * Falls back to character-based estimation (~4 chars per token) if encoder fails.
+ * This ensures token counting never blocks LLM calls.
+ */
+export const createTokenCounter = async (): Promise<
+  (message: BaseMessage) => number
+> => {
+  if (tokenCounterPromise) {
+    return tokenCounterPromise;
+  }
+
+  tokenCounterPromise = (async (): Promise<
+    (message: BaseMessage) => number
+  > => {
+    const enc = await getSharedEncoder();
+    const countTokens = enc
+      ? (text: string): number => enc.encode(text).length
+      : estimateTokens;
+    return (message: BaseMessage): number =>
+      getTokenCountForMessage(message, countTokens);
+  })();
+
+  return tokenCounterPromise;
+};
+
+/**
+ * Utility to manage the token encoder lifecycle explicitly.
+ * Useful for applications that need fine-grained control over resource management.
+ */
+export const TokenEncoderManager = {
+  /**
+   * Pre-initializes the encoder. This can be called during app startup
+   * to avoid lazy loading delays later.
+   */
+  async initialize(): Promise<void> {
+    await getSharedEncoder();
+  },
+
+  /**
+   * Clears the cached encoder and token counter.
+   * Useful for testing or when you need to force a fresh reload.
+   */
+  reset(): void {
+    encoderPromise = undefined;
+    tokenCounterPromise = undefined;
+  },
+
+  /**
+   * Checks if the encoder has been initialized.
+   */
+  isInitialized(): boolean {
+    return encoderPromise !== undefined;
+  },
+};

package/src/utils/toonFormat.ts

@@ -0,0 +1,475 @@
+/**
+ * TOON Format Utility
+ *
+ * Provides JSON to TOON conversion for token-efficient encoding.
+ * TOON format can reduce token count by 30-60% for JSON data.
+ *
+ * Uses the official @toon-format/toon library when available.
+ */
+
+// Dynamic import holder for the TOON library (ESM)
+let toonEncode: ((data: unknown) => string) | null = null;
+let toonLoadAttempted = false;
+let toonLoadPromise: Promise<boolean> | null = null;
+
+/**
+ * Lazily loads the TOON library
+ * Tries both ESM dynamic import and CommonJS require for compatibility
+ */
+async function loadToonLibrary(): Promise<boolean> {
+  if (toonLoadAttempted) return toonEncode !== null;
+  toonLoadAttempted = true;
+
+  // Try ESM dynamic import first
+  try {
+    const mod = await import('@toon-format/toon');
+    toonEncode = mod.encode;
+    return true;
+  } catch {
+    // ESM import failed, try CommonJS require
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      const mod = require('@toon-format/toon');
+      toonEncode = mod.encode;
+      return true;
+    } catch {
+      // Library not available - that's OK, we'll return original content
+      return false;
+    }
+  }
+}
+
+/**
+ * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.
+ */
+export async function ensureToonLoaded(): Promise<boolean> {
+  if (!toonLoadPromise) {
+    toonLoadPromise = loadToonLibrary();
+  }
+  return toonLoadPromise;
+}
+
+// Start loading immediately
+ensureToonLoaded();
+
+/**
+ * Check if a string appears to be in TOON format.
+ * TOON format characteristics:
+ * - Uses key: value syntax (like YAML)
+ * - Uses array notation like [3] or {fields}
+ * - Does NOT start with { or [
+ * - Often has patterns like "name[N]:" or "name{fields}:"
+ *
+ * @param str - String to check
+ * @returns true if string appears to be TOON format
+ */
+export function isToonFormat(str: string): boolean {
+  if (typeof str !== 'string' || str.length === 0) return false;
+
+  const trimmed = str.trim();
+
+  // TOON doesn't start with JSON brackets
+  if (trimmed.startsWith('{') || trimmed.startsWith('[')) return false;
+
+  // Check for TOON-specific patterns:
+  // 1. Key-value with colon (but not URL-like patterns)
+  // 2. Array notation like "items[3]:" or "data[10]:"
+  // 3. Object schema notation like "items{id,name}:"
+  const toonPatterns = [
+    /^\w+:$/m, // Simple key: at start of line
+    /^\w+\[\d+\]:/m, // Array notation: items[3]:
+    /^\w+\{[^}]+\}:/m, // Schema notation: items{id,name}:
+    /^\w+\[\d+\]\{[^}]+\}:/m, // Combined: items[3]{id,name}:
+  ];
+
+  // Must match at least one TOON pattern
+  const hasToonPattern = toonPatterns.some((pattern) => pattern.test(trimmed));
+
+  // Additional check: TOON typically has multiple lines with consistent indentation
+  const lines = trimmed.split('\n').filter((l) => l.trim());
+  const hasMultipleKeyValueLines =
+    lines.filter((l) => /^\s*\w+.*:/.test(l)).length >= 2;
+
+  return hasToonPattern || hasMultipleKeyValueLines;
+}
+
+/**
+ * Extract the first valid JSON object or array from a string.
+ * Handles pure JSON or JSON embedded in text (e.g., tool responses).
+ *
+ * @param str - The string to extract JSON from
+ * @returns Object with found status, parsed JSON, and indices
+ */
+export function extractFirstJson(str: string): {
+  found: boolean;
+  parsed: unknown;
+  startIndex: number;
+  endIndex: number;
+} {
+  if (typeof str !== 'string' || str.length === 0) {
+    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+  }
+
+  // Find the first { or [ character
+  const objStart = str.indexOf('{');
+  const arrStart = str.indexOf('[');
+
+  if (objStart === -1 && arrStart === -1) {
+    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+  }
+
+  let startIndex: number;
+  if (objStart === -1) {
+    startIndex = arrStart;
+  } else if (arrStart === -1) {
+    startIndex = objStart;
+  } else {
+    startIndex = Math.min(objStart, arrStart);
+  }
+
+  // Find the matching closing bracket using a stack-based approach
+  // Properly handle strings to avoid counting brackets inside strings
+  let depth = 0;
+  let inString = false;
+  let escapeNext = false;
+
+  for (let i = startIndex; i < str.length; i++) {
+    const char = str[i];
+
+    if (escapeNext) {
+      escapeNext = false;
+      continue;
+    }
+
+    if (char === '\\' && inString) {
+      escapeNext = true;
+      continue;
+    }
+
+    if (char === '"') {
+      inString = !inString;
+      continue;
+    }
+
+    if (inString) continue;
+
+    if (char === '{' || char === '[') depth++;
+    if (char === '}' || char === ']') depth--;
+
+    if (depth === 0) {
+      // Found matching bracket, try to parse
+      const jsonStr = str.substring(startIndex, i + 1);
+      try {
+        const parsed = JSON.parse(jsonStr);
+        return { found: true, parsed, startIndex, endIndex: i };
+      } catch {
+        // Not valid JSON at this bracket level, continue searching
+        // Reset and look for the next JSON start after this position
+        const nextResult = extractFirstJson(str.substring(i + 1));
+        if (nextResult.found) {
+          return {
+            found: true,
+            parsed: nextResult.parsed,
+            startIndex: i + 1 + nextResult.startIndex,
+            endIndex: i + 1 + nextResult.endIndex,
+          };
+        }
+        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+      }
+    }
+  }
+
+  return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
+}
+
+/**
+ * Convert JSON content to TOON format for token efficiency.
+ * Extracts JSON from string if embedded, converts to TOON.
+ * Returns original string if:
+ * - Already in TOON format
+ * - Not JSON
+ * - TOON conversion fails or is larger
+ *
+ * @param str - The string containing JSON to convert
+ * @returns Object with conversion status, result, and reduction percentage
+ */
+export function jsonToToon(str: string): {
+  converted: boolean;
+  result: string;
+  reduction: number;
+  alreadyToon?: boolean;
+  error?: string;
+} {
+  if (!toonEncode || typeof str !== 'string') {
+    return {
+      converted: false,
+      result: str,
+      reduction: 0,
+      error: !toonEncode ? 'TOON library not loaded' : undefined,
+    };
+  }
+
+  // Check if already in TOON format - skip conversion
+  if (isToonFormat(str)) {
+    return { converted: false, result: str, reduction: 0, alreadyToon: true };
+  }
+
+  try {
+    const { found, parsed, startIndex, endIndex } = extractFirstJson(str);
+
+    if (!found || !parsed) {
+      return {
+        converted: false,
+        result: str,
+        reduction: 0,
+        error: 'No JSON found in content',
+      };
+    }
+
+    // Check if this JSON structure would benefit from TOON
+    // Text-heavy content (emails, documents) won't compress well
+    if (!isToonBeneficial(parsed)) {
+      return {
+        converted: false,
+        result: str,
+        reduction: 0,
+        error:
+          'Content is text-heavy (>70% string values), TOON not beneficial',
+      };
+    }
+
+    const toonResult = toonEncode(parsed);
+
+    // Preserve text before and after the JSON block
+    const textBefore = str.substring(0, startIndex);
+    const textAfter = str.substring(endIndex + 1);
+
+    // Build the full result: textBefore + TOON + textAfter
+    const fullResult = textBefore + toonResult + textAfter;
+
+    if (fullResult.length < str.length) {
+      const reduction = Math.round(
+        ((str.length - fullResult.length) / str.length) * 100
+      );
+      return { converted: true, result: fullResult, reduction };
+    } else {
+      // TOON output was larger or same size - not beneficial
+      return {
+        converted: false,
+        result: str,
+        reduction: 0,
+        error: `TOON output not smaller (${fullResult.length} >= ${str.length})`,
+      };
+    }
+  } catch (err) {
+    // TOON encoding or extraction failed - log error for debugging
+    const errorMsg = err instanceof Error ? err.message : String(err);
+    return {
+      converted: false,
+      result: str,
+      reduction: 0,
+      error: `TOON conversion error: ${errorMsg}`,
+    };
+  }
+}
+
+/**
+ * Check if TOON library is loaded and available
+ */
+export function isToonAvailable(): boolean {
+  return toonEncode !== null;
+}
+
+/**
+ * Result of processing tool output
+ */
+export interface ProcessToolOutputResult {
+  /** The processed content string */
+  content: string;
+  /** Whether TOON conversion was applied */
+  toonConverted: boolean;
+  /** Whether content was truncated */
+  truncated: boolean;
+  /** Percentage reduction from TOON (0 if not converted) */
+  reduction: number;
+  /** Whether input was already in TOON format */
+  alreadyToon: boolean;
+  /** Original content length */
+  originalLength: number;
+  /** Estimated original tokens (~4 chars per token) */
+  originalTokens: number;
+  /** Final content length */
+  finalLength: number;
+  /** Estimated final tokens */
+  finalTokens: number;
+  /** Error message if TOON conversion failed (for debugging) */
+  toonError?: string;
+}
+
+/**
+ * Options for processing tool output
+ */
+export interface ProcessToolOutputOptions {
+  /** Maximum output length in characters (default: 100000) */
+  maxLength?: number;
+  /** Whether to apply TOON conversion (default: true) */
+  enableToon?: boolean;
+  /** Minimum content size to attempt TOON conversion (default: 1000) */
+  minSizeForToon?: number;
+  /** Minimum reduction % to accept TOON result (default: 10) */
+  minReductionPercent?: number;
+}
+
+/**
+ * Analyze JSON structure to determine if TOON conversion would be beneficial.
+ * Text-heavy content (emails, documents) doesn't compress well with TOON.
+ * Structured data (API responses, metadata) compresses much better.
+ *
+ * @param parsed - Parsed JSON object
+ * @returns true if TOON conversion is likely beneficial
+ */
+function isToonBeneficial(parsed: unknown): boolean {
+  if (!parsed || typeof parsed !== 'object') return false;
+
+  const jsonStr = JSON.stringify(parsed);
+  const totalLength = jsonStr.length;
+
+  // Count characters in string values (text content)
+  let textContentLength = 0;
+
+  function countTextContent(obj: unknown): void {
+    if (typeof obj === 'string') {
+      // Count string length (this is text content that TOON can't compress)
+      textContentLength += obj.length;
+    } else if (Array.isArray(obj)) {
+      obj.forEach(countTextContent);
+    } else if (obj && typeof obj === 'object') {
+      Object.values(obj).forEach(countTextContent);
+    }
+  }
+
+  countTextContent(parsed);
+
+  // Calculate ratio of text content to total JSON
+  const textRatio = textContentLength / totalLength;
+
+  // If more than 70% is text content, TOON won't help much
+  // TOON compresses structure (field names, brackets), not text
+  return textRatio < 0.7;
+}
+
+/**
+ * Process tool output: apply TOON conversion if beneficial, then truncate if needed.
+ * This is the main entry point for processing any tool output (regular or MCP).
+ *
+ * Flow:
+ * 1. Check if already TOON format → skip conversion
+ * 2. Try TOON conversion if content is JSON and large enough
+ * 3. Truncate if still exceeds maxLength (with smart break points)
+ *
+ * @param content - The tool output content
+ * @param options - Processing options
+ * @returns Processed content and metadata
+ */
+export function processToolOutput(
+  content: string,
+  options: ProcessToolOutputOptions = {}
+): ProcessToolOutputResult {
+  const {
+    maxLength = 100000,
+    enableToon = true,
+    minSizeForToon = 1000,
+    minReductionPercent = 10, // Increased from 5% - only apply TOON when clearly beneficial
+  } = options;
+
+  const originalLength = content.length;
+  const originalTokens = Math.ceil(originalLength / 4);
+
+  let result = content;
+  let toonConverted = false;
+  let alreadyToon = false;
+  let reduction = 0;
+  let truncated = false;
+  let toonError: string | undefined;
+
+  // Step 1: Check if already TOON format
+  if (isToonFormat(content)) {
+    alreadyToon = true;
+  }
+  // Step 2: Apply TOON conversion if enabled and content is large enough
+  else if (enableToon && content.length > minSizeForToon) {
+    const toonResult = jsonToToon(content);
+    if (toonResult.alreadyToon) {
+      alreadyToon = true;
+    } else if (
+      toonResult.converted &&
+      toonResult.reduction >= minReductionPercent
+    ) {
+      result = toonResult.result;
+      toonConverted = true;
+      reduction = toonResult.reduction;
+    } else if (toonResult.error) {
+      // Track error for debugging
+      toonError = toonResult.error;
+    } else if (
+      toonResult.converted &&
+      toonResult.reduction < minReductionPercent
+    ) {
+      // TOON converted but reduction was too small to be worth it
+      toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;
+    } else if (!toonResult.converted) {
+      // Conversion failed without explicit error - investigate
+      toonError =
+        toonResult.error || 'TOON conversion returned false with no error';
+    }
+  }
+
+  // Step 3: Truncate if still too long (with smart break points)
+  if (result.length > maxLength) {
+    let truncatedStr = result.substring(0, maxLength);
+
+    // Try to find a clean break point
+    if (toonConverted || alreadyToon) {
+      // For TOON format, break at newline for cleaner output
+      const lastNewline = truncatedStr.lastIndexOf('\n');
+      if (lastNewline > maxLength * 0.7) {
+        truncatedStr = truncatedStr.substring(0, lastNewline);
+      }
+    } else {
+      // For JSON, try to find a clean JSON break point
+      const lastCompleteItem = truncatedStr.lastIndexOf('},');
+      const lastArrayItem = truncatedStr.lastIndexOf('],');
+      const breakPoint = Math.max(lastCompleteItem, lastArrayItem);
+      if (breakPoint > maxLength * 0.5) {
+        truncatedStr = truncatedStr.substring(0, breakPoint + 1);
+      }
+    }
+
+    // Build truncation message
+    const truncationInfo = toonConverted
+      ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +
+        `Still exceeds ${maxLength.toLocaleString()} char limit.`
+      : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;
+
+    result =
+      truncatedStr +
+      `\n\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;
+    truncated = true;
+  }
+
+  const finalLength = result.length;
+  const finalTokens = Math.ceil(finalLength / 4);
+
+  return {
+    content: result,
+    toonConverted,
+    truncated,
+    reduction,
+    alreadyToon,
+    originalLength,
+    originalTokens,
+    finalLength,
+    finalTokens,
+    toonError,
+  };
+}