@illuma-ai/agents 1.1.21 → 1.1.22

package/src/utils/toonFormat.js

@@ -1,379 +0,0 @@
-/**
- * TOON Format Utility
- *
- * Provides JSON to TOON conversion for token-efficient encoding.
- * TOON format can reduce token count by 30-60% for JSON data.
- *
- * Uses the official @toon-format/toon library when available.
- */
-// Dynamic import holder for the TOON library (ESM)
-let toonEncode = null;
-let toonLoadAttempted = false;
-let toonLoadPromise = null;
-/**
- * Lazily loads the TOON library
- * Tries both ESM dynamic import and CommonJS require for compatibility
- */
-async function loadToonLibrary() {
-    if (toonLoadAttempted)
-        return toonEncode !== null;
-    toonLoadAttempted = true;
-    // Try ESM dynamic import first
-    try {
-        const mod = await import('@toon-format/toon');
-        toonEncode = mod.encode;
-        return true;
-    }
-    catch {
-        // ESM import failed, try CommonJS require
-        try {
-            // eslint-disable-next-line @typescript-eslint/no-require-imports
-            const mod = require('@toon-format/toon');
-            toonEncode = mod.encode;
-            return true;
-        }
-        catch {
-            // Library not available - that's OK, we'll return original content
-            return false;
-        }
-    }
-}
-/**
- * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.
- */
-export async function ensureToonLoaded() {
-    if (!toonLoadPromise) {
-        toonLoadPromise = loadToonLibrary();
-    }
-    return toonLoadPromise;
-}
-// Start loading immediately
-ensureToonLoaded();
-/**
- * Check if a string appears to be in TOON format.
- * TOON format characteristics:
- * - Uses key: value syntax (like YAML)
- * - Uses array notation like [3] or {fields}
- * - Does NOT start with { or [
- * - Often has patterns like "name[N]:" or "name{fields}:"
- *
- * @param str - String to check
- * @returns true if string appears to be TOON format
- */
-export function isToonFormat(str) {
-    if (typeof str !== 'string' || str.length === 0)
-        return false;
-    const trimmed = str.trim();
-    // TOON doesn't start with JSON brackets
-    if (trimmed.startsWith('{') || trimmed.startsWith('['))
-        return false;
-    // Check for TOON-specific patterns:
-    // 1. Key-value with colon (but not URL-like patterns)
-    // 2. Array notation like "items[3]:" or "data[10]:"
-    // 3. Object schema notation like "items{id,name}:"
-    const toonPatterns = [
-        /^\w+:$/m, // Simple key: at start of line
-        /^\w+\[\d+\]:/m, // Array notation: items[3]:
-        /^\w+\{[^}]+\}:/m, // Schema notation: items{id,name}:
-        /^\w+\[\d+\]\{[^}]+\}:/m, // Combined: items[3]{id,name}:
-    ];
-    // Must match at least one TOON pattern
-    const hasToonPattern = toonPatterns.some((pattern) => pattern.test(trimmed));
-    // Additional check: TOON typically has multiple lines with consistent indentation
-    const lines = trimmed.split('\n').filter((l) => l.trim());
-    const hasMultipleKeyValueLines = lines.filter((l) => /^\s*\w+.*:/.test(l)).length >= 2;
-    return hasToonPattern || hasMultipleKeyValueLines;
-}
-/**
- * Extract the first valid JSON object or array from a string.
- * Handles pure JSON or JSON embedded in text (e.g., tool responses).
- *
- * @param str - The string to extract JSON from
- * @returns Object with found status, parsed JSON, and indices
- */
-export function extractFirstJson(str) {
-    if (typeof str !== 'string' || str.length === 0) {
-        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
-    }
-    // Find the first { or [ character
-    const objStart = str.indexOf('{');
-    const arrStart = str.indexOf('[');
-    if (objStart === -1 && arrStart === -1) {
-        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
-    }
-    let startIndex;
-    if (objStart === -1) {
-        startIndex = arrStart;
-    }
-    else if (arrStart === -1) {
-        startIndex = objStart;
-    }
-    else {
-        startIndex = Math.min(objStart, arrStart);
-    }
-    // Find the matching closing bracket using a stack-based approach
-    // Properly handle strings to avoid counting brackets inside strings
-    let depth = 0;
-    let inString = false;
-    let escapeNext = false;
-    for (let i = startIndex; i < str.length; i++) {
-        const char = str[i];
-        if (escapeNext) {
-            escapeNext = false;
-            continue;
-        }
-        if (char === '\\' && inString) {
-            escapeNext = true;
-            continue;
-        }
-        if (char === '"') {
-            inString = !inString;
-            continue;
-        }
-        if (inString)
-            continue;
-        if (char === '{' || char === '[')
-            depth++;
-        if (char === '}' || char === ']')
-            depth--;
-        if (depth === 0) {
-            // Found matching bracket, try to parse
-            const jsonStr = str.substring(startIndex, i + 1);
-            try {
-                const parsed = JSON.parse(jsonStr);
-                return { found: true, parsed, startIndex, endIndex: i };
-            }
-            catch {
-                // Not valid JSON at this bracket level, continue searching
-                // Reset and look for the next JSON start after this position
-                const nextResult = extractFirstJson(str.substring(i + 1));
-                if (nextResult.found) {
-                    return {
-                        found: true,
-                        parsed: nextResult.parsed,
-                        startIndex: i + 1 + nextResult.startIndex,
-                        endIndex: i + 1 + nextResult.endIndex,
-                    };
-                }
-                return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
-            }
-        }
-    }
-    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };
-}
-/**
- * Convert JSON content to TOON format for token efficiency.
- * Extracts JSON from string if embedded, converts to TOON.
- * Returns original string if:
- * - Already in TOON format
- * - Not JSON
- * - TOON conversion fails or is larger
- *
- * @param str - The string containing JSON to convert
- * @returns Object with conversion status, result, and reduction percentage
- */
-export function jsonToToon(str) {
-    if (!toonEncode || typeof str !== 'string') {
-        return {
-            converted: false,
-            result: str,
-            reduction: 0,
-            error: !toonEncode ? 'TOON library not loaded' : undefined,
-        };
-    }
-    // Check if already in TOON format - skip conversion
-    if (isToonFormat(str)) {
-        return { converted: false, result: str, reduction: 0, alreadyToon: true };
-    }
-    try {
-        const { found, parsed, startIndex, endIndex } = extractFirstJson(str);
-        if (!found || !parsed) {
-            return {
-                converted: false,
-                result: str,
-                reduction: 0,
-                error: 'No JSON found in content',
-            };
-        }
-        // Check if this JSON structure would benefit from TOON
-        // Text-heavy content (emails, documents) won't compress well
-        if (!isToonBeneficial(parsed)) {
-            return {
-                converted: false,
-                result: str,
-                reduction: 0,
-                error: 'Content is text-heavy (>70% string values), TOON not beneficial',
-            };
-        }
-        const toonResult = toonEncode(parsed);
-        // Preserve text before and after the JSON block
-        const textBefore = str.substring(0, startIndex);
-        const textAfter = str.substring(endIndex + 1);
-        // Build the full result: textBefore + TOON + textAfter
-        const fullResult = textBefore + toonResult + textAfter;
-        if (fullResult.length < str.length) {
-            const reduction = Math.round(((str.length - fullResult.length) / str.length) * 100);
-            return { converted: true, result: fullResult, reduction };
-        }
-        else {
-            // TOON output was larger or same size - not beneficial
-            return {
-                converted: false,
-                result: str,
-                reduction: 0,
-                error: `TOON output not smaller (${fullResult.length} >= ${str.length})`,
-            };
-        }
-    }
-    catch (err) {
-        // TOON encoding or extraction failed - log error for debugging
-        const errorMsg = err instanceof Error ? err.message : String(err);
-        return {
-            converted: false,
-            result: str,
-            reduction: 0,
-            error: `TOON conversion error: ${errorMsg}`,
-        };
-    }
-}
-/**
- * Check if TOON library is loaded and available
- */
-export function isToonAvailable() {
-    return toonEncode !== null;
-}
-/**
- * Analyze JSON structure to determine if TOON conversion would be beneficial.
- * Text-heavy content (emails, documents) doesn't compress well with TOON.
- * Structured data (API responses, metadata) compresses much better.
- *
- * @param parsed - Parsed JSON object
- * @returns true if TOON conversion is likely beneficial
- */
-function isToonBeneficial(parsed) {
-    if (!parsed || typeof parsed !== 'object')
-        return false;
-    const jsonStr = JSON.stringify(parsed);
-    const totalLength = jsonStr.length;
-    // Count characters in string values (text content)
-    let textContentLength = 0;
-    function countTextContent(obj) {
-        if (typeof obj === 'string') {
-            // Count string length (this is text content that TOON can't compress)
-            textContentLength += obj.length;
-        }
-        else if (Array.isArray(obj)) {
-            obj.forEach(countTextContent);
-        }
-        else if (obj && typeof obj === 'object') {
-            Object.values(obj).forEach(countTextContent);
-        }
-    }
-    countTextContent(parsed);
-    // Calculate ratio of text content to total JSON
-    const textRatio = textContentLength / totalLength;
-    // If more than 70% is text content, TOON won't help much
-    // TOON compresses structure (field names, brackets), not text
-    return textRatio < 0.7;
-}
-/**
- * Process tool output: apply TOON conversion if beneficial, then truncate if needed.
- * This is the main entry point for processing any tool output (regular or MCP).
- *
- * Flow:
- * 1. Check if already TOON format → skip conversion
- * 2. Try TOON conversion if content is JSON and large enough
- * 3. Truncate if still exceeds maxLength (with smart break points)
- *
- * @param content - The tool output content
- * @param options - Processing options
- * @returns Processed content and metadata
- */
-export function processToolOutput(content, options = {}) {
-    const { maxLength = 100000, enableToon = true, minSizeForToon = 1000, minReductionPercent = 10, // Increased from 5% - only apply TOON when clearly beneficial
-     } = options;
-    const originalLength = content.length;
-    const originalTokens = Math.ceil(originalLength / 4);
-    let result = content;
-    let toonConverted = false;
-    let alreadyToon = false;
-    let reduction = 0;
-    let truncated = false;
-    let toonError;
-    // Step 1: Check if already TOON format
-    if (isToonFormat(content)) {
-        alreadyToon = true;
-    }
-    // Step 2: Apply TOON conversion if enabled and content is large enough
-    else if (enableToon && content.length > minSizeForToon) {
-        const toonResult = jsonToToon(content);
-        if (toonResult.alreadyToon) {
-            alreadyToon = true;
-        }
-        else if (toonResult.converted &&
-            toonResult.reduction >= minReductionPercent) {
-            result = toonResult.result;
-            toonConverted = true;
-            reduction = toonResult.reduction;
-        }
-        else if (toonResult.error) {
-            // Track error for debugging
-            toonError = toonResult.error;
-        }
-        else if (toonResult.converted &&
-            toonResult.reduction < minReductionPercent) {
-            // TOON converted but reduction was too small to be worth it
-            toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;
-        }
-        else if (!toonResult.converted) {
-            // Conversion failed without explicit error - investigate
-            toonError =
-                toonResult.error || 'TOON conversion returned false with no error';
-        }
-    }
-    // Step 3: Truncate if still too long (with smart break points)
-    if (result.length > maxLength) {
-        let truncatedStr = result.substring(0, maxLength);
-        // Try to find a clean break point
-        if (toonConverted || alreadyToon) {
-            // For TOON format, break at newline for cleaner output
-            const lastNewline = truncatedStr.lastIndexOf('\n');
-            if (lastNewline > maxLength * 0.7) {
-                truncatedStr = truncatedStr.substring(0, lastNewline);
-            }
-        }
-        else {
-            // For JSON, try to find a clean JSON break point
-            const lastCompleteItem = truncatedStr.lastIndexOf('},');
-            const lastArrayItem = truncatedStr.lastIndexOf('],');
-            const breakPoint = Math.max(lastCompleteItem, lastArrayItem);
-            if (breakPoint > maxLength * 0.5) {
-                truncatedStr = truncatedStr.substring(0, breakPoint + 1);
-            }
-        }
-        // Build truncation message
-        const truncationInfo = toonConverted
-            ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +
-                `Still exceeds ${maxLength.toLocaleString()} char limit.`
-            : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;
-        result =
-            truncatedStr +
-                `\n\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;
-        truncated = true;
-    }
-    const finalLength = result.length;
-    const finalTokens = Math.ceil(finalLength / 4);
-    return {
-        content: result,
-        toonConverted,
-        truncated,
-        reduction,
-        alreadyToon,
-        originalLength,
-        originalTokens,
-        finalLength,
-        finalTokens,
-        toonError,
-    };
-}
-//# sourceMappingURL=toonFormat.js.map