@illuma-ai/agents 1.0.90 → 1.0.93

package/dist/esm/utils/tokens.mjs

@@ -1,6 +1,25 @@
-import { getEncoding } from 'js-tiktoken';
+import { Tokenizer } from 'ai-tokenizer';
 import { ContentTypes } from '../common/enum.mjs';
 
+const tokenizers = {};
+async function getTokenizer(encoding = 'o200k_base') {
+    const cached = tokenizers[encoding];
+    if (cached) {
+        return cached;
+    }
+    const data = encoding === 'claude'
+        ? await import('ai-tokenizer/encoding/claude')
+        : await import('ai-tokenizer/encoding/o200k_base');
+    const instance = new Tokenizer(data);
+    tokenizers[encoding] = instance;
+    return instance;
+}
+function encodingForModel(model) {
+    if (model.toLowerCase().includes('claude')) {
+        return 'claude';
+    }
+    return 'o200k_base';
+}
 function getTokenCountForMessage(message, getTokenCount) {
     const tokensPerMessage = 3;
     const processValue = (value) => {
@@ -48,74 +67,29 @@ function getTokenCountForMessage(message, getTokenCount) {
     processValue(message.content);
     return numTokens;
 }
-let encoderPromise;
-let tokenCounterPromise;
-/**
- * Simple character-based token estimation (~4 chars per token).
- * Used as fallback when tiktoken encoder fails to load.
- */
-const estimateTokens = (text) => Math.ceil(text.length / 4);
-async function getSharedEncoder() {
-    if (encoderPromise) {
-        return encoderPromise;
-    }
-    // Use bundled tokenizer data - no external network calls
-    encoderPromise = Promise.resolve().then(() => {
-        try {
-            return getEncoding('o200k_base');
-        }
-        catch {
-            // Fallback to null - will use character estimation
-            return null;
-        }
-    });
-    return encoderPromise;
-}
 /**
- * Creates a singleton token counter function that reuses the same encoder instance.
- * Falls back to character-based estimation (~4 chars per token) if encoder fails.
- * This ensures token counting never blocks LLM calls.
+ * Creates a token counter function using the specified encoding.
+ * Lazily loads the encoding data on first use via dynamic import.
  */
-const createTokenCounter = async () => {
-    if (tokenCounterPromise) {
-        return tokenCounterPromise;
-    }
-    tokenCounterPromise = (async () => {
-        const enc = await getSharedEncoder();
-        const countTokens = enc
-            ? (text) => enc.encode(text).length
-            : estimateTokens;
-        return (message) => getTokenCountForMessage(message, countTokens);
-    })();
-    return tokenCounterPromise;
+const createTokenCounter = async (encoding = 'o200k_base') => {
+    const tok = await getTokenizer(encoding);
+    const countTokens = (text) => tok.count(text);
+    return (message) => getTokenCountForMessage(message, countTokens);
 };
-/**
- * Utility to manage the token encoder lifecycle explicitly.
- * Useful for applications that need fine-grained control over resource management.
- */
+/** Utility to manage the token encoder lifecycle explicitly. */
 const TokenEncoderManager = {
-    /**
-     * Pre-initializes the encoder. This can be called during app startup
-     * to avoid lazy loading delays later.
-     */
     async initialize() {
-        await getSharedEncoder();
+        // No-op: ai-tokenizer is synchronously initialized from bundled data.
     },
-    /**
-     * Clears the cached encoder and token counter.
-     * Useful for testing or when you need to force a fresh reload.
-     */
     reset() {
-        encoderPromise = undefined;
-        tokenCounterPromise = undefined;
+        for (const key of Object.keys(tokenizers)) {
+            delete tokenizers[key];
+        }
     },
-    /**
-     * Checks if the encoder has been initialized.
-     */
     isInitialized() {
-        return encoderPromise !== undefined;
+        return Object.keys(tokenizers).length > 0;
     },
 };
 
-export { TokenEncoderManager, createTokenCounter, getTokenCountForMessage };
+export { TokenEncoderManager, createTokenCounter, encodingForModel, getTokenCountForMessage };
 //# sourceMappingURL=tokens.mjs.map

package/dist/esm/utils/tokens.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.mjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { getEncoding, type Tiktoken } from 'js-tiktoken';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number\n): number {\n  const tokensPerMessage = 3;\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const item of value) {\n        if (\n          !item ||\n          !item.type ||\n          item.type === ContentTypes.ERROR ||\n          item.type === ContentTypes.IMAGE_URL\n        ) {\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call?.name || '';\n          if (toolName != null && toolName && typeof toolName === 'string') {\n            numTokens += getTokenCount(toolName);\n          }\n\n          const args = item.tool_call?.args || '';\n          if (args != null && args && typeof args === 'string') {\n            numTokens += getTokenCount(args);\n          }\n\n          const output = item.tool_call?.output || '';\n          if (output != null && output && typeof output === 'string') {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n\n        if (!nestedValue) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\nlet encoderPromise: Promise<Tiktoken | null> | undefined;\nlet tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;\n\n/**\n * Simple character-based token estimation (~4 chars per token).\n * Used as fallback when tiktoken encoder fails to load.\n */\nconst estimateTokens = (text: string): number => Math.ceil(text.length / 4);\n\nasync function getSharedEncoder(): Promise<Tiktoken | null> {\n  if (encoderPromise) {\n    return encoderPromise;\n  }\n  // Use bundled tokenizer data - no external network calls\n  encoderPromise = Promise.resolve().then(() => {\n    try {\n      return getEncoding('o200k_base');\n    } catch {\n      // Fallback to null - will use character estimation\n      return null;\n    }\n  });\n  return encoderPromise;\n}\n\n/**\n * Creates a singleton token counter function that reuses the same encoder instance.\n * Falls back to character-based estimation (~4 chars per token) if encoder fails.\n * This ensures token counting never blocks LLM calls.\n */\nexport const createTokenCounter = async (): Promise<\n  (message: BaseMessage) => number\n> => {\n  if (tokenCounterPromise) {\n    return tokenCounterPromise;\n  }\n\n  tokenCounterPromise = (async (): Promise<\n    (message: BaseMessage) => number\n  > => {\n    const enc = await getSharedEncoder();\n    const countTokens = enc\n      ? (text: string): number => enc.encode(text).length\n      : estimateTokens;\n    return (message: BaseMessage): number =>\n      getTokenCountForMessage(message, countTokens);\n  })();\n\n  return tokenCounterPromise;\n};\n\n/**\n * Utility to manage the token encoder lifecycle explicitly.\n * Useful for applications that need fine-grained control over resource management.\n */\nexport const TokenEncoderManager = {\n  /**\n   * Pre-initializes the encoder. This can be called during app startup\n   * to avoid lazy loading delays later.\n   */\n  async initialize(): Promise<void> {\n    await getSharedEncoder();\n  },\n\n  /**\n   * Clears the cached encoder and token counter.\n   * Useful for testing or when you need to force a fresh reload.\n   */\n  reset(): void {\n    encoderPromise = undefined;\n    tokenCounterPromise = undefined;\n  },\n\n  /**\n   * Checks if the encoder has been initialized.\n   */\n  isInitialized(): boolean {\n    return encoderPromise !== undefined;\n  },\n};\n"],"names":[],"mappings":";;;AAIgB,SAAA,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EACpC;oBACA;;AAGF,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;;oBAGtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;;oBAGlC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;;oBAEpC;;gBAGF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;;gBAGF,YAAY,CAAC,WAAW,CAAC;;;AAEtB,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;;AAC5B,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AACvC,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AAEhD,KAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA,IAAI,cAAoD;AACxD,IAAI,mBAA0E;AAE9E;;;AAGG;AACH,MAAM,cAAc,GAAG,CAAC,IAAY,KAAa,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AAE3E,eAAe,gBAAgB,GAAA;IAC7B,IAAI,cAAc,EAAE;AAClB,QAAA,OAAO,cAAc;;;IAGvB,cAAc,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,MAAK;AAC3C,QAAA,IAAI;AACF,YAAA,OAAO,WAAW,CAAC,YAAY,CAAC;;AAChC,QAAA,MAAM;;AAEN,YAAA,OAAO,IAAI;;AAEf,KAAC,CAAC;AACF,IAAA,OAAO,cAAc;AACvB;AAEA;;;;AAIG;AACU,MAAA,kBAAkB,GAAG,YAE9B;IACF,IAAI,mBAAmB,EAAE;AACvB,QAAA,OAAO,mBAAmB;;AAG5B,IAAA,mBAAmB,GAAG,CAAC,YAEnB;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE;QACpC,MAAM,WAAW,GAAG;AAClB,cAAE,CAAC,IAAY,KAAa,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;cAC3C,cAAc;QAClB,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;KAChD,GAAG;AAEJ,IAAA,OAAO,mBAAmB;AAC5B;AAEA;;;AAGG;AACU,MAAA,mBAAmB,GAAG;AACjC;;;AAGG;AACH,IAAA,MAAM,UAAU,GAAA;QACd,MAAM,gBAAgB,EAAE;KACzB;AAED;;;AAGG;IACH,KAAK,GAAA;QACH,cAAc,GAAG,SAAS;QAC1B,mBAAmB,GAAG,SAAS;KAChC;AAED;;AAEG;IACH,aAAa,GAAA;QACX,OAAO,cAAc,KAAK,SAAS;KACpC;;;;;"}
+{"version":3,"file":"tokens.mjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tokenizer } from 'ai-tokenizer';\r\nimport type { BaseMessage } from '@langchain/core/messages';\r\nimport { ContentTypes } from '@/common/enum';\r\n\r\nexport type EncodingName = 'o200k_base' | 'claude';\r\n\r\nconst tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};\r\n\r\nasync function getTokenizer(\r\n  encoding: EncodingName = 'o200k_base'\r\n): Promise<Tokenizer> {\r\n  const cached = tokenizers[encoding];\r\n  if (cached) {\r\n    return cached;\r\n  }\r\n  const data =\r\n    encoding === 'claude'\r\n      ? await import('ai-tokenizer/encoding/claude')\r\n      : await import('ai-tokenizer/encoding/o200k_base');\r\n  const instance = new Tokenizer(data);\r\n  tokenizers[encoding] = instance;\r\n  return instance;\r\n}\r\n\r\nexport function encodingForModel(model: string): EncodingName {\r\n  if (model.toLowerCase().includes('claude')) {\r\n    return 'claude';\r\n  }\r\n  return 'o200k_base';\r\n}\r\n\r\nexport function getTokenCountForMessage(\r\n  message: BaseMessage,\r\n  getTokenCount: (text: string) => number\r\n): number {\r\n  const tokensPerMessage = 3;\r\n\r\n  const processValue = (value: unknown): void => {\r\n    if (Array.isArray(value)) {\r\n      for (const item of value) {\r\n        if (\r\n          !item ||\r\n          !item.type ||\r\n          item.type === ContentTypes.ERROR ||\r\n          item.type === ContentTypes.IMAGE_URL\r\n        ) {\r\n          continue;\r\n        }\r\n\r\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\r\n          const toolName = item.tool_call?.name || '';\r\n          if (toolName != null && toolName && typeof toolName === 'string') {\r\n            numTokens += getTokenCount(toolName);\r\n          }\r\n\r\n          const args = item.tool_call?.args || '';\r\n          if (args != null && args && typeof args === 'string') {\r\n            numTokens += getTokenCount(args);\r\n          }\r\n\r\n          const output = item.tool_call?.output || '';\r\n          if (output != null && output && typeof output === 'string') {\r\n            numTokens += getTokenCount(output);\r\n          }\r\n          continue;\r\n        }\r\n\r\n        const nestedValue = item[item.type];\r\n\r\n        if (!nestedValue) {\r\n          continue;\r\n        }\r\n\r\n        processValue(nestedValue);\r\n      }\r\n    } else if (typeof value === 'string') {\r\n      numTokens += getTokenCount(value);\r\n    } else if (typeof value === 'number') {\r\n      numTokens += getTokenCount(value.toString());\r\n    } else if (typeof value === 'boolean') {\r\n      numTokens += getTokenCount(value.toString());\r\n    }\r\n  };\r\n\r\n  let numTokens = tokensPerMessage;\r\n  processValue(message.content);\r\n  return numTokens;\r\n}\r\n\r\n/**\r\n * Creates a token counter function using the specified encoding.\r\n * Lazily loads the encoding data on first use via dynamic import.\r\n */\r\nexport const createTokenCounter = async (\r\n  encoding: EncodingName = 'o200k_base'\r\n): Promise<(message: BaseMessage) => number> => {\r\n  const tok = await getTokenizer(encoding);\r\n  const countTokens = (text: string): number => tok.count(text);\r\n  return (message: BaseMessage): number =>\r\n    getTokenCountForMessage(message, countTokens);\r\n};\r\n\r\n/** Utility to manage the token encoder lifecycle explicitly. */\r\nexport const TokenEncoderManager = {\r\n  async initialize(): Promise<void> {\r\n    // No-op: ai-tokenizer is synchronously initialized from bundled data.\r\n  },\r\n\r\n  reset(): void {\r\n    for (const key of Object.keys(tokenizers)) {\r\n      delete tokenizers[key as EncodingName];\r\n    }\r\n  },\r\n\r\n  isInitialized(): boolean {\r\n    return Object.keys(tokenizers).length > 0;\r\n  },\r\n};\r\n"],"names":[],"mappings":";;;AAMA,MAAM,UAAU,GAA6C,EAAE;AAE/D,eAAe,YAAY,CACzB,QAAA,GAAyB,YAAY,EAAA;AAErC,IAAA,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;IACnC,IAAI,MAAM,EAAE;AACV,QAAA,OAAO,MAAM;IACf;AACA,IAAA,MAAM,IAAI,GACR,QAAQ,KAAK;AACX,UAAE,MAAM,OAAO,8BAA8B;AAC7C,UAAE,MAAM,OAAO,kCAAkC,CAAC;AACtD,IAAA,MAAM,QAAQ,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC;AACpC,IAAA,UAAU,CAAC,QAAQ,CAAC,GAAG,QAAQ;AAC/B,IAAA,OAAO,QAAQ;AACjB;AAEM,SAAU,gBAAgB,CAAC,KAAa,EAAA;IAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE;AAC1C,QAAA,OAAO,QAAQ;IACjB;AACA,IAAA,OAAO,YAAY;AACrB;AAEM,SAAU,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EACpC;oBACA;gBACF;AAEA,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;oBACtC;oBAEA,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;oBAClC;oBAEA,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;oBACpC;oBACA;gBACF;gBAEA,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;gBACF;gBAEA,YAAY,CAAC,WAAW,CAAC;YAC3B;QACF;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;QACnC;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9C;AAAO,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC9C;AACF,IAAA,CAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA;;;AAGG;MACU,kBAAkB,GAAG,OAChC,QAAA,GAAyB,YAAY,KACQ;AAC7C,IAAA,MAAM,GAAG,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC;AACxC,IAAA,MAAM,WAAW,GAAG,CAAC,IAAY,KAAa,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;IAC7D,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;AACjD;AAEA;AACO,MAAM,mBAAmB,GAAG;AACjC,IAAA,MAAM,UAAU,GAAA;;IAEhB,CAAC;IAED,KAAK,GAAA;QACH,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE;AACzC,YAAA,OAAO,UAAU,CAAC,GAAmB,CAAC;QACxC;IACF,CAAC;IAED,aAAa,GAAA;QACX,OAAO,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC;IAC3C,CAAC;;;;;"}

package/dist/esm/utils/toolCallContinuation.mjs

@@ -0,0 +1,52 @@
+import { FinishReasons } from '../common/enum.mjs';
+
+/**
+ * toolCallContinuation — Utilities for detecting and classifying LLM finish
+ * reasons across providers. Used to detect max_tokens truncation and normalize
+ * stop reason extraction from provider-specific metadata formats.
+ *
+ * @module
+ */
+/**
+ * Extract the finish/stop reason from an AIMessageChunk's response_metadata.
+ * Centralizes provider-specific metadata access patterns.
+ *
+ * Provider formats:
+ * - OpenAI/Azure: `response_metadata.finish_reason`
+ * - Anthropic direct: `response_metadata.stop_reason`
+ * - Bedrock invoke: `response_metadata.stopReason`
+ * - Bedrock streaming: `response_metadata.messageStop.stopReason`
+ * - VertexAI/Google: `response_metadata.finishReason`
+ *
+ * @param chunk - The final accumulated AIMessageChunk
+ * @returns The finish reason string, or undefined if not found
+ */
+function extractFinishReason(chunk) {
+    if (!chunk || !('response_metadata' in chunk))
+        return undefined;
+    const meta = chunk.response_metadata;
+    if (!meta)
+        return undefined;
+    // Bedrock streaming nests stopReason inside messageStop: { stopReason: '...' }
+    const messageStop = meta.messageStop;
+    return (meta.finish_reason ?? // OpenAI/Azure
+        meta.stop_reason ?? // Anthropic direct API
+        meta.stopReason ?? // Bedrock invoke (non-streaming)
+        messageStop?.stopReason ?? // Bedrock streaming
+        meta.finishReason // VertexAI/Google
+    );
+}
+/**
+ * Check if a finish reason indicates the response was truncated by token limits.
+ *
+ * @param reason - The finish reason from extractFinishReason()
+ * @returns true if the response was cut short by max_tokens
+ */
+function isMaxTokensFinish(reason) {
+    if (!reason)
+        return false;
+    return reason === FinishReasons.MAX_TOKENS || reason === FinishReasons.LENGTH;
+}
+
+export { extractFinishReason, isMaxTokensFinish };
+//# sourceMappingURL=toolCallContinuation.mjs.map

package/dist/esm/utils/toolCallContinuation.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolCallContinuation.mjs","sources":["../../../src/utils/toolCallContinuation.ts"],"sourcesContent":["/**\n * toolCallContinuation — Utilities for detecting and classifying LLM finish\n * reasons across providers. Used to detect max_tokens truncation and normalize\n * stop reason extraction from provider-specific metadata formats.\n *\n * @module\n */\n\nimport type { AIMessageChunk } from '@langchain/core/messages';\nimport { FinishReasons } from '@/common';\n\n/**\n * Extract the finish/stop reason from an AIMessageChunk's response_metadata.\n * Centralizes provider-specific metadata access patterns.\n *\n * Provider formats:\n * - OpenAI/Azure: `response_metadata.finish_reason`\n * - Anthropic direct: `response_metadata.stop_reason`\n * - Bedrock invoke: `response_metadata.stopReason`\n * - Bedrock streaming: `response_metadata.messageStop.stopReason`\n * - VertexAI/Google: `response_metadata.finishReason`\n *\n * @param chunk - The final accumulated AIMessageChunk\n * @returns The finish reason string, or undefined if not found\n */\nexport function extractFinishReason(\n  chunk: AIMessageChunk | undefined\n): string | undefined {\n  if (!chunk || !('response_metadata' in chunk)) return undefined;\n\n  const meta = chunk.response_metadata as Record<string, unknown>;\n  if (!meta) return undefined;\n\n  // Bedrock streaming nests stopReason inside messageStop: { stopReason: '...' }\n  const messageStop = meta.messageStop as Record<string, unknown> | undefined;\n\n  return (\n    (meta.finish_reason as string | undefined) ?? // OpenAI/Azure\n    (meta.stop_reason as string | undefined) ?? // Anthropic direct API\n    (meta.stopReason as string | undefined) ?? // Bedrock invoke (non-streaming)\n    (messageStop?.stopReason as string | undefined) ?? // Bedrock streaming\n    (meta.finishReason as string | undefined) // VertexAI/Google\n  );\n}\n\n/**\n * Check if a finish reason indicates the response was truncated by token limits.\n *\n * @param reason - The finish reason from extractFinishReason()\n * @returns true if the response was cut short by max_tokens\n */\nexport function isMaxTokensFinish(reason: string | undefined): boolean {\n  if (!reason) return false;\n  return reason === FinishReasons.MAX_TOKENS || reason === FinishReasons.LENGTH;\n}\n"],"names":[],"mappings":";;AAAA;;;;;;AAMG;AAKH;;;;;;;;;;;;;AAaG;AACG,SAAU,mBAAmB,CACjC,KAAiC,EAAA;IAEjC,IAAI,CAAC,KAAK,IAAI,EAAE,mBAAmB,IAAI,KAAK,CAAC;AAAE,QAAA,OAAO,SAAS;AAE/D,IAAA,MAAM,IAAI,GAAG,KAAK,CAAC,iBAA4C;AAC/D,IAAA,IAAI,CAAC,IAAI;AAAE,QAAA,OAAO,SAAS;;AAG3B,IAAA,MAAM,WAAW,GAAG,IAAI,CAAC,WAAkD;AAE3E,IAAA,QACG,IAAI,CAAC,aAAoC;QACzC,IAAI,CAAC,WAAkC;QACvC,IAAI,CAAC,UAAiC;QACtC,WAAW,EAAE,UAAiC;QAC9C,IAAI,CAAC,YAAmC;;AAE7C;AAEA;;;;;AAKG;AACG,SAAU,iBAAiB,CAAC,MAA0B,EAAA;AAC1D,IAAA,IAAI,CAAC,MAAM;AAAE,QAAA,OAAO,KAAK;IACzB,OAAO,MAAM,KAAK,aAAa,CAAC,UAAU,IAAI,MAAM,KAAK,aAAa,CAAC,MAAM;AAC/E;;;;"}

package/dist/esm/utils/toonFormat.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"toonFormat.mjs","sources":["../../../src/utils/toonFormat.ts"],"sourcesContent":["/**\n * TOON Format Utility\n *\n * Provides JSON to TOON conversion for token-efficient encoding.\n * TOON format can reduce token count by 30-60% for JSON data.\n *\n * Uses the official @toon-format/toon library when available.\n */\n\n// Dynamic import holder for the TOON library (ESM)\nlet toonEncode: ((data: unknown) => string) | null = null;\nlet toonLoadAttempted = false;\nlet toonLoadPromise: Promise<boolean> | null = null;\n\n/**\n * Lazily loads the TOON library\n * Tries both ESM dynamic import and CommonJS require for compatibility\n */\nasync function loadToonLibrary(): Promise<boolean> {\n  if (toonLoadAttempted) return toonEncode !== null;\n  toonLoadAttempted = true;\n\n  // Try ESM dynamic import first\n  try {\n    const mod = await import('@toon-format/toon');\n    toonEncode = mod.encode;\n    return true;\n  } catch {\n    // ESM import failed, try CommonJS require\n    try {\n      // eslint-disable-next-line @typescript-eslint/no-require-imports\n      const mod = require('@toon-format/toon');\n      toonEncode = mod.encode;\n      return true;\n    } catch {\n      // Library not available - that's OK, we'll return original content\n      return false;\n    }\n  }\n}\n\n/**\n * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.\n */\nexport async function ensureToonLoaded(): Promise<boolean> {\n  if (!toonLoadPromise) {\n    toonLoadPromise = loadToonLibrary();\n  }\n  return toonLoadPromise;\n}\n\n// Start loading immediately\nensureToonLoaded();\n\n/**\n * Check if a string appears to be in TOON format.\n * TOON format characteristics:\n * - Uses key: value syntax (like YAML)\n * - Uses array notation like [3] or {fields}\n * - Does NOT start with { or [\n * - Often has patterns like \"name[N]:\" or \"name{fields}:\"\n *\n * @param str - String to check\n * @returns true if string appears to be TOON format\n */\nexport function isToonFormat(str: string): boolean {\n  if (typeof str !== 'string' || str.length === 0) return false;\n\n  const trimmed = str.trim();\n\n  // TOON doesn't start with JSON brackets\n  if (trimmed.startsWith('{') || trimmed.startsWith('[')) return false;\n\n  // Check for TOON-specific patterns:\n  // 1. Key-value with colon (but not URL-like patterns)\n  // 2. Array notation like \"items[3]:\" or \"data[10]:\"\n  // 3. Object schema notation like \"items{id,name}:\"\n  const toonPatterns = [\n    /^\\w+:$/m, // Simple key: at start of line\n    /^\\w+\\[\\d+\\]:/m, // Array notation: items[3]:\n    /^\\w+\\{[^}]+\\}:/m, // Schema notation: items{id,name}:\n    /^\\w+\\[\\d+\\]\\{[^}]+\\}:/m, // Combined: items[3]{id,name}:\n  ];\n\n  // Must match at least one TOON pattern\n  const hasToonPattern = toonPatterns.some((pattern) => pattern.test(trimmed));\n\n  // Additional check: TOON typically has multiple lines with consistent indentation\n  const lines = trimmed.split('\\n').filter((l) => l.trim());\n  const hasMultipleKeyValueLines =\n    lines.filter((l) => /^\\s*\\w+.*:/.test(l)).length >= 2;\n\n  return hasToonPattern || hasMultipleKeyValueLines;\n}\n\n/**\n * Extract the first valid JSON object or array from a string.\n * Handles pure JSON or JSON embedded in text (e.g., tool responses).\n *\n * @param str - The string to extract JSON from\n * @returns Object with found status, parsed JSON, and indices\n */\nexport function extractFirstJson(str: string): {\n  found: boolean;\n  parsed: unknown;\n  startIndex: number;\n  endIndex: number;\n} {\n  if (typeof str !== 'string' || str.length === 0) {\n    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n  }\n\n  // Find the first { or [ character\n  const objStart = str.indexOf('{');\n  const arrStart = str.indexOf('[');\n\n  if (objStart === -1 && arrStart === -1) {\n    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n  }\n\n  let startIndex: number;\n  if (objStart === -1) {\n    startIndex = arrStart;\n  } else if (arrStart === -1) {\n    startIndex = objStart;\n  } else {\n    startIndex = Math.min(objStart, arrStart);\n  }\n\n  // Find the matching closing bracket using a stack-based approach\n  // Properly handle strings to avoid counting brackets inside strings\n  let depth = 0;\n  let inString = false;\n  let escapeNext = false;\n\n  for (let i = startIndex; i < str.length; i++) {\n    const char = str[i];\n\n    if (escapeNext) {\n      escapeNext = false;\n      continue;\n    }\n\n    if (char === '\\\\' && inString) {\n      escapeNext = true;\n      continue;\n    }\n\n    if (char === '\"') {\n      inString = !inString;\n      continue;\n    }\n\n    if (inString) continue;\n\n    if (char === '{' || char === '[') depth++;\n    if (char === '}' || char === ']') depth--;\n\n    if (depth === 0) {\n      // Found matching bracket, try to parse\n      const jsonStr = str.substring(startIndex, i + 1);\n      try {\n        const parsed = JSON.parse(jsonStr);\n        return { found: true, parsed, startIndex, endIndex: i };\n      } catch {\n        // Not valid JSON at this bracket level, continue searching\n        // Reset and look for the next JSON start after this position\n        const nextResult = extractFirstJson(str.substring(i + 1));\n        if (nextResult.found) {\n          return {\n            found: true,\n            parsed: nextResult.parsed,\n            startIndex: i + 1 + nextResult.startIndex,\n            endIndex: i + 1 + nextResult.endIndex,\n          };\n        }\n        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n      }\n    }\n  }\n\n  return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n}\n\n/**\n * Convert JSON content to TOON format for token efficiency.\n * Extracts JSON from string if embedded, converts to TOON.\n * Returns original string if:\n * - Already in TOON format\n * - Not JSON\n * - TOON conversion fails or is larger\n *\n * @param str - The string containing JSON to convert\n * @returns Object with conversion status, result, and reduction percentage\n */\nexport function jsonToToon(str: string): {\n  converted: boolean;\n  result: string;\n  reduction: number;\n  alreadyToon?: boolean;\n  error?: string;\n} {\n  if (!toonEncode || typeof str !== 'string') {\n    return {\n      converted: false,\n      result: str,\n      reduction: 0,\n      error: !toonEncode ? 'TOON library not loaded' : undefined,\n    };\n  }\n\n  // Check if already in TOON format - skip conversion\n  if (isToonFormat(str)) {\n    return { converted: false, result: str, reduction: 0, alreadyToon: true };\n  }\n\n  try {\n    const { found, parsed, startIndex, endIndex } = extractFirstJson(str);\n\n    if (!found || !parsed) {\n      return {\n        converted: false,\n        result: str,\n        reduction: 0,\n        error: 'No JSON found in content',\n      };\n    }\n\n    // Check if this JSON structure would benefit from TOON\n    // Text-heavy content (emails, documents) won't compress well\n    if (!isToonBeneficial(parsed)) {\n      return {\n        converted: false,\n        result: str,\n        reduction: 0,\n        error:\n          'Content is text-heavy (>70% string values), TOON not beneficial',\n      };\n    }\n\n    const toonResult = toonEncode(parsed);\n\n    // Preserve text before and after the JSON block\n    const textBefore = str.substring(0, startIndex);\n    const textAfter = str.substring(endIndex + 1);\n\n    // Build the full result: textBefore + TOON + textAfter\n    const fullResult = textBefore + toonResult + textAfter;\n\n    if (fullResult.length < str.length) {\n      const reduction = Math.round(\n        ((str.length - fullResult.length) / str.length) * 100\n      );\n      return { converted: true, result: fullResult, reduction };\n    } else {\n      // TOON output was larger or same size - not beneficial\n      return {\n        converted: false,\n        result: str,\n        reduction: 0,\n        error: `TOON output not smaller (${fullResult.length} >= ${str.length})`,\n      };\n    }\n  } catch (err) {\n    // TOON encoding or extraction failed - log error for debugging\n    const errorMsg = err instanceof Error ? err.message : String(err);\n    return {\n      converted: false,\n      result: str,\n      reduction: 0,\n      error: `TOON conversion error: ${errorMsg}`,\n    };\n  }\n}\n\n/**\n * Check if TOON library is loaded and available\n */\nexport function isToonAvailable(): boolean {\n  return toonEncode !== null;\n}\n\n/**\n * Result of processing tool output\n */\nexport interface ProcessToolOutputResult {\n  /** The processed content string */\n  content: string;\n  /** Whether TOON conversion was applied */\n  toonConverted: boolean;\n  /** Whether content was truncated */\n  truncated: boolean;\n  /** Percentage reduction from TOON (0 if not converted) */\n  reduction: number;\n  /** Whether input was already in TOON format */\n  alreadyToon: boolean;\n  /** Original content length */\n  originalLength: number;\n  /** Estimated original tokens (~4 chars per token) */\n  originalTokens: number;\n  /** Final content length */\n  finalLength: number;\n  /** Estimated final tokens */\n  finalTokens: number;\n  /** Error message if TOON conversion failed (for debugging) */\n  toonError?: string;\n}\n\n/**\n * Options for processing tool output\n */\nexport interface ProcessToolOutputOptions {\n  /** Maximum output length in characters (default: 100000) */\n  maxLength?: number;\n  /** Whether to apply TOON conversion (default: true) */\n  enableToon?: boolean;\n  /** Minimum content size to attempt TOON conversion (default: 1000) */\n  minSizeForToon?: number;\n  /** Minimum reduction % to accept TOON result (default: 10) */\n  minReductionPercent?: number;\n}\n\n/**\n * Analyze JSON structure to determine if TOON conversion would be beneficial.\n * Text-heavy content (emails, documents) doesn't compress well with TOON.\n * Structured data (API responses, metadata) compresses much better.\n *\n * @param parsed - Parsed JSON object\n * @returns true if TOON conversion is likely beneficial\n */\nfunction isToonBeneficial(parsed: unknown): boolean {\n  if (!parsed || typeof parsed !== 'object') return false;\n\n  const jsonStr = JSON.stringify(parsed);\n  const totalLength = jsonStr.length;\n\n  // Count characters in string values (text content)\n  let textContentLength = 0;\n\n  function countTextContent(obj: unknown): void {\n    if (typeof obj === 'string') {\n      // Count string length (this is text content that TOON can't compress)\n      textContentLength += obj.length;\n    } else if (Array.isArray(obj)) {\n      obj.forEach(countTextContent);\n    } else if (obj && typeof obj === 'object') {\n      Object.values(obj).forEach(countTextContent);\n    }\n  }\n\n  countTextContent(parsed);\n\n  // Calculate ratio of text content to total JSON\n  const textRatio = textContentLength / totalLength;\n\n  // If more than 70% is text content, TOON won't help much\n  // TOON compresses structure (field names, brackets), not text\n  return textRatio < 0.7;\n}\n\n/**\n * Process tool output: apply TOON conversion if beneficial, then truncate if needed.\n * This is the main entry point for processing any tool output (regular or MCP).\n *\n * Flow:\n * 1. Check if already TOON format → skip conversion\n * 2. Try TOON conversion if content is JSON and large enough\n * 3. Truncate if still exceeds maxLength (with smart break points)\n *\n * @param content - The tool output content\n * @param options - Processing options\n * @returns Processed content and metadata\n */\nexport function processToolOutput(\n  content: string,\n  options: ProcessToolOutputOptions = {}\n): ProcessToolOutputResult {\n  const {\n    maxLength = 100000,\n    enableToon = true,\n    minSizeForToon = 1000,\n    minReductionPercent = 10, // Increased from 5% - only apply TOON when clearly beneficial\n  } = options;\n\n  const originalLength = content.length;\n  const originalTokens = Math.ceil(originalLength / 4);\n\n  let result = content;\n  let toonConverted = false;\n  let alreadyToon = false;\n  let reduction = 0;\n  let truncated = false;\n  let toonError: string | undefined;\n\n  // Step 1: Check if already TOON format\n  if (isToonFormat(content)) {\n    alreadyToon = true;\n  }\n  // Step 2: Apply TOON conversion if enabled and content is large enough\n  else if (enableToon && content.length > minSizeForToon) {\n    const toonResult = jsonToToon(content);\n    if (toonResult.alreadyToon) {\n      alreadyToon = true;\n    } else if (\n      toonResult.converted &&\n      toonResult.reduction >= minReductionPercent\n    ) {\n      result = toonResult.result;\n      toonConverted = true;\n      reduction = toonResult.reduction;\n    } else if (toonResult.error) {\n      // Track error for debugging\n      toonError = toonResult.error;\n    } else if (\n      toonResult.converted &&\n      toonResult.reduction < minReductionPercent\n    ) {\n      // TOON converted but reduction was too small to be worth it\n      toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;\n    } else if (!toonResult.converted) {\n      // Conversion failed without explicit error - investigate\n      toonError =\n        toonResult.error || 'TOON conversion returned false with no error';\n    }\n  }\n\n  // Step 3: Truncate if still too long (with smart break points)\n  if (result.length > maxLength) {\n    let truncatedStr = result.substring(0, maxLength);\n\n    // Try to find a clean break point\n    if (toonConverted || alreadyToon) {\n      // For TOON format, break at newline for cleaner output\n      const lastNewline = truncatedStr.lastIndexOf('\\n');\n      if (lastNewline > maxLength * 0.7) {\n        truncatedStr = truncatedStr.substring(0, lastNewline);\n      }\n    } else {\n      // For JSON, try to find a clean JSON break point\n      const lastCompleteItem = truncatedStr.lastIndexOf('},');\n      const lastArrayItem = truncatedStr.lastIndexOf('],');\n      const breakPoint = Math.max(lastCompleteItem, lastArrayItem);\n      if (breakPoint > maxLength * 0.5) {\n        truncatedStr = truncatedStr.substring(0, breakPoint + 1);\n      }\n    }\n\n    // Build truncation message\n    const truncationInfo = toonConverted\n      ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +\n        `Still exceeds ${maxLength.toLocaleString()} char limit.`\n      : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;\n\n    result =\n      truncatedStr +\n      `\\n\\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;\n    truncated = true;\n  }\n\n  const finalLength = result.length;\n  const finalTokens = Math.ceil(finalLength / 4);\n\n  return {\n    content: result,\n    toonConverted,\n    truncated,\n    reduction,\n    alreadyToon,\n    originalLength,\n    originalTokens,\n    finalLength,\n    finalTokens,\n    toonError,\n  };\n}\n"],"names":[],"mappings":"AAAA;;;;;;;AAOG;AAEH;AACA,IAAI,UAAU,GAAuC,IAAI;AACzD,IAAI,iBAAiB,GAAG,KAAK;AAC7B,IAAI,eAAe,GAA4B,IAAI;AAEnD;;;AAGG;AACH,eAAe,eAAe,GAAA;AAC5B,IAAA,IAAI,iBAAiB;QAAE,OAAO,UAAU,KAAK,IAAI;IACjD,iBAAiB,GAAG,IAAI;;AAGxB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,OAAO,mBAAmB,CAAC;AAC7C,QAAA,UAAU,GAAG,GAAG,CAAC,MAAM;AACvB,QAAA,OAAO,IAAI;;AACX,IAAA,MAAM;;AAEN,QAAA,IAAI;;AAEF,YAAA,MAAM,GAAG,GAAG,OAAO,CAAC,mBAAmB,CAAC;AACxC,YAAA,UAAU,GAAG,GAAG,CAAC,MAAM;AACvB,YAAA,OAAO,IAAI;;AACX,QAAA,MAAM;;AAEN,YAAA,OAAO,KAAK;;;AAGlB;AAEA;;AAEG;AACI,eAAe,gBAAgB,GAAA;IACpC,IAAI,CAAC,eAAe,EAAE;QACpB,eAAe,GAAG,eAAe,EAAE;;AAErC,IAAA,OAAO,eAAe;AACxB;AAEA;AACA,gBAAgB,EAAE;AAElB;;;;;;;;;;AAUG;AACG,SAAU,YAAY,CAAC,GAAW,EAAA;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,KAAK;AAE7D,IAAA,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE;;AAG1B,IAAA,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;AAAE,QAAA,OAAO,KAAK;;;;;AAMpE,IAAA,MAAM,YAAY,GAAG;AACnB,QAAA,SAAS;AACT,QAAA,eAAe;AACf,QAAA,iBAAiB;AACjB,QAAA,wBAAwB;KACzB;;AAGD,IAAA,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;IAG5E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,wBAAwB,GAC5B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC;IAEvD,OAAO,cAAc,IAAI,wBAAwB;AACnD;AAEA;;;;;;AAMG;AACG,SAAU,gBAAgB,CAAC,GAAW,EAAA;IAM1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;AAC/C,QAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;;;IAIrE,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IAEjC,IAAI,QAAQ,KAAK,EAAE,IAAI,QAAQ,KAAK,EAAE,EAAE;AACtC,QAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;;AAGrE,IAAA,IAAI,UAAkB;AACtB,IAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;QACnB,UAAU,GAAG,QAAQ;;AAChB,SAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;QAC1B,UAAU,GAAG,QAAQ;;SAChB;QACL,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC;;;;IAK3C,IAAI,KAAK,GAAG,CAAC;IACb,IAAI,QAAQ,GAAG,KAAK;IACpB,IAAI,UAAU,GAAG,KAAK;AAEtB,IAAA,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC5C,QAAA,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QAEnB,IAAI,UAAU,EAAE;YACd,UAAU,GAAG,KAAK;YAClB;;AAGF,QAAA,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,EAAE;YAC7B,UAAU,GAAG,IAAI;YACjB;;AAGF,QAAA,IAAI,IAAI,KAAK,GAAG,EAAE;YAChB,QAAQ,GAAG,CAAC,QAAQ;YACpB;;AAGF,QAAA,IAAI,QAAQ;YAAE;AAEd,QAAA,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;AAAE,YAAA,KAAK,EAAE;AACzC,QAAA,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;AAAE,YAAA,KAAK,EAAE;AAEzC,QAAA,IAAI,KAAK,KAAK,CAAC,EAAE;;AAEf,YAAA,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;AAChD,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;AAClC,gBAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE;;AACvD,YAAA,MAAM;;;AAGN,gBAAA,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACzD,gBAAA,IAAI,UAAU,CAAC,KAAK,EAAE;oBACpB,OAAO;AACL,wBAAA,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,UAAU,CAAC,MAAM;AACzB,wBAAA,UAAU,EAAE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU;AACzC,wBAAA,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,QAAQ;qBACtC;;AAEH,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;;;;AAKzE,IAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;AACrE;AAEA;;;;;;;;;;AAUG;AACG,SAAU,UAAU,CAAC,GAAW,EAAA;IAOpC,IAAI,CAAC,UAAU,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;QAC1C,OAAO;AACL,YAAA,SAAS,EAAE,KAAK;AAChB,YAAA,MAAM,EAAE,GAAG;AACX,YAAA,SAAS,EAAE,CAAC;YACZ,KAAK,EAAE,CAAC,UAAU,GAAG,yBAAyB,GAAG,SAAS;SAC3D;;;AAIH,IAAA,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE;AACrB,QAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE;;AAG3E,IAAA,IAAI;AACF,QAAA,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC;AAErE,QAAA,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE;YACrB,OAAO;AACL,gBAAA,SAAS,EAAE,KAAK;AAChB,gBAAA,MAAM,EAAE,GAAG;AACX,gBAAA,SAAS,EAAE,CAAC;AACZ,gBAAA,KAAK,EAAE,0BAA0B;aAClC;;;;AAKH,QAAA,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE;YAC7B,OAAO;AACL,gBAAA,SAAS,EAAE,KAAK;AAChB,gBAAA,MAAM,EAAE,GAAG;AACX,gBAAA,SAAS,EAAE,CAAC;AACZ,gBAAA,KAAK,EACH,iEAAiE;aACpE;;AAGH,QAAA,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;;QAGrC,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC;QAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC;;AAG7C,QAAA,MAAM,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS;QAEtD,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAC1B,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CACtD;YACD,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE;;aACpD;;YAEL,OAAO;AACL,gBAAA,SAAS,EAAE,KAAK;AAChB,gBAAA,MAAM,EAAE,GAAG;AACX,gBAAA,SAAS,EAAE,CAAC;gBACZ,KAAK,EAAE,4BAA4B,UAAU,CAAC,MAAM,CAAO,IAAA,EAAA,GAAG,CAAC,MAAM,CAAG,CAAA,CAAA;aACzE;;;IAEH,OAAO,GAAG,EAAE;;AAEZ,QAAA,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,GAAG,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC;QACjE,OAAO;AACL,YAAA,SAAS,EAAE,KAAK;AAChB,YAAA,MAAM,EAAE,GAAG;AACX,YAAA,SAAS,EAAE,CAAC;YACZ,KAAK,EAAE,CAA0B,uBAAA,EAAA,QAAQ,CAAE,CAAA;SAC5C;;AAEL;AAEA;;AAEG;SACa,eAAe,GAAA;IAC7B,OAAO,UAAU,KAAK,IAAI;AAC5B;AA0CA;;;;;;;AAOG;AACH,SAAS,gBAAgB,CAAC,MAAe,EAAA;AACvC,IAAA,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;AAAE,QAAA,OAAO,KAAK;IAEvD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACtC,IAAA,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM;;IAGlC,IAAI,iBAAiB,GAAG,CAAC;IAEzB,SAAS,gBAAgB,CAAC,GAAY,EAAA;AACpC,QAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;;AAE3B,YAAA,iBAAiB,IAAI,GAAG,CAAC,MAAM;;AAC1B,aAAA,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;AAC7B,YAAA,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC;;AACxB,aAAA,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YACzC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;;;IAIhD,gBAAgB,CAAC,MAAM,CAAC;;AAGxB,IAAA,MAAM,SAAS,GAAG,iBAAiB,GAAG,WAAW;;;IAIjD,OAAO,SAAS,GAAG,GAAG;AACxB;AAEA;;;;;;;;;;;;AAYG;SACa,iBAAiB,CAC/B,OAAe,EACf,UAAoC,EAAE,EAAA;AAEtC,IAAA,MAAM,EACJ,SAAS,GAAG,MAAM,EAClB,UAAU,GAAG,IAAI,EACjB,cAAc,GAAG,IAAI,EACrB,mBAAmB,GAAG,EAAE;AACzB,MAAA,GAAG,OAAO;AAEX,IAAA,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM;IACrC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAEpD,IAAI,MAAM,GAAG,OAAO;IACpB,IAAI,aAAa,GAAG,KAAK;IACzB,IAAI,WAAW,GAAG,KAAK;IACvB,IAAI,SAAS,GAAG,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK;AACrB,IAAA,IAAI,SAA6B;;AAGjC,IAAA,IAAI,YAAY,CAAC,OAAO,CAAC,EAAE;QACzB,WAAW,GAAG,IAAI;;;SAGf,IAAI,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE;AACtD,QAAA,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;AACtC,QAAA,IAAI,UAAU,CAAC,WAAW,EAAE;YAC1B,WAAW,GAAG,IAAI;;aACb,IACL,UAAU,CAAC,SAAS;AACpB,YAAA,UAAU,CAAC,SAAS,IAAI,mBAAmB,EAC3C;AACA,YAAA,MAAM,GAAG,UAAU,CAAC,MAAM;YAC1B,aAAa,GAAG,IAAI;AACpB,YAAA,SAAS,GAAG,UAAU,CAAC,SAAS;;AAC3B,aAAA,IAAI,UAAU,CAAC,KAAK,EAAE;;AAE3B,YAAA,SAAS,GAAG,UAAU,CAAC,KAAK;;aACvB,IACL,UAAU,CAAC,SAAS;AACpB,YAAA,UAAU,CAAC,SAAS,GAAG,mBAAmB,EAC1C;;YAEA,SAAS,GAAG,kBAAkB,UAAU,CAAC,SAAS,CAAqB,kBAAA,EAAA,mBAAmB,GAAG;;AACxF,aAAA,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;;YAEhC,SAAS;AACP,gBAAA,UAAU,CAAC,KAAK,IAAI,8CAA8C;;;;AAKxE,IAAA,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE;QAC7B,IAAI,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC;;AAGjD,QAAA,IAAI,aAAa,IAAI,WAAW,EAAE;;YAEhC,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;AAClD,YAAA,IAAI,WAAW,GAAG,SAAS,GAAG,GAAG,EAAE;gBACjC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,WAAW,CAAC;;;aAElD;;YAEL,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;YACvD,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;YACpD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,aAAa,CAAC;AAC5D,YAAA,IAAI,UAAU,GAAG,SAAS,GAAG,GAAG,EAAE;gBAChC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC;;;;QAK5D,MAAM,cAAc,GAAG;AACrB,cAAE,CAAA,SAAA,EAAY,cAAc,CAAC,cAAc,EAAE,CAAA,cAAA,EAAiB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA,QAAA,EAAW,SAAS,CAAY,UAAA,CAAA;AAC1H,gBAAA,CAAA,cAAA,EAAiB,SAAS,CAAC,cAAc,EAAE,CAAc,YAAA;AAC3D,cAAE,CAAA,oBAAA,EAAuB,cAAc,CAAC,cAAc,EAAE,CAAiB,cAAA,EAAA,cAAc,CAAC,cAAc,EAAE,CAAA,SAAA,CAAW;QAErH,MAAM;YACJ,YAAY;gBACZ,CAA0B,uBAAA,EAAA,cAAc,4DAA4D;QACtG,SAAS,GAAG,IAAI;;AAGlB,IAAA,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM;IACjC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IAE9C,OAAO;AACL,QAAA,OAAO,EAAE,MAAM;QACf,aAAa;QACb,SAAS;QACT,SAAS;QACT,WAAW;QACX,cAAc;QACd,cAAc;QACd,WAAW;QACX,WAAW;QACX,SAAS;KACV;AACH;;;;"}
+{"version":3,"file":"toonFormat.mjs","sources":["../../../src/utils/toonFormat.ts"],"sourcesContent":["/**\n * TOON Format Utility\n *\n * Provides JSON to TOON conversion for token-efficient encoding.\n * TOON format can reduce token count by 30-60% for JSON data.\n *\n * Uses the official @toon-format/toon library when available.\n */\n\n// Dynamic import holder for the TOON library (ESM)\nlet toonEncode: ((data: unknown) => string) | null = null;\nlet toonLoadAttempted = false;\nlet toonLoadPromise: Promise<boolean> | null = null;\n\n/**\n * Lazily loads the TOON library\n * Tries both ESM dynamic import and CommonJS require for compatibility\n */\nasync function loadToonLibrary(): Promise<boolean> {\n  if (toonLoadAttempted) return toonEncode !== null;\n  toonLoadAttempted = true;\n\n  // Try ESM dynamic import first\n  try {\n    const mod = await import('@toon-format/toon');\n    toonEncode = mod.encode;\n    return true;\n  } catch {\n    // ESM import failed, try CommonJS require\n    try {\n      // eslint-disable-next-line @typescript-eslint/no-require-imports\n      const mod = require('@toon-format/toon');\n      toonEncode = mod.encode;\n      return true;\n    } catch {\n      // Library not available - that's OK, we'll return original content\n      return false;\n    }\n  }\n}\n\n/**\n * Ensures the TOON library is loaded. Call this before using jsonToToon in tests.\n */\nexport async function ensureToonLoaded(): Promise<boolean> {\n  if (!toonLoadPromise) {\n    toonLoadPromise = loadToonLibrary();\n  }\n  return toonLoadPromise;\n}\n\n// Start loading immediately\nensureToonLoaded();\n\n/**\n * Check if a string appears to be in TOON format.\n * TOON format characteristics:\n * - Uses key: value syntax (like YAML)\n * - Uses array notation like [3] or {fields}\n * - Does NOT start with { or [\n * - Often has patterns like \"name[N]:\" or \"name{fields}:\"\n *\n * @param str - String to check\n * @returns true if string appears to be TOON format\n */\nexport function isToonFormat(str: string): boolean {\n  if (typeof str !== 'string' || str.length === 0) return false;\n\n  const trimmed = str.trim();\n\n  // TOON doesn't start with JSON brackets\n  if (trimmed.startsWith('{') || trimmed.startsWith('[')) return false;\n\n  // Check for TOON-specific patterns:\n  // 1. Key-value with colon (but not URL-like patterns)\n  // 2. Array notation like \"items[3]:\" or \"data[10]:\"\n  // 3. Object schema notation like \"items{id,name}:\"\n  const toonPatterns = [\n    /^\\w+:$/m, // Simple key: at start of line\n    /^\\w+\\[\\d+\\]:/m, // Array notation: items[3]:\n    /^\\w+\\{[^}]+\\}:/m, // Schema notation: items{id,name}:\n    /^\\w+\\[\\d+\\]\\{[^}]+\\}:/m, // Combined: items[3]{id,name}:\n  ];\n\n  // Must match at least one TOON pattern\n  const hasToonPattern = toonPatterns.some((pattern) => pattern.test(trimmed));\n\n  // Additional check: TOON typically has multiple lines with consistent indentation\n  const lines = trimmed.split('\\n').filter((l) => l.trim());\n  const hasMultipleKeyValueLines =\n    lines.filter((l) => /^\\s*\\w+.*:/.test(l)).length >= 2;\n\n  return hasToonPattern || hasMultipleKeyValueLines;\n}\n\n/**\n * Extract the first valid JSON object or array from a string.\n * Handles pure JSON or JSON embedded in text (e.g., tool responses).\n *\n * @param str - The string to extract JSON from\n * @returns Object with found status, parsed JSON, and indices\n */\nexport function extractFirstJson(str: string): {\n  found: boolean;\n  parsed: unknown;\n  startIndex: number;\n  endIndex: number;\n} {\n  if (typeof str !== 'string' || str.length === 0) {\n    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n  }\n\n  // Find the first { or [ character\n  const objStart = str.indexOf('{');\n  const arrStart = str.indexOf('[');\n\n  if (objStart === -1 && arrStart === -1) {\n    return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n  }\n\n  let startIndex: number;\n  if (objStart === -1) {\n    startIndex = arrStart;\n  } else if (arrStart === -1) {\n    startIndex = objStart;\n  } else {\n    startIndex = Math.min(objStart, arrStart);\n  }\n\n  // Find the matching closing bracket using a stack-based approach\n  // Properly handle strings to avoid counting brackets inside strings\n  let depth = 0;\n  let inString = false;\n  let escapeNext = false;\n\n  for (let i = startIndex; i < str.length; i++) {\n    const char = str[i];\n\n    if (escapeNext) {\n      escapeNext = false;\n      continue;\n    }\n\n    if (char === '\\\\' && inString) {\n      escapeNext = true;\n      continue;\n    }\n\n    if (char === '\"') {\n      inString = !inString;\n      continue;\n    }\n\n    if (inString) continue;\n\n    if (char === '{' || char === '[') depth++;\n    if (char === '}' || char === ']') depth--;\n\n    if (depth === 0) {\n      // Found matching bracket, try to parse\n      const jsonStr = str.substring(startIndex, i + 1);\n      try {\n        const parsed = JSON.parse(jsonStr);\n        return { found: true, parsed, startIndex, endIndex: i };\n      } catch {\n        // Not valid JSON at this bracket level, continue searching\n        // Reset and look for the next JSON start after this position\n        const nextResult = extractFirstJson(str.substring(i + 1));\n        if (nextResult.found) {\n          return {\n            found: true,\n            parsed: nextResult.parsed,\n            startIndex: i + 1 + nextResult.startIndex,\n            endIndex: i + 1 + nextResult.endIndex,\n          };\n        }\n        return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n      }\n    }\n  }\n\n  return { found: false, parsed: null, startIndex: -1, endIndex: -1 };\n}\n\n/**\n * Convert JSON content to TOON format for token efficiency.\n * Extracts JSON from string if embedded, converts to TOON.\n * Returns original string if:\n * - Already in TOON format\n * - Not JSON\n * - TOON conversion fails or is larger\n *\n * @param str - The string containing JSON to convert\n * @returns Object with conversion status, result, and reduction percentage\n */\nexport function jsonToToon(str: string): {\n  converted: boolean;\n  result: string;\n  reduction: number;\n  alreadyToon?: boolean;\n  error?: string;\n} {\n  if (!toonEncode || typeof str !== 'string') {\n    return {\n      converted: false,\n      result: str,\n      reduction: 0,\n      error: !toonEncode ? 'TOON library not loaded' : undefined,\n    };\n  }\n\n  // Check if already in TOON format - skip conversion\n  if (isToonFormat(str)) {\n    return { converted: false, result: str, reduction: 0, alreadyToon: true };\n  }\n\n  try {\n    const { found, parsed, startIndex, endIndex } = extractFirstJson(str);\n\n    if (!found || !parsed) {\n      return {\n        converted: false,\n        result: str,\n        reduction: 0,\n        error: 'No JSON found in content',\n      };\n    }\n\n    // Check if this JSON structure would benefit from TOON\n    // Text-heavy content (emails, documents) won't compress well\n    if (!isToonBeneficial(parsed)) {\n      return {\n        converted: false,\n        result: str,\n        reduction: 0,\n        error:\n          'Content is text-heavy (>70% string values), TOON not beneficial',\n      };\n    }\n\n    const toonResult = toonEncode(parsed);\n\n    // Preserve text before and after the JSON block\n    const textBefore = str.substring(0, startIndex);\n    const textAfter = str.substring(endIndex + 1);\n\n    // Build the full result: textBefore + TOON + textAfter\n    const fullResult = textBefore + toonResult + textAfter;\n\n    if (fullResult.length < str.length) {\n      const reduction = Math.round(\n        ((str.length - fullResult.length) / str.length) * 100\n      );\n      return { converted: true, result: fullResult, reduction };\n    } else {\n      // TOON output was larger or same size - not beneficial\n      return {\n        converted: false,\n        result: str,\n        reduction: 0,\n        error: `TOON output not smaller (${fullResult.length} >= ${str.length})`,\n      };\n    }\n  } catch (err) {\n    // TOON encoding or extraction failed - log error for debugging\n    const errorMsg = err instanceof Error ? err.message : String(err);\n    return {\n      converted: false,\n      result: str,\n      reduction: 0,\n      error: `TOON conversion error: ${errorMsg}`,\n    };\n  }\n}\n\n/**\n * Check if TOON library is loaded and available\n */\nexport function isToonAvailable(): boolean {\n  return toonEncode !== null;\n}\n\n/**\n * Result of processing tool output\n */\nexport interface ProcessToolOutputResult {\n  /** The processed content string */\n  content: string;\n  /** Whether TOON conversion was applied */\n  toonConverted: boolean;\n  /** Whether content was truncated */\n  truncated: boolean;\n  /** Percentage reduction from TOON (0 if not converted) */\n  reduction: number;\n  /** Whether input was already in TOON format */\n  alreadyToon: boolean;\n  /** Original content length */\n  originalLength: number;\n  /** Estimated original tokens (~4 chars per token) */\n  originalTokens: number;\n  /** Final content length */\n  finalLength: number;\n  /** Estimated final tokens */\n  finalTokens: number;\n  /** Error message if TOON conversion failed (for debugging) */\n  toonError?: string;\n}\n\n/**\n * Options for processing tool output\n */\nexport interface ProcessToolOutputOptions {\n  /** Maximum output length in characters (default: 100000) */\n  maxLength?: number;\n  /** Whether to apply TOON conversion (default: true) */\n  enableToon?: boolean;\n  /** Minimum content size to attempt TOON conversion (default: 1000) */\n  minSizeForToon?: number;\n  /** Minimum reduction % to accept TOON result (default: 10) */\n  minReductionPercent?: number;\n}\n\n/**\n * Analyze JSON structure to determine if TOON conversion would be beneficial.\n * Text-heavy content (emails, documents) doesn't compress well with TOON.\n * Structured data (API responses, metadata) compresses much better.\n *\n * @param parsed - Parsed JSON object\n * @returns true if TOON conversion is likely beneficial\n */\nfunction isToonBeneficial(parsed: unknown): boolean {\n  if (!parsed || typeof parsed !== 'object') return false;\n\n  const jsonStr = JSON.stringify(parsed);\n  const totalLength = jsonStr.length;\n\n  // Count characters in string values (text content)\n  let textContentLength = 0;\n\n  function countTextContent(obj: unknown): void {\n    if (typeof obj === 'string') {\n      // Count string length (this is text content that TOON can't compress)\n      textContentLength += obj.length;\n    } else if (Array.isArray(obj)) {\n      obj.forEach(countTextContent);\n    } else if (obj && typeof obj === 'object') {\n      Object.values(obj).forEach(countTextContent);\n    }\n  }\n\n  countTextContent(parsed);\n\n  // Calculate ratio of text content to total JSON\n  const textRatio = textContentLength / totalLength;\n\n  // If more than 70% is text content, TOON won't help much\n  // TOON compresses structure (field names, brackets), not text\n  return textRatio < 0.7;\n}\n\n/**\n * Process tool output: apply TOON conversion if beneficial, then truncate if needed.\n * This is the main entry point for processing any tool output (regular or MCP).\n *\n * Flow:\n * 1. Check if already TOON format → skip conversion\n * 2. Try TOON conversion if content is JSON and large enough\n * 3. Truncate if still exceeds maxLength (with smart break points)\n *\n * @param content - The tool output content\n * @param options - Processing options\n * @returns Processed content and metadata\n */\nexport function processToolOutput(\n  content: string,\n  options: ProcessToolOutputOptions = {}\n): ProcessToolOutputResult {\n  const {\n    maxLength = 100000,\n    enableToon = true,\n    minSizeForToon = 1000,\n    minReductionPercent = 10, // Increased from 5% - only apply TOON when clearly beneficial\n  } = options;\n\n  const originalLength = content.length;\n  const originalTokens = Math.ceil(originalLength / 4);\n\n  let result = content;\n  let toonConverted = false;\n  let alreadyToon = false;\n  let reduction = 0;\n  let truncated = false;\n  let toonError: string | undefined;\n\n  // Step 1: Check if already TOON format\n  if (isToonFormat(content)) {\n    alreadyToon = true;\n  }\n  // Step 2: Apply TOON conversion if enabled and content is large enough\n  else if (enableToon && content.length > minSizeForToon) {\n    const toonResult = jsonToToon(content);\n    if (toonResult.alreadyToon) {\n      alreadyToon = true;\n    } else if (\n      toonResult.converted &&\n      toonResult.reduction >= minReductionPercent\n    ) {\n      result = toonResult.result;\n      toonConverted = true;\n      reduction = toonResult.reduction;\n    } else if (toonResult.error) {\n      // Track error for debugging\n      toonError = toonResult.error;\n    } else if (\n      toonResult.converted &&\n      toonResult.reduction < minReductionPercent\n    ) {\n      // TOON converted but reduction was too small to be worth it\n      toonError = `TOON reduction ${toonResult.reduction}% below threshold ${minReductionPercent}%`;\n    } else if (!toonResult.converted) {\n      // Conversion failed without explicit error - investigate\n      toonError =\n        toonResult.error || 'TOON conversion returned false with no error';\n    }\n  }\n\n  // Step 3: Truncate if still too long (with smart break points)\n  if (result.length > maxLength) {\n    let truncatedStr = result.substring(0, maxLength);\n\n    // Try to find a clean break point\n    if (toonConverted || alreadyToon) {\n      // For TOON format, break at newline for cleaner output\n      const lastNewline = truncatedStr.lastIndexOf('\\n');\n      if (lastNewline > maxLength * 0.7) {\n        truncatedStr = truncatedStr.substring(0, lastNewline);\n      }\n    } else {\n      // For JSON, try to find a clean JSON break point\n      const lastCompleteItem = truncatedStr.lastIndexOf('},');\n      const lastArrayItem = truncatedStr.lastIndexOf('],');\n      const breakPoint = Math.max(lastCompleteItem, lastArrayItem);\n      if (breakPoint > maxLength * 0.5) {\n        truncatedStr = truncatedStr.substring(0, breakPoint + 1);\n      }\n    }\n\n    // Build truncation message\n    const truncationInfo = toonConverted\n      ? `Original ${originalLength.toLocaleString()} chars → TOON ${result.length.toLocaleString()} chars (${reduction}% saved). ` +\n        `Still exceeds ${maxLength.toLocaleString()} char limit.`\n      : `Original output was ${originalLength.toLocaleString()} characters (~${originalTokens.toLocaleString()} tokens).`;\n\n    result =\n      truncatedStr +\n      `\\n\\n[OUTPUT_TRUNCATED: ${truncationInfo} Please use more specific queries or smaller date ranges.]`;\n    truncated = true;\n  }\n\n  const finalLength = result.length;\n  const finalTokens = Math.ceil(finalLength / 4);\n\n  return {\n    content: result,\n    toonConverted,\n    truncated,\n    reduction,\n    alreadyToon,\n    originalLength,\n    originalTokens,\n    finalLength,\n    finalTokens,\n    toonError,\n  };\n}\n"],"names":[],"mappings":"AAAA;;;;;;;AAOG;AAEH;AACA,IAAI,UAAU,GAAuC,IAAI;AACzD,IAAI,iBAAiB,GAAG,KAAK;AAC7B,IAAI,eAAe,GAA4B,IAAI;AAEnD;;;AAGG;AACH,eAAe,eAAe,GAAA;AAC5B,IAAA,IAAI,iBAAiB;QAAE,OAAO,UAAU,KAAK,IAAI;IACjD,iBAAiB,GAAG,IAAI;;AAGxB,IAAA,IAAI;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,OAAO,mBAAmB,CAAC;AAC7C,QAAA,UAAU,GAAG,GAAG,CAAC,MAAM;AACvB,QAAA,OAAO,IAAI;IACb;AAAE,IAAA,MAAM;;AAEN,QAAA,IAAI;;AAEF,YAAA,MAAM,GAAG,GAAG,OAAO,CAAC,mBAAmB,CAAC;AACxC,YAAA,UAAU,GAAG,GAAG,CAAC,MAAM;AACvB,YAAA,OAAO,IAAI;QACb;AAAE,QAAA,MAAM;;AAEN,YAAA,OAAO,KAAK;QACd;IACF;AACF;AAEA;;AAEG;AACI,eAAe,gBAAgB,GAAA;IACpC,IAAI,CAAC,eAAe,EAAE;QACpB,eAAe,GAAG,eAAe,EAAE;IACrC;AACA,IAAA,OAAO,eAAe;AACxB;AAEA;AACA,gBAAgB,EAAE;AAElB;;;;;;;;;;AAUG;AACG,SAAU,YAAY,CAAC,GAAW,EAAA;IACtC,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;AAAE,QAAA,OAAO,KAAK;AAE7D,IAAA,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE;;AAG1B,IAAA,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;AAAE,QAAA,OAAO,KAAK;;;;;AAMpE,IAAA,MAAM,YAAY,GAAG;AACnB,QAAA,SAAS;AACT,QAAA,eAAe;AACf,QAAA,iBAAiB;AACjB,QAAA,wBAAwB;KACzB;;AAGD,IAAA,MAAM,cAAc,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;IAG5E,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,wBAAwB,GAC5B,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC;IAEvD,OAAO,cAAc,IAAI,wBAAwB;AACnD;AAEA;;;;;;AAMG;AACG,SAAU,gBAAgB,CAAC,GAAW,EAAA;IAM1C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;AAC/C,QAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;IACrE;;IAGA,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC;IAEjC,IAAI,QAAQ,KAAK,EAAE,IAAI,QAAQ,KAAK,EAAE,EAAE;AACtC,QAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;IACrE;AAEA,IAAA,IAAI,UAAkB;AACtB,IAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;QACnB,UAAU,GAAG,QAAQ;IACvB;AAAO,SAAA,IAAI,QAAQ,KAAK,EAAE,EAAE;QAC1B,UAAU,GAAG,QAAQ;IACvB;SAAO;QACL,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC;IAC3C;;;IAIA,IAAI,KAAK,GAAG,CAAC;IACb,IAAI,QAAQ,GAAG,KAAK;IACpB,IAAI,UAAU,GAAG,KAAK;AAEtB,IAAA,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;AAC5C,QAAA,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAC;QAEnB,IAAI,UAAU,EAAE;YACd,UAAU,GAAG,KAAK;YAClB;QACF;AAEA,QAAA,IAAI,IAAI,KAAK,IAAI,IAAI,QAAQ,EAAE;YAC7B,UAAU,GAAG,IAAI;YACjB;QACF;AAEA,QAAA,IAAI,IAAI,KAAK,GAAG,EAAE;YAChB,QAAQ,GAAG,CAAC,QAAQ;YACpB;QACF;AAEA,QAAA,IAAI,QAAQ;YAAE;AAEd,QAAA,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;AAAE,YAAA,KAAK,EAAE;AACzC,QAAA,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,KAAK,GAAG;AAAE,YAAA,KAAK,EAAE;AAEzC,QAAA,IAAI,KAAK,KAAK,CAAC,EAAE;;AAEf,YAAA,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,CAAC;AAChD,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC;AAClC,gBAAA,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,EAAE;YACzD;AAAE,YAAA,MAAM;;;AAGN,gBAAA,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACzD,gBAAA,IAAI,UAAU,CAAC,KAAK,EAAE;oBACpB,OAAO;AACL,wBAAA,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,UAAU,CAAC,MAAM;AACzB,wBAAA,UAAU,EAAE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,UAAU;AACzC,wBAAA,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,QAAQ;qBACtC;gBACH;AACA,gBAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;YACrE;QACF;IACF;AAEA,IAAA,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;AACrE;AAEA;;;;;;;;;;AAUG;AACG,SAAU,UAAU,CAAC,GAAW,EAAA;IAOpC,IAAI,CAAC,UAAU,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;QAC1C,OAAO;AACL,YAAA,SAAS,EAAE,KAAK;AAChB,YAAA,MAAM,EAAE,GAAG;AACX,YAAA,SAAS,EAAE,CAAC;YACZ,KAAK,EAAE,CAAC,UAAU,GAAG,yBAAyB,GAAG,SAAS;SAC3D;IACH;;AAGA,IAAA,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE;AACrB,QAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE;IAC3E;AAEA,IAAA,IAAI;AACF,QAAA,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,GAAG,gBAAgB,CAAC,GAAG,CAAC;AAErE,QAAA,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE;YACrB,OAAO;AACL,gBAAA,SAAS,EAAE,KAAK;AAChB,gBAAA,MAAM,EAAE,GAAG;AACX,gBAAA,SAAS,EAAE,CAAC;AACZ,gBAAA,KAAK,EAAE,0BAA0B;aAClC;QACH;;;AAIA,QAAA,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE;YAC7B,OAAO;AACL,gBAAA,SAAS,EAAE,KAAK;AAChB,gBAAA,MAAM,EAAE,GAAG;AACX,gBAAA,SAAS,EAAE,CAAC;AACZ,gBAAA,KAAK,EACH,iEAAiE;aACpE;QACH;AAEA,QAAA,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,CAAC;;QAGrC,MAAM,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC;QAC/C,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,CAAC,QAAQ,GAAG,CAAC,CAAC;;AAG7C,QAAA,MAAM,UAAU,GAAG,UAAU,GAAG,UAAU,GAAG,SAAS;QAEtD,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE;YAClC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAC1B,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,CACtD;YACD,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE;QAC3D;aAAO;;YAEL,OAAO;AACL,gBAAA,SAAS,EAAE,KAAK;AAChB,gBAAA,MAAM,EAAE,GAAG;AACX,gBAAA,SAAS,EAAE,CAAC;gBACZ,KAAK,EAAE,4BAA4B,UAAU,CAAC,MAAM,CAAA,IAAA,EAAO,GAAG,CAAC,MAAM,CAAA,CAAA,CAAG;aACzE;QACH;IACF;IAAE,OAAO,GAAG,EAAE;;AAEZ,QAAA,MAAM,QAAQ,GAAG,GAAG,YAAY,KAAK,GAAG,GAAG,CAAC,OAAO,GAAG,MAAM,CAAC,GAAG,CAAC;QACjE,OAAO;AACL,YAAA,SAAS,EAAE,KAAK;AAChB,YAAA,MAAM,EAAE,GAAG;AACX,YAAA,SAAS,EAAE,CAAC;YACZ,KAAK,EAAE,CAAA,uBAAA,EAA0B,QAAQ,CAAA,CAAE;SAC5C;IACH;AACF;AAEA;;AAEG;SACa,eAAe,GAAA;IAC7B,OAAO,UAAU,KAAK,IAAI;AAC5B;AA0CA;;;;;;;AAOG;AACH,SAAS,gBAAgB,CAAC,MAAe,EAAA;AACvC,IAAA,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;AAAE,QAAA,OAAO,KAAK;IAEvD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;AACtC,IAAA,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM;;IAGlC,IAAI,iBAAiB,GAAG,CAAC;IAEzB,SAAS,gBAAgB,CAAC,GAAY,EAAA;AACpC,QAAA,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;;AAE3B,YAAA,iBAAiB,IAAI,GAAG,CAAC,MAAM;QACjC;AAAO,aAAA,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;AAC7B,YAAA,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC;QAC/B;AAAO,aAAA,IAAI,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE;YACzC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,gBAAgB,CAAC;QAC9C;IACF;IAEA,gBAAgB,CAAC,MAAM,CAAC;;AAGxB,IAAA,MAAM,SAAS,GAAG,iBAAiB,GAAG,WAAW;;;IAIjD,OAAO,SAAS,GAAG,GAAG;AACxB;AAEA;;;;;;;;;;;;AAYG;SACa,iBAAiB,CAC/B,OAAe,EACf,UAAoC,EAAE,EAAA;AAEtC,IAAA,MAAM,EACJ,SAAS,GAAG,MAAM,EAClB,UAAU,GAAG,IAAI,EACjB,cAAc,GAAG,IAAI,EACrB,mBAAmB,GAAG,EAAE;AACzB,MAAA,GAAG,OAAO;AAEX,IAAA,MAAM,cAAc,GAAG,OAAO,CAAC,MAAM;IACrC,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;IAEpD,IAAI,MAAM,GAAG,OAAO;IACpB,IAAI,aAAa,GAAG,KAAK;IACzB,IAAI,WAAW,GAAG,KAAK;IACvB,IAAI,SAAS,GAAG,CAAC;IACjB,IAAI,SAAS,GAAG,KAAK;AACrB,IAAA,IAAI,SAA6B;;AAGjC,IAAA,IAAI,YAAY,CAAC,OAAO,CAAC,EAAE;QACzB,WAAW,GAAG,IAAI;IACpB;;SAEK,IAAI,UAAU,IAAI,OAAO,CAAC,MAAM,GAAG,cAAc,EAAE;AACtD,QAAA,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC;AACtC,QAAA,IAAI,UAAU,CAAC,WAAW,EAAE;YAC1B,WAAW,GAAG,IAAI;QACpB;aAAO,IACL,UAAU,CAAC,SAAS;AACpB,YAAA,UAAU,CAAC,SAAS,IAAI,mBAAmB,EAC3C;AACA,YAAA,MAAM,GAAG,UAAU,CAAC,MAAM;YAC1B,aAAa,GAAG,IAAI;AACpB,YAAA,SAAS,GAAG,UAAU,CAAC,SAAS;QAClC;AAAO,aAAA,IAAI,UAAU,CAAC,KAAK,EAAE;;AAE3B,YAAA,SAAS,GAAG,UAAU,CAAC,KAAK;QAC9B;aAAO,IACL,UAAU,CAAC,SAAS;AACpB,YAAA,UAAU,CAAC,SAAS,GAAG,mBAAmB,EAC1C;;YAEA,SAAS,GAAG,kBAAkB,UAAU,CAAC,SAAS,CAAA,kBAAA,EAAqB,mBAAmB,GAAG;QAC/F;AAAO,aAAA,IAAI,CAAC,UAAU,CAAC,SAAS,EAAE;;YAEhC,SAAS;AACP,gBAAA,UAAU,CAAC,KAAK,IAAI,8CAA8C;QACtE;IACF;;AAGA,IAAA,IAAI,MAAM,CAAC,MAAM,GAAG,SAAS,EAAE;QAC7B,IAAI,YAAY,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,SAAS,CAAC;;AAGjD,QAAA,IAAI,aAAa,IAAI,WAAW,EAAE;;YAEhC,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;AAClD,YAAA,IAAI,WAAW,GAAG,SAAS,GAAG,GAAG,EAAE;gBACjC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,WAAW,CAAC;YACvD;QACF;aAAO;;YAEL,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;YACvD,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,CAAC;YACpD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,aAAa,CAAC;AAC5D,YAAA,IAAI,UAAU,GAAG,SAAS,GAAG,GAAG,EAAE;gBAChC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC;YAC1D;QACF;;QAGA,MAAM,cAAc,GAAG;AACrB,cAAE,CAAA,SAAA,EAAY,cAAc,CAAC,cAAc,EAAE,CAAA,cAAA,EAAiB,MAAM,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA,QAAA,EAAW,SAAS,CAAA,UAAA,CAAY;AAC1H,gBAAA,CAAA,cAAA,EAAiB,SAAS,CAAC,cAAc,EAAE,CAAA,YAAA;AAC7C,cAAE,CAAA,oBAAA,EAAuB,cAAc,CAAC,cAAc,EAAE,CAAA,cAAA,EAAiB,cAAc,CAAC,cAAc,EAAE,CAAA,SAAA,CAAW;QAErH,MAAM;YACJ,YAAY;gBACZ,CAAA,uBAAA,EAA0B,cAAc,4DAA4D;QACtG,SAAS,GAAG,IAAI;IAClB;AAEA,IAAA,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM;IACjC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IAE9C,OAAO;AACL,QAAA,OAAO,EAAE,MAAM;QACf,aAAa;QACb,SAAS;QACT,SAAS;QACT,WAAW;QACX,cAAc;QACd,cAAc;QACd,WAAW;QACX,WAAW;QACX,SAAS;KACV;AACH;;;;"}

package/dist/types/agents/AgentContext.d.ts

@@ -24,6 +24,8 @@ export declare class AgentContext {
     clientOptions?: t.ClientOptions;
     /** Token count map indexed by message position */
     indexTokenCountMap: Record<string, number | undefined>;
+    /** Canonical pre-run token map used to restore token accounting on reset */
+    baseIndexTokenCountMap: Record<string, number>;
     /** Maximum context tokens for this agent */
     maxContextTokens?: number;
     /** Current usage metadata for this agent */
@@ -40,6 +42,8 @@ export declare class AgentContext {
     lastStreamCall?: number;
     /** Tools available to this agent */
     tools?: t.GraphTools;
+    /** Graph-managed tools (e.g., handoff tools created by MultiAgentGraph) that bypass event-driven dispatch */
+    graphTools?: t.GraphTools;
     /** Tool map for this agent */
     toolMap?: t.ToolMap;
     /**
@@ -70,6 +74,8 @@ export declare class AgentContext {
     lastToken?: string;
     /** Token type switch state */
     tokenTypeSwitch?: 'reasoning' | 'content';
+    /** Tracks how many reasoning→text transitions have occurred (ensures unique post-reasoning step keys) */
+    reasoningTransitionCount: number;
     /** Current token type being processed */
     currentTokenType: ContentTypes.TEXT | ContentTypes.THINK | 'think_and_text';
     /** Whether tools should end the workflow */
@@ -106,12 +112,11 @@ export declare class AgentContext {
      * instead of streaming text.
      */
     structuredOutput?: t.StructuredOutputConfig;
-    /**
-     * Optional callback for summarizing messages that were pruned from context.
-     * When provided, discarded messages are summarized instead of silently dropped.
-     */
-    summarizeCallback?: (messagesToRefine: BaseMessage[]) => Promise<string | undefined>;
-    constructor({ agentId, name, description, provider, clientOptions, maxContextTokens, streamBuffer, tokenCounter, tools, toolMap, toolRegistry, toolDefinitions, instructions, additionalInstructions, dynamicContext, reasoningKey, toolEnd, instructionTokens, useLegacyContent, structuredOutput, discoveredTools, }: {
+    /** Optional callback to summarize discarded messages during context pruning */
+    summarizeCallback?: (messages: BaseMessage[]) => Promise<string | undefined>;
+    /** Pre-existing summary loaded from persistent storage, injected into context on new turns */
+    persistedSummary?: string;
+    constructor({ agentId, name, description, provider, clientOptions, maxContextTokens, streamBuffer, tokenCounter, tools, toolMap, toolRegistry, toolDefinitions, instructions, additionalInstructions, dynamicContext, reasoningKey, toolEnd, instructionTokens, useLegacyContent, structuredOutput, discoveredTools, summarizeCallback, persistedSummary, }: {
         agentId: string;
         name?: string;
         description?: string;
@@ -133,6 +138,8 @@ export declare class AgentContext {
         useLegacyContent?: boolean;
         structuredOutput?: t.StructuredOutputConfig;
         discoveredTools?: string[];
+        summarizeCallback?: (messages: BaseMessage[]) => Promise<string | undefined>;
+        persistedSummary?: string;
     });
     /**
      * Checks if structured output mode is enabled for this agent.
@@ -289,7 +296,7 @@ export declare class AgentContext {
      * @returns Array of tools to bind to model
      */
     getToolsForBinding(): t.GraphTools | undefined;
-    /** Creates schema-only tools from toolDefinitions for event-driven mode */
+    /** Creates schema-only tools from toolDefinitions for event-driven mode, merged with native tools */
     private getEventDrivenToolsForBinding;
     /** Filters tool instances for binding based on registry config */
     private filterToolsForBinding;

package/dist/types/common/constants.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Minimum thinking budget allowed by the Anthropic API.
+ * Extended thinking requires at least 1024 budget_tokens.
+ * @see https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking
+ */
+export declare const MIN_THINKING_BUDGET = 1024;
+/**
+ * Reduced thinking budget for subsequent ReAct iterations (tool-result turns).
+ *
+ * In a ReAct agent loop, the first LLM call processes the user's query and
+ * may need deep reasoning. Subsequent iterations (after tool results return)
+ * typically only need to decide "call next tool" or "generate final response"
+ * — 1024 tokens is sufficient for this routing logic.
+ *
+ * This reduces wall-clock time per iteration from ~20-30s to ~5-10s,
+ * compounding across multi-tool conversations (e.g., 10 tool calls).
+ */
+export declare const TOOL_TURN_THINKING_BUDGET = 1024;

package/dist/types/common/enum.d.ts

@@ -23,6 +23,8 @@ export declare enum GraphEvents {
     ON_STRUCTURED_OUTPUT = "on_structured_output",
     /** [Custom] Request to execute tools - dispatched by ToolNode, handled by host */
     ON_TOOL_EXECUTE = "on_tool_execute",
+    /** [Custom] Context pressure event for monitoring compaction triggers */
+    ON_CONTEXT_PRESSURE = "on_context_pressure",
     /** [Custom] Tool requires human approval before execution - dispatched by ToolNode HITL */
     ON_TOOL_APPROVAL_REQUIRED = "on_tool_approval_required",
     /** Custom event, emitted by system */
@@ -72,6 +74,12 @@ export declare enum Providers {
     XAI = "xai",
     MOONSHOT = "moonshot"
 }
+export declare enum EdgeType {
+    /** Creates handoff tools for dynamic agent routing (default for single-to-single edges) */
+    HANDOFF = "handoff",
+    /** Creates direct edges for automatic sequential/parallel transitions */
+    DIRECT = "direct"
+}
 export declare enum GraphNodeKeys {
     TOOLS = "tools=",
     AGENT = "agent=",
@@ -128,6 +136,8 @@ export declare enum Constants {
     WEB_SEARCH = "web_search",
     CONTENT_AND_ARTIFACT = "content_and_artifact",
     LC_TRANSFER_TO_ = "lc_transfer_to_",
+    /** Tool name for the AskUser structured question tool */
+    ASK_USER = "ask_user",
     /** Delimiter for MCP tools: toolName_mcp_serverName */
     MCP_DELIMITER = "_mcp_"
 }
@@ -140,6 +150,24 @@ export declare enum EnvVar {
     CODE_API_KEY = "CODE_EXECUTOR_API_KEY",
     CODE_BASEURL = "CODE_EXECUTOR_BASEURL"
 }
+/**
+ * Normalized LLM finish/stop reasons across providers.
+ * Used by toolCallContinuation to detect max_tokens truncation.
+ */
+export declare enum FinishReasons {
+    /** Anthropic / Bedrock stop reason for token limit */
+    MAX_TOKENS = "max_tokens",
+    /** OpenAI / Azure finish reason for token limit */
+    LENGTH = "length",
+    /** Normal completion */
+    STOP = "stop",
+    /** Anthropic / Bedrock stop reason for normal completion */
+    END_TURN = "end_turn",
+    /** Model chose to call tools */
+    TOOL_USE = "tool_use",
+    /** OpenAI finish reason for tool calls */
+    TOOL_CALLS = "tool_calls"
+}
 /**
  * Message type identifiers used by LangChain's BaseMessage.getType().
  * Use these constants instead of instanceof checks to avoid module mismatch issues

package/dist/types/common/index.d.ts

@@ -1 +1,2 @@
 export * from './enum';
+export * from './constants';

package/dist/types/events.d.ts

@@ -1,4 +1,4 @@
-import type { UsageMetadata, BaseMessageFields } from '@langchain/core/messages';
+import type { BaseMessageFields, UsageMetadata } from '@langchain/core/messages';
 import type { MultiAgentGraph, StandardGraph } from '@/graphs';
 import type { Logger } from 'winston';
 import type * as t from '@/types';
@@ -15,8 +15,15 @@ export declare class ModelEndHandler implements t.EventHandler {
 export declare class ToolEndHandler implements t.EventHandler {
     private callback?;
     private logger?;
-    private omitOutput?;
-    constructor(callback?: t.ToolEndCallback, logger?: Logger, omitOutput?: (name?: string) => boolean);
+    constructor(callback?: t.ToolEndCallback, logger?: Logger);
+    /**
+     * Handles on_tool_end events from the for-await stream consumer.
+     *
+     * This handler is now purely a consumer callback — tool completion
+     * (ON_RUN_STEP_COMPLETED dispatch + session context storage) is handled
+     * in graph context by ToolNode directly, eliminating the race between
+     * the stream consumer and graph execution.
+     */
     handle(event: string, data: t.StreamEventData | undefined, metadata?: Record<string, unknown>, graph?: StandardGraph | MultiAgentGraph): Promise<void>;
 }
 export declare class TestLLMStreamHandler implements t.EventHandler {

package/dist/types/graphs/Graph.d.ts

@@ -8,6 +8,7 @@ import { type ContextAnalytics } from '@/utils/contextAnalytics';
 import { ToolNode as CustomToolNode } from '@/tools/ToolNode';
 import { AgentContext } from '@/agents/AgentContext';
 import { HandlerRegistry } from '@/events';
+import { StreamingToolCallBuffer } from '@/tools/StreamingToolCallBuffer';
 export declare abstract class Graph<T extends t.BaseGraphState = t.BaseGraphState, _TNodeName extends string = string> {
     abstract resetValues(): void;
     abstract initializeTools({ currentTools, currentToolMap, }: {
@@ -51,6 +52,18 @@ export declare abstract class Graph<T extends t.BaseGraphState = t.BaseGraphStat
      * Currently supports code execution session tracking (session_id, files).
      */
     sessions: t.ToolSessionMap;
+    /**
+     * Streaming tool call buffer — accumulates raw arg strings during streaming
+     * so that truncated tool call content can be recovered by the ToolNode.
+     * Fed by handleToolCallChunks, consumed by ToolNode.run when args are incomplete.
+     */
+    streamingToolCallBuffer: StreamingToolCallBuffer;
+    /**
+     * Clears heavy references to allow GC to reclaim memory held by
+     * LangGraph's internal config / AsyncLocalStorage RunTree chain.
+     * Call after a run completes and content has been extracted.
+     */
+    clearHeavyState(): void;
 }
 export declare class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     overrideModel?: t.ChatModel;
@@ -68,6 +81,30 @@ export declare class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode>
     lastFinishReason: string | undefined;
     constructor({ runId, signal, agents, tokenCounter, indexTokenCountMap, }: t.StandardGraphInput);
     resetValues(keepContent?: boolean): void;
+    clearHeavyState(): void;
+    /**
+     * Returns clientOptions with a reduced thinking budget for subsequent
+     * ReAct loop iterations (tool-result turns).
+     *
+     * **Rationale:** The first LLM call in a conversation processes the user's
+     * original query and may benefit from deep extended thinking. Subsequent
+     * iterations — where the model receives tool results and decides whether
+     * to call another tool or produce a final response — require minimal
+     * reasoning. Reducing the thinking budget from the user's configured
+     * value to TOOL_TURN_THINKING_BUDGET (1024 tokens) cuts wall-clock
+     * latency by ~15-20s per iteration, compounding across multi-tool flows.
+     *
+     * Provider handling:
+     * - **Anthropic (direct):** Reduces `thinking.budget_tokens` if > threshold
+     * - **Bedrock (Anthropic models):** Reduces `additionalModelRequestFields.thinking.budget_tokens`
+     * - **VertexAI / Google:** Reduces `thinkingConfig.thinkingBudget` if > threshold
+     * - **All others:** Returns clientOptions unchanged (no-op)
+     *
+     * @param clientOptions - The original client options from AgentContext
+     * @param provider - The LLM provider enum value
+     * @returns Shallow-cloned clientOptions with reduced thinking budget, or the original if no reduction needed
+     */
+    getAdaptiveClientOptions(clientOptions: t.ClientOptions, provider: Providers): t.ClientOptions;
     /**
      * Returns the normalized finish/stop reason from the last LLM invocation.
      * Used by callers to detect when the response was truncated due to max_tokens.

package/dist/types/index.d.ts

@@ -3,6 +3,7 @@ export * from './stream';
 export * from './splitStream';
 export * from './events';
 export * from './messages';
+export type { GuardrailTraceData, GuardrailOutcome, } from '@illuma-ai/observability-langchain';
 export * from './graphs';
 export * from './tools/Calculator';
 export * from './tools/CodeExecutor';
@@ -10,6 +11,7 @@ export * from './tools/BrowserTools';
 export * from './tools/ProgrammaticToolCalling';
 export * from './tools/ToolSearch';
 export * from './tools/ToolNode';
+export * from './tools/AskUser';
 export * from './tools/schema';
 export * from './tools/handlers';
 export * from './tools/search';
@@ -18,4 +20,6 @@ export * from './common';
 export * from './utils';
 export type * from './types';
 export { CustomOpenAIClient } from './llm/openai';
+export { ChatOpenRouter } from './llm/openrouter';
+export type { OpenRouterReasoning, OpenRouterReasoningEffort, ChatOpenRouterCallOptions, } from './llm/openrouter';
 export { getChatModelClass, llmProviders } from './llm/providers';

package/dist/types/llm/anthropic/index.d.ts

@@ -4,9 +4,12 @@ import type { BaseChatModelParams } from '@langchain/core/language_models/chat_m
 import type { BaseMessage, UsageMetadata } from '@langchain/core/messages';
 import type { CallbackManagerForLLMRun } from '@langchain/core/callbacks/manager';
 import type { AnthropicInput } from '@langchain/anthropic';
-import type { AnthropicMessageCreateParams, AnthropicStreamingMessageCreateParams } from '@/llm/anthropic/types';
+import type { AnthropicMessageCreateParams, AnthropicStreamingMessageCreateParams, AnthropicOutputConfig } from '@/llm/anthropic/types';
 export type CustomAnthropicInput = AnthropicInput & {
     _lc_stream_delay?: number;
+    outputConfig?: AnthropicOutputConfig;
+    inferenceGeo?: string;
+    contextManagement?: any;
 } & BaseChatModelParams;
 /**
  * A type representing additional parameters that can be passed to the
@@ -20,6 +23,9 @@ export declare class CustomAnthropic extends ChatAnthropicMessages {
     private tools_in_params?;
     private emitted_usage?;
     top_k: number | undefined;
+    outputConfig?: AnthropicOutputConfig;
+    inferenceGeo?: string;
+    contextManagement?: any;
     constructor(fields?: CustomAnthropicInput);
     static lc_name(): 'IllumaAnthropic';
     /**

package/dist/types/llm/anthropic/types.d.ts

@@ -32,6 +32,9 @@ export type AnthropicServerToolUseBlockParam = Anthropic.Messages.ServerToolUseB
 export type AnthropicWebSearchToolResultBlockParam = Anthropic.Messages.WebSearchToolResultBlockParam;
 export type AnthropicWebSearchResultBlockParam = Anthropic.Messages.WebSearchResultBlockParam;
 export type AnthropicSearchResultBlockParam = Anthropic.Beta.BetaSearchResultBlockParam;
-export type AnthropicContentBlock = AnthropicTextBlockParam | AnthropicImageBlockParam | AnthropicToolUseBlockParam | AnthropicToolResultBlockParam | AnthropicDocumentBlockParam | AnthropicThinkingBlockParam | AnthropicRedactedThinkingBlockParam | AnthropicServerToolUseBlockParam | AnthropicWebSearchToolResultBlockParam | AnthropicWebSearchResultBlockParam;
-export type ChatAnthropicContentBlock = AnthropicTextBlockParam | AnthropicImageBlockParam | AnthropicToolUseBlockParam | AnthropicToolResultBlockParam | AnthropicDocumentBlockParam | AnthropicThinkingBlockParam | AnthropicRedactedThinkingBlockParam | AnthropicServerToolUseBlockParam | AnthropicWebSearchToolResultBlockParam | AnthropicWebSearchResultBlockParam | AnthropicSearchResultBlockParam;
+export type AnthropicCompactionBlockParam = Anthropic.Beta.BetaCompactionBlockParam;
+export type AnthropicOutputConfig = Anthropic.Messages.OutputConfig;
+export type ChatAnthropicOutputFormat = Anthropic.Messages.JSONOutputFormat;
+export type AnthropicContentBlock = AnthropicTextBlockParam | AnthropicImageBlockParam | AnthropicToolUseBlockParam | AnthropicToolResultBlockParam | AnthropicDocumentBlockParam | AnthropicThinkingBlockParam | AnthropicRedactedThinkingBlockParam | AnthropicServerToolUseBlockParam | AnthropicWebSearchToolResultBlockParam | AnthropicWebSearchResultBlockParam | AnthropicCompactionBlockParam;
+export type ChatAnthropicContentBlock = AnthropicTextBlockParam | AnthropicImageBlockParam | AnthropicToolUseBlockParam | AnthropicToolResultBlockParam | AnthropicDocumentBlockParam | AnthropicThinkingBlockParam | AnthropicRedactedThinkingBlockParam | AnthropicServerToolUseBlockParam | AnthropicWebSearchToolResultBlockParam | AnthropicWebSearchResultBlockParam | AnthropicSearchResultBlockParam | AnthropicCompactionBlockParam;
 export declare function isAnthropicImageBlockParam(block: unknown): block is AnthropicImageBlockParam;