illuma-agents 1.0.79 → 1.0.80

package/dist/cjs/graphs/Graph.cjs

@@ -16,7 +16,7 @@ var tools = require('../messages/tools.cjs');
 var graph = require('../utils/graph.cjs');
 var llm = require('../utils/llm.cjs');
 var run = require('../utils/run.cjs');
-require('js-tiktoken/lite');
+require('js-tiktoken');
 require('../utils/toonFormat.cjs');
 var contextAnalytics = require('../utils/contextAnalytics.cjs');
 require('zod-to-json-schema');

package/dist/cjs/llm/openai/index.cjs

@@ -13,7 +13,7 @@ require('nanoid');
 require('../../messages/core.cjs');
 require('../../utils/toonFormat.cjs');
 var run = require('../../utils/run.cjs');
-require('js-tiktoken/lite');
+require('js-tiktoken');
 require('zod-to-json-schema');
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type

package/dist/cjs/tools/ToolNode.cjs

@@ -7,7 +7,7 @@ require('nanoid');
 require('../messages/core.cjs');
 var toonFormat = require('../utils/toonFormat.cjs');
 var run = require('../utils/run.cjs');
-require('js-tiktoken/lite');
+require('js-tiktoken');
 require('zod-to-json-schema');
 var events = require('../utils/events.cjs');
 

package/dist/cjs/utils/tokens.cjs

@@ -1,6 +1,6 @@
 'use strict';
 
-var lite = require('js-tiktoken/lite');
+var jsTiktoken = require('js-tiktoken');
 var _enum = require('../common/enum.cjs');
 
 function getTokenCountForMessage(message, getTokenCount) {
@@ -52,20 +52,31 @@ function getTokenCountForMessage(message, getTokenCount) {
 }
 let encoderPromise;
 let tokenCounterPromise;
+/**
+ * Simple character-based token estimation (~4 chars per token).
+ * Used as fallback when tiktoken encoder fails to load.
+ */
+const estimateTokens = (text) => Math.ceil(text.length / 4);
 async function getSharedEncoder() {
     if (encoderPromise) {
         return encoderPromise;
     }
-    encoderPromise = (async () => {
-        const res = await fetch('https://tiktoken.pages.dev/js/o200k_base.json');
-        const o200k_base = await res.json();
-        return new lite.Tiktoken(o200k_base);
-    })();
+    // Use bundled tokenizer data - no external network calls
+    encoderPromise = Promise.resolve().then(() => {
+        try {
+            return jsTiktoken.getEncoding('o200k_base');
+        }
+        catch {
+            // Fallback to null - will use character estimation
+            return null;
+        }
+    });
     return encoderPromise;
 }
 /**
  * Creates a singleton token counter function that reuses the same encoder instance.
- * This avoids creating multiple function closures and prevents potential memory issues.
+ * Falls back to character-based estimation (~4 chars per token) if encoder fails.
+ * This ensures token counting never blocks LLM calls.
  */
 const createTokenCounter = async () => {
     if (tokenCounterPromise) {
@@ -73,7 +84,9 @@ const createTokenCounter = async () => {
     }
     tokenCounterPromise = (async () => {
         const enc = await getSharedEncoder();
-        const countTokens = (text) => enc.encode(text).length;
+        const countTokens = enc
+            ? (text) => enc.encode(text).length
+            : estimateTokens;
         return (message) => getTokenCountForMessage(message, countTokens);
     })();
     return tokenCounterPromise;

package/dist/cjs/utils/tokens.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.cjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tiktoken } from 'js-tiktoken/lite';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number\n): number {\n  const tokensPerMessage = 3;\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const item of value) {\n        if (\n          !item ||\n          !item.type ||\n          item.type === ContentTypes.ERROR ||\n          item.type === ContentTypes.IMAGE_URL\n        ) {\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call?.name || '';\n          if (toolName != null && toolName && typeof toolName === 'string') {\n            numTokens += getTokenCount(toolName);\n          }\n\n          const args = item.tool_call?.args || '';\n          if (args != null && args && typeof args === 'string') {\n            numTokens += getTokenCount(args);\n          }\n\n          const output = item.tool_call?.output || '';\n          if (output != null && output && typeof output === 'string') {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n\n        if (!nestedValue) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\nlet encoderPromise: Promise<Tiktoken> | undefined;\nlet tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;\n\nasync function getSharedEncoder(): Promise<Tiktoken> {\n  if (encoderPromise) {\n    return encoderPromise;\n  }\n  encoderPromise = (async (): Promise<Tiktoken> => {\n    const res = await fetch('https://tiktoken.pages.dev/js/o200k_base.json');\n    const o200k_base = await res.json();\n    return new Tiktoken(o200k_base);\n  })();\n  return encoderPromise;\n}\n\n/**\n * Creates a singleton token counter function that reuses the same encoder instance.\n * This avoids creating multiple function closures and prevents potential memory issues.\n */\nexport const createTokenCounter = async (): Promise<\n  (message: BaseMessage) => number\n> => {\n  if (tokenCounterPromise) {\n    return tokenCounterPromise;\n  }\n\n  tokenCounterPromise = (async (): Promise<\n    (message: BaseMessage) => number\n  > => {\n    const enc = await getSharedEncoder();\n    const countTokens = (text: string): number => enc.encode(text).length;\n    return (message: BaseMessage): number =>\n      getTokenCountForMessage(message, countTokens);\n  })();\n\n  return tokenCounterPromise;\n};\n\n/**\n * Utility to manage the token encoder lifecycle explicitly.\n * Useful for applications that need fine-grained control over resource management.\n */\nexport const TokenEncoderManager = {\n  /**\n   * Pre-initializes the encoder. This can be called during app startup\n   * to avoid lazy loading delays later.\n   */\n  async initialize(): Promise<void> {\n    await getSharedEncoder();\n  },\n\n  /**\n   * Clears the cached encoder and token counter.\n   * Useful for testing or when you need to force a fresh reload.\n   */\n  reset(): void {\n    encoderPromise = undefined;\n    tokenCounterPromise = undefined;\n  },\n\n  /**\n   * Checks if the encoder has been initialized.\n   */\n  isInitialized(): boolean {\n    return encoderPromise !== undefined;\n  },\n};\n"],"names":["ContentTypes","Tiktoken"],"mappings":";;;;;AAIgB,SAAA,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,SAAS,EACpC;oBACA;;AAGF,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;;oBAGtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;;oBAGlC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;;oBAEpC;;gBAGF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;;gBAGF,YAAY,CAAC,WAAW,CAAC;;;AAEtB,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;;AAC5B,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AACvC,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AAEhD,KAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA,IAAI,cAA6C;AACjD,IAAI,mBAA0E;AAE9E,eAAe,gBAAgB,GAAA;IAC7B,IAAI,cAAc,EAAE;AAClB,QAAA,OAAO,cAAc;;AAEvB,IAAA,cAAc,GAAG,CAAC,YAA8B;AAC9C,QAAA,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,+CAA+C,CAAC;AACxE,QAAA,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE;AACnC,QAAA,OAAO,IAAIC,aAAQ,CAAC,UAAU,CAAC;KAChC,GAAG;AACJ,IAAA,OAAO,cAAc;AACvB;AAEA;;;AAGG;AACU,MAAA,kBAAkB,GAAG,YAE9B;IACF,IAAI,mBAAmB,EAAE;AACvB,QAAA,OAAO,mBAAmB;;AAG5B,IAAA,mBAAmB,GAAG,CAAC,YAEnB;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE;AACpC,QAAA,MAAM,WAAW,GAAG,CAAC,IAAY,KAAa,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;QACrE,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;KAChD,GAAG;AAEJ,IAAA,OAAO,mBAAmB;AAC5B;AAEA;;;AAGG;AACU,MAAA,mBAAmB,GAAG;AACjC;;;AAGG;AACH,IAAA,MAAM,UAAU,GAAA;QACd,MAAM,gBAAgB,EAAE;KACzB;AAED;;;AAGG;IACH,KAAK,GAAA;QACH,cAAc,GAAG,SAAS;QAC1B,mBAAmB,GAAG,SAAS;KAChC;AAED;;AAEG;IACH,aAAa,GAAA;QACX,OAAO,cAAc,KAAK,SAAS;KACpC;;;;;;;"}
+{"version":3,"file":"tokens.cjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { getEncoding, type Tiktoken } from 'js-tiktoken';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number\n): number {\n  const tokensPerMessage = 3;\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const item of value) {\n        if (\n          !item ||\n          !item.type ||\n          item.type === ContentTypes.ERROR ||\n          item.type === ContentTypes.IMAGE_URL\n        ) {\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call?.name || '';\n          if (toolName != null && toolName && typeof toolName === 'string') {\n            numTokens += getTokenCount(toolName);\n          }\n\n          const args = item.tool_call?.args || '';\n          if (args != null && args && typeof args === 'string') {\n            numTokens += getTokenCount(args);\n          }\n\n          const output = item.tool_call?.output || '';\n          if (output != null && output && typeof output === 'string') {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n\n        if (!nestedValue) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\nlet encoderPromise: Promise<Tiktoken | null> | undefined;\nlet tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;\n\n/**\n * Simple character-based token estimation (~4 chars per token).\n * Used as fallback when tiktoken encoder fails to load.\n */\nconst estimateTokens = (text: string): number => Math.ceil(text.length / 4);\n\nasync function getSharedEncoder(): Promise<Tiktoken | null> {\n  if (encoderPromise) {\n    return encoderPromise;\n  }\n  // Use bundled tokenizer data - no external network calls\n  encoderPromise = Promise.resolve().then(() => {\n    try {\n      return getEncoding('o200k_base');\n    } catch {\n      // Fallback to null - will use character estimation\n      return null;\n    }\n  });\n  return encoderPromise;\n}\n\n/**\n * Creates a singleton token counter function that reuses the same encoder instance.\n * Falls back to character-based estimation (~4 chars per token) if encoder fails.\n * This ensures token counting never blocks LLM calls.\n */\nexport const createTokenCounter = async (): Promise<\n  (message: BaseMessage) => number\n> => {\n  if (tokenCounterPromise) {\n    return tokenCounterPromise;\n  }\n\n  tokenCounterPromise = (async (): Promise<\n    (message: BaseMessage) => number\n  > => {\n    const enc = await getSharedEncoder();\n    const countTokens = enc\n      ? (text: string): number => enc.encode(text).length\n      : estimateTokens;\n    return (message: BaseMessage): number =>\n      getTokenCountForMessage(message, countTokens);\n  })();\n\n  return tokenCounterPromise;\n};\n\n/**\n * Utility to manage the token encoder lifecycle explicitly.\n * Useful for applications that need fine-grained control over resource management.\n */\nexport const TokenEncoderManager = {\n  /**\n   * Pre-initializes the encoder. This can be called during app startup\n   * to avoid lazy loading delays later.\n   */\n  async initialize(): Promise<void> {\n    await getSharedEncoder();\n  },\n\n  /**\n   * Clears the cached encoder and token counter.\n   * Useful for testing or when you need to force a fresh reload.\n   */\n  reset(): void {\n    encoderPromise = undefined;\n    tokenCounterPromise = undefined;\n  },\n\n  /**\n   * Checks if the encoder has been initialized.\n   */\n  isInitialized(): boolean {\n    return encoderPromise !== undefined;\n  },\n};\n"],"names":["ContentTypes","getEncoding"],"mappings":";;;;;AAIgB,SAAA,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,SAAS,EACpC;oBACA;;AAGF,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAKA,kBAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;;oBAGtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;;oBAGlC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;;oBAEpC;;gBAGF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;;gBAGF,YAAY,CAAC,WAAW,CAAC;;;AAEtB,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;;AAC5B,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AACvC,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AAEhD,KAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA,IAAI,cAAoD;AACxD,IAAI,mBAA0E;AAE9E;;;AAGG;AACH,MAAM,cAAc,GAAG,CAAC,IAAY,KAAa,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AAE3E,eAAe,gBAAgB,GAAA;IAC7B,IAAI,cAAc,EAAE;AAClB,QAAA,OAAO,cAAc;;;IAGvB,cAAc,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,MAAK;AAC3C,QAAA,IAAI;AACF,YAAA,OAAOC,sBAAW,CAAC,YAAY,CAAC;;AAChC,QAAA,MAAM;;AAEN,YAAA,OAAO,IAAI;;AAEf,KAAC,CAAC;AACF,IAAA,OAAO,cAAc;AACvB;AAEA;;;;AAIG;AACU,MAAA,kBAAkB,GAAG,YAE9B;IACF,IAAI,mBAAmB,EAAE;AACvB,QAAA,OAAO,mBAAmB;;AAG5B,IAAA,mBAAmB,GAAG,CAAC,YAEnB;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE;QACpC,MAAM,WAAW,GAAG;AAClB,cAAE,CAAC,IAAY,KAAa,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;cAC3C,cAAc;QAClB,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;KAChD,GAAG;AAEJ,IAAA,OAAO,mBAAmB;AAC5B;AAEA;;;AAGG;AACU,MAAA,mBAAmB,GAAG;AACjC;;;AAGG;AACH,IAAA,MAAM,UAAU,GAAA;QACd,MAAM,gBAAgB,EAAE;KACzB;AAED;;;AAGG;IACH,KAAK,GAAA;QACH,cAAc,GAAG,SAAS;QAC1B,mBAAmB,GAAG,SAAS;KAChC;AAED;;AAEG;IACH,aAAa,GAAA;QACX,OAAO,cAAc,KAAK,SAAS;KACpC;;;;;;;"}

package/dist/esm/graphs/Graph.mjs

@@ -14,7 +14,7 @@ import { extractToolDiscoveries } from '../messages/tools.mjs';
 import { resetIfNotEmpty, joinKeys } from '../utils/graph.mjs';
 import { isOpenAILike, isGoogleLike } from '../utils/llm.mjs';
 import { sleep } from '../utils/run.mjs';
-import 'js-tiktoken/lite';
+import 'js-tiktoken';
 import '../utils/toonFormat.mjs';
 import { buildContextAnalytics } from '../utils/contextAnalytics.mjs';
 import 'zod-to-json-schema';

package/dist/esm/llm/openai/index.mjs

@@ -11,7 +11,7 @@ import 'nanoid';
 import '../../messages/core.mjs';
 import '../../utils/toonFormat.mjs';
 import { sleep } from '../../utils/run.mjs';
-import 'js-tiktoken/lite';
+import 'js-tiktoken';
 import 'zod-to-json-schema';
 
 // eslint-disable-next-line @typescript-eslint/explicit-function-return-type

package/dist/esm/tools/ToolNode.mjs

@@ -5,7 +5,7 @@ import 'nanoid';
 import '../messages/core.mjs';
 import { processToolOutput } from '../utils/toonFormat.mjs';
 import { RunnableCallable } from '../utils/run.mjs';
-import 'js-tiktoken/lite';
+import 'js-tiktoken';
 import 'zod-to-json-schema';
 import { safeDispatchCustomEvent } from '../utils/events.mjs';
 

package/dist/esm/utils/tokens.mjs

@@ -1,4 +1,4 @@
-import { Tiktoken } from 'js-tiktoken/lite';
+import { getEncoding } from 'js-tiktoken';
 import { ContentTypes } from '../common/enum.mjs';
 
 function getTokenCountForMessage(message, getTokenCount) {
@@ -50,20 +50,31 @@ function getTokenCountForMessage(message, getTokenCount) {
 }
 let encoderPromise;
 let tokenCounterPromise;
+/**
+ * Simple character-based token estimation (~4 chars per token).
+ * Used as fallback when tiktoken encoder fails to load.
+ */
+const estimateTokens = (text) => Math.ceil(text.length / 4);
 async function getSharedEncoder() {
     if (encoderPromise) {
         return encoderPromise;
     }
-    encoderPromise = (async () => {
-        const res = await fetch('https://tiktoken.pages.dev/js/o200k_base.json');
-        const o200k_base = await res.json();
-        return new Tiktoken(o200k_base);
-    })();
+    // Use bundled tokenizer data - no external network calls
+    encoderPromise = Promise.resolve().then(() => {
+        try {
+            return getEncoding('o200k_base');
+        }
+        catch {
+            // Fallback to null - will use character estimation
+            return null;
+        }
+    });
     return encoderPromise;
 }
 /**
  * Creates a singleton token counter function that reuses the same encoder instance.
- * This avoids creating multiple function closures and prevents potential memory issues.
+ * Falls back to character-based estimation (~4 chars per token) if encoder fails.
+ * This ensures token counting never blocks LLM calls.
  */
 const createTokenCounter = async () => {
     if (tokenCounterPromise) {
@@ -71,7 +82,9 @@ const createTokenCounter = async () => {
     }
     tokenCounterPromise = (async () => {
         const enc = await getSharedEncoder();
-        const countTokens = (text) => enc.encode(text).length;
+        const countTokens = enc
+            ? (text) => enc.encode(text).length
+            : estimateTokens;
         return (message) => getTokenCountForMessage(message, countTokens);
     })();
     return tokenCounterPromise;

package/dist/esm/utils/tokens.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.mjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { Tiktoken } from 'js-tiktoken/lite';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number\n): number {\n  const tokensPerMessage = 3;\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const item of value) {\n        if (\n          !item ||\n          !item.type ||\n          item.type === ContentTypes.ERROR ||\n          item.type === ContentTypes.IMAGE_URL\n        ) {\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call?.name || '';\n          if (toolName != null && toolName && typeof toolName === 'string') {\n            numTokens += getTokenCount(toolName);\n          }\n\n          const args = item.tool_call?.args || '';\n          if (args != null && args && typeof args === 'string') {\n            numTokens += getTokenCount(args);\n          }\n\n          const output = item.tool_call?.output || '';\n          if (output != null && output && typeof output === 'string') {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n\n        if (!nestedValue) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\nlet encoderPromise: Promise<Tiktoken> | undefined;\nlet tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;\n\nasync function getSharedEncoder(): Promise<Tiktoken> {\n  if (encoderPromise) {\n    return encoderPromise;\n  }\n  encoderPromise = (async (): Promise<Tiktoken> => {\n    const res = await fetch('https://tiktoken.pages.dev/js/o200k_base.json');\n    const o200k_base = await res.json();\n    return new Tiktoken(o200k_base);\n  })();\n  return encoderPromise;\n}\n\n/**\n * Creates a singleton token counter function that reuses the same encoder instance.\n * This avoids creating multiple function closures and prevents potential memory issues.\n */\nexport const createTokenCounter = async (): Promise<\n  (message: BaseMessage) => number\n> => {\n  if (tokenCounterPromise) {\n    return tokenCounterPromise;\n  }\n\n  tokenCounterPromise = (async (): Promise<\n    (message: BaseMessage) => number\n  > => {\n    const enc = await getSharedEncoder();\n    const countTokens = (text: string): number => enc.encode(text).length;\n    return (message: BaseMessage): number =>\n      getTokenCountForMessage(message, countTokens);\n  })();\n\n  return tokenCounterPromise;\n};\n\n/**\n * Utility to manage the token encoder lifecycle explicitly.\n * Useful for applications that need fine-grained control over resource management.\n */\nexport const TokenEncoderManager = {\n  /**\n   * Pre-initializes the encoder. This can be called during app startup\n   * to avoid lazy loading delays later.\n   */\n  async initialize(): Promise<void> {\n    await getSharedEncoder();\n  },\n\n  /**\n   * Clears the cached encoder and token counter.\n   * Useful for testing or when you need to force a fresh reload.\n   */\n  reset(): void {\n    encoderPromise = undefined;\n    tokenCounterPromise = undefined;\n  },\n\n  /**\n   * Checks if the encoder has been initialized.\n   */\n  isInitialized(): boolean {\n    return encoderPromise !== undefined;\n  },\n};\n"],"names":[],"mappings":";;;AAIgB,SAAA,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EACpC;oBACA;;AAGF,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;;oBAGtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;;oBAGlC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;;oBAEpC;;gBAGF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;;gBAGF,YAAY,CAAC,WAAW,CAAC;;;AAEtB,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;;AAC5B,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AACvC,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AAEhD,KAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA,IAAI,cAA6C;AACjD,IAAI,mBAA0E;AAE9E,eAAe,gBAAgB,GAAA;IAC7B,IAAI,cAAc,EAAE;AAClB,QAAA,OAAO,cAAc;;AAEvB,IAAA,cAAc,GAAG,CAAC,YAA8B;AAC9C,QAAA,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,+CAA+C,CAAC;AACxE,QAAA,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE;AACnC,QAAA,OAAO,IAAI,QAAQ,CAAC,UAAU,CAAC;KAChC,GAAG;AACJ,IAAA,OAAO,cAAc;AACvB;AAEA;;;AAGG;AACU,MAAA,kBAAkB,GAAG,YAE9B;IACF,IAAI,mBAAmB,EAAE;AACvB,QAAA,OAAO,mBAAmB;;AAG5B,IAAA,mBAAmB,GAAG,CAAC,YAEnB;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE;AACpC,QAAA,MAAM,WAAW,GAAG,CAAC,IAAY,KAAa,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;QACrE,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;KAChD,GAAG;AAEJ,IAAA,OAAO,mBAAmB;AAC5B;AAEA;;;AAGG;AACU,MAAA,mBAAmB,GAAG;AACjC;;;AAGG;AACH,IAAA,MAAM,UAAU,GAAA;QACd,MAAM,gBAAgB,EAAE;KACzB;AAED;;;AAGG;IACH,KAAK,GAAA;QACH,cAAc,GAAG,SAAS;QAC1B,mBAAmB,GAAG,SAAS;KAChC;AAED;;AAEG;IACH,aAAa,GAAA;QACX,OAAO,cAAc,KAAK,SAAS;KACpC;;;;;"}
+{"version":3,"file":"tokens.mjs","sources":["../../../src/utils/tokens.ts"],"sourcesContent":["import { getEncoding, type Tiktoken } from 'js-tiktoken';\nimport type { BaseMessage } from '@langchain/core/messages';\nimport { ContentTypes } from '@/common/enum';\n\nexport function getTokenCountForMessage(\n  message: BaseMessage,\n  getTokenCount: (text: string) => number\n): number {\n  const tokensPerMessage = 3;\n\n  const processValue = (value: unknown): void => {\n    if (Array.isArray(value)) {\n      for (const item of value) {\n        if (\n          !item ||\n          !item.type ||\n          item.type === ContentTypes.ERROR ||\n          item.type === ContentTypes.IMAGE_URL\n        ) {\n          continue;\n        }\n\n        if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {\n          const toolName = item.tool_call?.name || '';\n          if (toolName != null && toolName && typeof toolName === 'string') {\n            numTokens += getTokenCount(toolName);\n          }\n\n          const args = item.tool_call?.args || '';\n          if (args != null && args && typeof args === 'string') {\n            numTokens += getTokenCount(args);\n          }\n\n          const output = item.tool_call?.output || '';\n          if (output != null && output && typeof output === 'string') {\n            numTokens += getTokenCount(output);\n          }\n          continue;\n        }\n\n        const nestedValue = item[item.type];\n\n        if (!nestedValue) {\n          continue;\n        }\n\n        processValue(nestedValue);\n      }\n    } else if (typeof value === 'string') {\n      numTokens += getTokenCount(value);\n    } else if (typeof value === 'number') {\n      numTokens += getTokenCount(value.toString());\n    } else if (typeof value === 'boolean') {\n      numTokens += getTokenCount(value.toString());\n    }\n  };\n\n  let numTokens = tokensPerMessage;\n  processValue(message.content);\n  return numTokens;\n}\n\nlet encoderPromise: Promise<Tiktoken | null> | undefined;\nlet tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;\n\n/**\n * Simple character-based token estimation (~4 chars per token).\n * Used as fallback when tiktoken encoder fails to load.\n */\nconst estimateTokens = (text: string): number => Math.ceil(text.length / 4);\n\nasync function getSharedEncoder(): Promise<Tiktoken | null> {\n  if (encoderPromise) {\n    return encoderPromise;\n  }\n  // Use bundled tokenizer data - no external network calls\n  encoderPromise = Promise.resolve().then(() => {\n    try {\n      return getEncoding('o200k_base');\n    } catch {\n      // Fallback to null - will use character estimation\n      return null;\n    }\n  });\n  return encoderPromise;\n}\n\n/**\n * Creates a singleton token counter function that reuses the same encoder instance.\n * Falls back to character-based estimation (~4 chars per token) if encoder fails.\n * This ensures token counting never blocks LLM calls.\n */\nexport const createTokenCounter = async (): Promise<\n  (message: BaseMessage) => number\n> => {\n  if (tokenCounterPromise) {\n    return tokenCounterPromise;\n  }\n\n  tokenCounterPromise = (async (): Promise<\n    (message: BaseMessage) => number\n  > => {\n    const enc = await getSharedEncoder();\n    const countTokens = enc\n      ? (text: string): number => enc.encode(text).length\n      : estimateTokens;\n    return (message: BaseMessage): number =>\n      getTokenCountForMessage(message, countTokens);\n  })();\n\n  return tokenCounterPromise;\n};\n\n/**\n * Utility to manage the token encoder lifecycle explicitly.\n * Useful for applications that need fine-grained control over resource management.\n */\nexport const TokenEncoderManager = {\n  /**\n   * Pre-initializes the encoder. This can be called during app startup\n   * to avoid lazy loading delays later.\n   */\n  async initialize(): Promise<void> {\n    await getSharedEncoder();\n  },\n\n  /**\n   * Clears the cached encoder and token counter.\n   * Useful for testing or when you need to force a fresh reload.\n   */\n  reset(): void {\n    encoderPromise = undefined;\n    tokenCounterPromise = undefined;\n  },\n\n  /**\n   * Checks if the encoder has been initialized.\n   */\n  isInitialized(): boolean {\n    return encoderPromise !== undefined;\n  },\n};\n"],"names":[],"mappings":";;;AAIgB,SAAA,uBAAuB,CACrC,OAAoB,EACpB,aAAuC,EAAA;IAEvC,MAAM,gBAAgB,GAAG,CAAC;AAE1B,IAAA,MAAM,YAAY,GAAG,CAAC,KAAc,KAAU;AAC5C,QAAA,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;AACxB,YAAA,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;AACxB,gBAAA,IACE,CAAC,IAAI;oBACL,CAAC,IAAI,CAAC,IAAI;AACV,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,KAAK;AAChC,oBAAA,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,EACpC;oBACA;;AAGF,gBAAA,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,IAAI,IAAI,EAAE;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBAC3C,IAAI,QAAQ,IAAI,IAAI,IAAI,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;AAChE,wBAAA,SAAS,IAAI,aAAa,CAAC,QAAQ,CAAC;;oBAGtC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,EAAE;oBACvC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;AACpD,wBAAA,SAAS,IAAI,aAAa,CAAC,IAAI,CAAC;;oBAGlC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,IAAI,EAAE;oBAC3C,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE;AAC1D,wBAAA,SAAS,IAAI,aAAa,CAAC,MAAM,CAAC;;oBAEpC;;gBAGF,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;gBAEnC,IAAI,CAAC,WAAW,EAAE;oBAChB;;gBAGF,YAAY,CAAC,WAAW,CAAC;;;AAEtB,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;AACpC,YAAA,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC;;AAC5B,aAAA,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;YACpC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AACvC,aAAA,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YACrC,SAAS,IAAI,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;;AAEhD,KAAC;IAED,IAAI,SAAS,GAAG,gBAAgB;AAChC,IAAA,YAAY,CAAC,OAAO,CAAC,OAAO,CAAC;AAC7B,IAAA,OAAO,SAAS;AAClB;AAEA,IAAI,cAAoD;AACxD,IAAI,mBAA0E;AAE9E;;;AAGG;AACH,MAAM,cAAc,GAAG,CAAC,IAAY,KAAa,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AAE3E,eAAe,gBAAgB,GAAA;IAC7B,IAAI,cAAc,EAAE;AAClB,QAAA,OAAO,cAAc;;;IAGvB,cAAc,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,MAAK;AAC3C,QAAA,IAAI;AACF,YAAA,OAAO,WAAW,CAAC,YAAY,CAAC;;AAChC,QAAA,MAAM;;AAEN,YAAA,OAAO,IAAI;;AAEf,KAAC,CAAC;AACF,IAAA,OAAO,cAAc;AACvB;AAEA;;;;AAIG;AACU,MAAA,kBAAkB,GAAG,YAE9B;IACF,IAAI,mBAAmB,EAAE;AACvB,QAAA,OAAO,mBAAmB;;AAG5B,IAAA,mBAAmB,GAAG,CAAC,YAEnB;AACF,QAAA,MAAM,GAAG,GAAG,MAAM,gBAAgB,EAAE;QACpC,MAAM,WAAW,GAAG;AAClB,cAAE,CAAC,IAAY,KAAa,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;cAC3C,cAAc;QAClB,OAAO,CAAC,OAAoB,KAC1B,uBAAuB,CAAC,OAAO,EAAE,WAAW,CAAC;KAChD,GAAG;AAEJ,IAAA,OAAO,mBAAmB;AAC5B;AAEA;;;AAGG;AACU,MAAA,mBAAmB,GAAG;AACjC;;;AAGG;AACH,IAAA,MAAM,UAAU,GAAA;QACd,MAAM,gBAAgB,EAAE;KACzB;AAED;;;AAGG;IACH,KAAK,GAAA;QACH,cAAc,GAAG,SAAS;QAC1B,mBAAmB,GAAG,SAAS;KAChC;AAED;;AAEG;IACH,aAAa,GAAA;QACX,OAAO,cAAc,KAAK,SAAS;KACpC;;;;;"}

package/dist/types/utils/tokens.d.ts

@@ -2,7 +2,8 @@ import type { BaseMessage } from '@langchain/core/messages';
 export declare function getTokenCountForMessage(message: BaseMessage, getTokenCount: (text: string) => number): number;
 /**
  * Creates a singleton token counter function that reuses the same encoder instance.
- * This avoids creating multiple function closures and prevents potential memory issues.
+ * Falls back to character-based estimation (~4 chars per token) if encoder fails.
+ * This ensures token counting never blocks LLM calls.
  */
 export declare const createTokenCounter: () => Promise<(message: BaseMessage) => number>;
 /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.79",
+  "version": "1.0.80",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/utils/tokens.ts

@@ -1,4 +1,4 @@
-import { Tiktoken } from 'js-tiktoken/lite';
+import { getEncoding, type Tiktoken } from 'js-tiktoken';
 import type { BaseMessage } from '@langchain/core/messages';
 import { ContentTypes } from '@/common/enum';
 
@@ -60,24 +60,35 @@ export function getTokenCountForMessage(
   return numTokens;
 }
 
-let encoderPromise: Promise<Tiktoken> | undefined;
+let encoderPromise: Promise<Tiktoken | null> | undefined;
 let tokenCounterPromise: Promise<(message: BaseMessage) => number> | undefined;
 
-async function getSharedEncoder(): Promise<Tiktoken> {
+/**
+ * Simple character-based token estimation (~4 chars per token).
+ * Used as fallback when tiktoken encoder fails to load.
+ */
+const estimateTokens = (text: string): number => Math.ceil(text.length / 4);
+
+async function getSharedEncoder(): Promise<Tiktoken | null> {
   if (encoderPromise) {
     return encoderPromise;
   }
-  encoderPromise = (async (): Promise<Tiktoken> => {
-    const res = await fetch('https://tiktoken.pages.dev/js/o200k_base.json');
-    const o200k_base = await res.json();
-    return new Tiktoken(o200k_base);
-  })();
+  // Use bundled tokenizer data - no external network calls
+  encoderPromise = Promise.resolve().then(() => {
+    try {
+      return getEncoding('o200k_base');
+    } catch {
+      // Fallback to null - will use character estimation
+      return null;
+    }
+  });
   return encoderPromise;
 }
 
 /**
  * Creates a singleton token counter function that reuses the same encoder instance.
- * This avoids creating multiple function closures and prevents potential memory issues.
+ * Falls back to character-based estimation (~4 chars per token) if encoder fails.
+ * This ensures token counting never blocks LLM calls.
  */
 export const createTokenCounter = async (): Promise<
   (message: BaseMessage) => number
@@ -90,7 +101,9 @@ export const createTokenCounter = async (): Promise<
     (message: BaseMessage) => number
   > => {
     const enc = await getSharedEncoder();
-    const countTokens = (text: string): number => enc.encode(text).length;
+    const countTokens = enc
+      ? (text: string): number => enc.encode(text).length
+      : estimateTokens;
     return (message: BaseMessage): number =>
       getTokenCountForMessage(message, countTokens);
   })();