foliko 2.0.26 → 2.0.28

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "foliko",
-  "version": "2.0.26",
+  "version": "2.0.28",
   "description": "简约的插件化 Agent 框架",
   "main": "src/index.js",
   "type": "commonjs",

package/src/agent/chat.js

@@ -718,44 +718,31 @@ class AgentChatHandler extends EventEmitter {
   async *chatStream(message, options = {}) {
     const sessionId = options.sessionId || this.getDefaultSessionId();
     const framework = this.agent.framework;
-    // ★ 提到外层作用域，catch 中要回滚 userMessage 时需要访问
     let messageStore = null;
     let messages = null;
     let userMessage = null;
+    let didRetryWithCompression = false;
 
-    try {
-      const prepared = await this._prepareSession(message, sessionId);
-      messageStore = prepared.messageStore;
-      messages = prepared.messages;
-      userMessage = prepared.userMessage;
-
-      if (!this._aiClient) {
-        throw new Error('AI 客户端未配置');
-      }
-      const systemPrompt = framework.prompts.build();
-      //await fs.promises.writeFile(`.${sessionId}_systemPrompt.md`, systemPrompt); // 调试用：保存系统提示词
-      const tools = this._getToolMap();
+    // 内部异步函数执行真正的流式处理，便于压缩重试时重启
+    const doStream = async function* (self, sessionId, message, messageStore, messages, userMessage) {
+      const tools = self._getToolMap();
       const toolLoop = new ToolLoop({
         tools,
-        maxSteps: this._maxSteps || 1000,
-        prepareStep: this._createPrepareStep(),
-        repairToolCall: this._repairToolCall.bind(this),
+        maxSteps: self._maxSteps || 1000,
+        prepareStep: self._createPrepareStep(),
+        repairToolCall: self._repairToolCall.bind(self),
         abortSignal: framework.getAbortSignal?.(),
-        thinkingMode: this._thinkingMode,
+        thinkingMode: self._thinkingMode,
       });
 
-      // DeepSeek thinking mode: 收集 reasoning_content
-      const isThinking = this._thinkingMode;
       let reasoningContent = '';
-
       const stream = toolLoop.stream({
         messages,
-        systemPrompt,
-        model: this.model,
-        client: this._aiClient,
-        ...this.providerOptions,
+        systemPrompt: framework.prompts.build(),
+        model: self.model,
+        client: self._aiClient,
+        ...self.providerOptions,
       });
-      // 遍历流式响应
       let fullText = '';
       let usageRecv = {};
 
@@ -778,67 +765,94 @@ class AgentChatHandler extends EventEmitter {
           yield { type: 'usage', inputTokens: usageRecv.inputTokens, outputTokens: usageRecv.outputTokens };
         }
       }
+
       // 持久化用量
       messageStore.usage = usageRecv;
-
-      // 同时通过 framework 事件广播（绕开 queue 事件链路）
-      if (this.agent && this.agent.framework) {
-        this.agent.framework.emit('agent:usage', {
-          sessionId,
-          usage: usageRecv,
-        });
+      if (self.agent && self.agent.framework) {
+        self.agent.framework.emit('agent:usage', { sessionId, usage: usageRecv });
       }
 
-      // yield usage 数据，让 UI 层能直接获取
-      // （usage 已在 stream 循环中 yield，无需重复）
-
       const userMsg = messages.length > 0 ? messages[messages.length - 1] : null;
-      this.emit('message', { content: fullText, sessionId: sessionId, userMessage: userMsg });
+      self.emit('message', { content: fullText, sessionId, userMessage: userMsg });
 
-      // ★ 修复：成功路径才持久化。AI 失败时若 finally 也 save，
-      //   会把刚 push 的 userMessage 写盘 → 下次请求 loadHistory 再次
-      //   load 到，_prepareSession 再 push 一份 → 连续两条相同 user
-      //   → AI SDK 抛 "Invalid prompt: ModelMessage[] schema"
       try {
         await messageStore.save();
       } catch (err) {
         logger.error(`[${sessionId}] Failed to save message store: ${err.message}`);
       }
-    } catch (err) {
-      // ★ 修复：失败时打印 InvalidPromptError 的 cause（含 Zod issue 路径），
-      //   方便定位是哪条消息、哪个字段不匹配 ModelMessage[] schema
-      logger.error('[AgentChat] chatStream error:', err.message);
-      if (err.cause) {
-        const cause = err.cause;
-        if (cause.issues) {
-          for (const issue of cause.issues.slice(0, 5)) {
-            logger.error(
-              `[AgentChat] schema issue: path=${JSON.stringify(issue.path)} code=${issue.code} message=${issue.message}`
-            );
-          }
+    }.bind(null, this);
+
+    // 压缩重试循环
+    for (let attempt = 0; attempt <= 1; attempt++) {
+      try {
+        // 首次尝试前做 prepareSession；压缩重试时跳过
+        if (attempt === 0) {
+          const prepared = await this._prepareSession(message, sessionId);
+          messageStore = prepared.messageStore;
+          messages = prepared.messages;
+          userMessage = prepared.userMessage;
         } else {
-          logger.error(`[AgentChat] error cause: ${cause.message || String(cause)}`);
+          // 重试前重新 build 系统提示词
+          this._systemPrompt = this.agent._buildSystemPrompt();
+          this._invalidateMessageTokensCache(messageStore);
         }
-      }
 
-      // ★ 修复：失败时 ROLLBACK 掉本轮 push 的 userMessage
-      try {
-        if (messages && userMessage) {
-          const last = messages[messages.length - 1];
-          if (last && last.role === 'user' && last === userMessage) {
-            messages.pop();
+        if (!this._aiClient) {
+          throw new Error('AI 客户端未配置');
+        }
+
+        // 执行流式生成
+        for await (const chunk of doStream(this, sessionId, message, messageStore, messages, userMessage)) {
+          yield chunk;
+        }
+        return; // 成功完成
+      } catch (err) {
+        // ★ 上下文超限：压缩后重试一次
+        if (!didRetryWithCompression && this._isContextOverflowError(err)) {
+          logger.warn(`[AgentChat] chatStream context overflow (attempt ${attempt + 1}), compressing and retrying...`);
+          try {
+            await this._compressContext(sessionId, messages, messageStore);
+            this._invalidateMessageTokensCache(messageStore);
+            logger.info(`[AgentChat] Compression done, retrying stream...`);
+            didRetryWithCompression = true;
+            continue; // 重试
+          } catch (compressErr) {
+            logger.error(`[AgentChat] Compression failed: ${compressErr.message}`);
           }
         }
-      } catch { /* ignore */ }
 
-      // 统一错误处理：提取简洁消息并通过 yield 传递
-      const friendlyMessage = err?.message?.split('\n')[0] || '未知错误';
+        // 打印错误
+        logger.error('[AgentChat] chatStream error:', err.message);
+        if (err.cause) {
+          const cause = err.cause;
+          if (cause.issues) {
+            for (const issue of cause.issues.slice(0, 5)) {
+              logger.error(
+                `[AgentChat] schema issue: path=${JSON.stringify(issue.path)} code=${issue.code} message=${issue.message}`
+              );
+            }
+          } else {
+            logger.error(`[AgentChat] error cause: ${cause.message || String(cause)}`);
+          }
+        }
 
-      // 用户主动中断（framework.stop()）不上报 message:error，避免与上游重复
-      if (err.name !== 'AbortError') {
-        this.emit('message:error', { sessionId, error: friendlyMessage, originalError: err });
+        // 回滚 userMessage
+        try {
+          if (messages && userMessage) {
+            const last = messages[messages.length - 1];
+            if (last && last.role === 'user' && last === userMessage) {
+              messages.pop();
+            }
+          }
+        } catch { /* ignore */ }
+
+        const friendlyMessage = err?.message?.split('\n')[0] || '未知错误';
+        if (err.name !== 'AbortError') {
+          this.emit('message:error', { sessionId, error: friendlyMessage, originalError: err });
+        }
+        yield { type: 'error', error: friendlyMessage };
+        return; // 失败返回
       }
-      yield { type: 'error', error: friendlyMessage };
     }
   }
 
@@ -891,109 +905,132 @@ class AgentChatHandler extends EventEmitter {
     let messageStore = null;
     let messages = null;
     let userMessage = null;
+    let didRetryWithCompression = false;
 
-    try {
-      const prepared = await this._prepareSession(message, sessionId);
-      messageStore = prepared.messageStore;
-      messages = prepared.messages;
-      userMessage = prepared.userMessage;
+    // 最多压缩重试 1 次
+    const maxCompressionRetries = 1;
 
-      if (!this._aiClient) {
-        throw new Error('AI 客户端未配置');
-      }
-      const systemPrompt = framework.prompts.build();
-      const tools = this._getToolMap();
+    for (let attempt = 0; attempt <= maxCompressionRetries; attempt++) {
+      try {
+        // 首次尝试前做 prepareSession；压缩重试时跳过（已压缩过）
+        if (attempt === 0) {
+          const prepared = await this._prepareSession(message, sessionId);
+          messageStore = prepared.messageStore;
+          messages = prepared.messages;
+          userMessage = prepared.userMessage;
+        } else {
+          // 压缩重试前重新 build 系统提示词（压缩后 messages 变了）
+          this._systemPrompt = this.agent._buildSystemPrompt();
+          this._invalidateMessageTokensCache(messageStore);
+        }
 
-      // DeepSeek thinking mode: 处理参数
-      const apiOptions = { ...this.providerOptions };
-      if (this._thinkingMode) {
-        delete apiOptions.temperature;
-      }
+        if (!this._aiClient) {
+          throw new Error('AI 客户端未配置');
+        }
+        const systemPrompt = framework.prompts.build();
+        const tools = this._getToolMap();
 
-      const toolLoop = new ToolLoop({
-        tools,
-        maxSteps: this._maxSteps || 1000,
-        prepareStep: this._createPrepareStep(),
-        repairToolCall: this._repairToolCall.bind(this),
-        abortSignal: framework.getAbortSignal?.(),
-        thinkingMode: this._thinkingMode,
-      });
+        // DeepSeek thinking mode: 处理参数
+        const apiOptions = { ...this.providerOptions };
+        if (this._thinkingMode) {
+          delete apiOptions.temperature;
+        }
 
-      const result = await toolLoop.generate({
-        messages,
-        systemPrompt,
-        model: this.model,
-        client: this._aiClient,
-        ...apiOptions,
-      });
-      // result = { text, response: { messages }, steps, usage, reasoningText }
+        const toolLoop = new ToolLoop({
+          tools,
+          maxSteps: this._maxSteps || 1000,
+          prepareStep: this._createPrepareStep(),
+          repairToolCall: this._repairToolCall.bind(this),
+          abortSignal: framework.getAbortSignal?.(),
+          thinkingMode: this._thinkingMode,
+        });
 
-      if (result.usage && (result.usage.promptTokens > 0 || result.usage.completionTokens > 0)) {
-        this._updateMessageStoreUsage(messageStore, result.usage);
-      } else {
-        const inputTokens = this._tokenCounter.countMessages(messages);
-        const outputTokens = this._tokenCounter.countText(result.text || '');
-        messageStore.usage = { promptTokens: inputTokens, completionTokens: outputTokens };
-      }
+        const result = await toolLoop.generate({
+          messages,
+          systemPrompt,
+          model: this.model,
+          client: this._aiClient,
+          ...apiOptions,
+        });
+        // result = { text, response: { messages }, steps, usage, reasoningText }
 
-      // ToolLoop 已经直接在 messages 末尾添加了新消息，不需要再 push
-      // 计算 original 的 user message
-      const userMsg = result.text
-        ? messages.slice(-1)[0]
-        : messages[messages.length - 1];
-      this.emit('message', { content: result.text, sessionId: sessionId, userMessage: userMsg });
-
-      // ★ 修复：AI 调用成功后才持久化（之前 finally 总是 save，
-      //   会把刚 push 的 userMessage 写盘；下次请求 loadHistory 再次 load 到，
-      //   _prepareSession 又 push 一份 → 连续两条相同 user 消息 →
-      //   AI SDK 抛 "Invalid prompt: ModelMessage[] schema"）
-      try {
-        await messageStore.save();
-      } catch (err) {
-        logger.error(`[${sessionId}] Failed to save message store: ${err.message}`);
-      }
+        if (result.usage && (result.usage.promptTokens > 0 || result.usage.completionTokens > 0)) {
+          this._updateMessageStoreUsage(messageStore, result.usage);
+        } else {
+          const inputTokens = this._tokenCounter.countMessages(messages);
+          const outputTokens = this._tokenCounter.countText(result.text || '');
+          messageStore.usage = { promptTokens: inputTokens, completionTokens: outputTokens };
+        }
 
-      return {
-        success: true,
-        message: cleanResponse(result.text || ''),
-        stepCount: result.stepCount || 1,
-      };
-    } catch (err) {
-      // ★ 修复：失败时打印 InvalidPromptError 的 cause（含 Zod issue 路径），
-      //   方便定位是哪条消息、哪个字段不匹配 ModelMessage[] schema
-      logger.error('[AgentChat] _processMessage error:', err.message);
-      if (err.cause) {
-        const cause = err.cause;
-        if (cause.issues) {
-          // ZodError：逐条打印
-          for (const issue of cause.issues.slice(0, 5)) {
-            logger.error(
-              `[AgentChat] schema issue: path=${JSON.stringify(issue.path)} code=${issue.code} message=${issue.message}`
-            );
+        // ToolLoop 已经直接在 messages 末尾添加了新消息，不需要再 push
+        // 计算 original 的 user message
+        const userMsg = result.text
+          ? messages.slice(-1)[0]
+          : messages[messages.length - 1];
+        this.emit('message', { content: result.text, sessionId: sessionId, userMessage: userMsg });
+
+        // ★ 修复：AI 调用成功后才持久化
+        try {
+          await messageStore.save();
+        } catch (err) {
+          logger.error(`[${sessionId}] Failed to save message store: ${err.message}`);
+        }
+
+        return {
+          success: true,
+          message: cleanResponse(result.text || ''),
+          stepCount: result.stepCount || 1,
+        };
+      } catch (err) {
+        // ★ 上下文超限：压缩后重试一次
+        if (!didRetryWithCompression && this._isContextOverflowError(err)) {
+          logger.warn(`[AgentChat] Context overflow (attempt ${attempt + 1}), compressing and retrying...`);
+          try {
+            // 强制压缩（忽略阈值检查）
+            await this._compressContext(sessionId, messages, messageStore);
+            this._invalidateMessageTokensCache(messageStore);
+            logger.info(`[AgentChat] Compression done, retrying...`);
+            didRetryWithCompression = true;
+            continue; // 重试
+          } catch (compressErr) {
+            logger.error(`[AgentChat] Compression failed: ${compressErr.message}`);
+            // 压缩失败，继续走下方错误处理
           }
-        } else {
-          logger.error(`[AgentChat] error cause: ${cause.message || String(cause)}`);
         }
-      }
-      const errorMsg = err.message || String(err);
 
-      // ★ 修复：失败时 ROLLBACK 掉本轮 push 的 userMessage，
-      //   避免下次请求把它从 jsonl load 出来再 push 一遍
-      try {
-        if (messages && userMessage) {
-          const last = messages[messages.length - 1];
-          if (last && last.role === 'user' && last === userMessage) {
-            messages.pop();
+        // 打印错误信息
+        logger.error('[AgentChat] _processMessage error:', err.message);
+        if (err.cause) {
+          const cause = err.cause;
+          if (cause.issues) {
+            for (const issue of cause.issues.slice(0, 5)) {
+              logger.error(
+                `[AgentChat] schema issue: path=${JSON.stringify(issue.path)} code=${issue.code} message=${issue.message}`
+              );
+            }
+          } else {
+            logger.error(`[AgentChat] error cause: ${cause.message || String(cause)}`);
           }
         }
-      } catch { /* ignore */ }
+        const errorMsg = err.message || String(err);
 
-      return {
-        success: false,
-        message: 'AI 服务暂时不可用，请稍后重试。',
-        error: errorMsg,
-        stepCount: 0,
-      };
+        // ★ 失败时 ROLLBACK 掉本轮 push 的 userMessage
+        try {
+          if (messages && userMessage) {
+            const last = messages[messages.length - 1];
+            if (last && last.role === 'user' && last === userMessage) {
+              messages.pop();
+            }
+          }
+        } catch { /* ignore */ }
+
+        return {
+          success: false,
+          message: 'AI 服务暂时不可用，请稍后重试。',
+          error: errorMsg,
+          stepCount: 0,
+        };
+      }
     }
   }
 
@@ -1028,16 +1065,16 @@ class AgentChatHandler extends EventEmitter {
     const totalTokens = cachedMsgTokens + toolsTokens + systemPromptTokens;
     const limit = this._maxContextTokens * SIMPLE_COMPRESS_TOKEN_RATIO;
 
-    // logger.info(`BEFORE: messages=${messages.length}, tokens=${totalTokens}/${limit}`);
+    logger.info(`[_prepareSession] BEFORE: messages=${messages.length}, tokens=${totalTokens}/${limit}, threshold=${this._compressionMessageThreshold}`);
 
     if (totalTokens > limit || messages.length > this._compressionMessageThreshold) {
-      // logger.info(
-      //   `Context large (${messages.length} msgs, ${totalTokens}/${this._maxContextTokens} tokens), compressing...`
-      // );
+      logger.info(
+        `Context large (${messages.length} msgs, ${totalTokens}/${this._maxContextTokens} tokens), compressing...`
+      );
       await this._compressContext(sessionId, messages, messageStore);
       // 压缩后消息被截断/替换，缓存已失效，下次 _getCachedMessageTokens 会重新计算
       this._invalidateMessageTokensCache(messageStore);
-      logger.debug(`[_prepareSession] AFTER compress: messages=${messages.length}`);
+      logger.info(`[_prepareSession] AFTER compress: messages=${messages.length}`);
     }
 
     // 单次遍历完成：修复 invalid tool-call + 修复错误 text + 重映射历史 ID
@@ -1500,6 +1537,33 @@ class AgentChatHandler extends EventEmitter {
     return this._tokenCounter.countTools(this._toolExecutor.getAllTools());
   }
 
+  /**
+   * 判断是否为上下文超限错误
+   * @param {Error} err
+   * @returns {boolean}
+   * @private
+   */
+  _isContextOverflowError(err) {
+    if (!err) return false;
+    const msg = err.message || '';
+    const lowerMsg = msg.toLowerCase();
+    // DeepSeek / OpenAI 兼容的上下文超限错误关键词
+    const keywords = [
+      'context_length',
+      'context length',
+      'maximum context',
+      'max context',
+      'too many tokens',
+      'token limit',
+      'tokens exceed',
+      'exceed maximum',
+      'maximum tokens',
+      'context_exceeded',
+      'context overflow',
+    ];
+    return keywords.some(k => lowerMsg.includes(k));
+  }
+
   /**
    * 获取缓存的工具 token 数（工具列表不变时复用）
    * @private

package/src/common/constants.js

@@ -51,10 +51,10 @@ const DEFAULT_MAX_MESSAGES_PER_SESSION = 1000;
 // ============================================================================
 
 /** 保留最近的 N 条消息 */
-const KEEP_RECENT_MESSAGES = 50;
+const KEEP_RECENT_MESSAGES = 20;
 
 /** 压缩消息数量阈值 */
-const COMPRESSION_MESSAGE_THRESHOLD = 300;
+const COMPRESSION_MESSAGE_THRESHOLD = 80;
 
 /** 智能压缩启用 */
 const ENABLE_SMART_COMPRESS = true;

package/src/llm/tokens.js

@@ -5,7 +5,70 @@
  * Ported from pi's compaction/compaction.ts with enhancements
  */
 
-function encode(text, bytesPerToken = 4) {
+// CJK 字符 Unicode 范围
+const CJK_RANGES = [
+  [0x4e00, 0x9fff],   // CJK Unified Ideographs (中文简体/繁体)
+  [0x3400, 0x4dbf],   // CJK Unified Ideographs Extension A
+  [0x20000, 0x2a6df], // CJK Unified Ideographs Extension B
+  [0x3040, 0x309f],   // Hiragana (日文平假名)
+  [0x30a0, 0x30ff],   // Katakana (日文片假名)
+  [0xac00, 0xd7af],   // Hangul Syllables (韩文)
+  [0x1100, 0x11ff],   // Hangul Jamo
+  [0x3000, 0x303f],   // CJK Symbols and Punctuation
+  [0xff00, 0xffef],   // Halfwidth and Fullwidth Forms
+];
+
+function isCJKChar(char) {
+  const code = char.codePointAt(0);
+  if (!code) return false;
+  for (const [start, end] of CJK_RANGES) {
+    if (code >= start && code <= end) return true;
+  }
+  return false;
+}
+
+/**
+ * 改进的 Token 估算
+ * 根据文本语言类型采用不同估算系数
+ *
+ * @param {string} text - 输入文本
+ * @returns {number} 估算的 token 数
+ */
+function encode(text) {
+  if (!text || typeof text !== 'string') return 0;
+
+  let cjkChars = 0;
+  let asciiChars = 0;
+  let otherChars = 0;
+
+  // 逐字符统计
+  for (const char of text) {
+    if (isCJKChar(char)) {
+      cjkChars++;
+    } else if (char.charCodeAt(0) < 128) {
+      // ASCII 字符 (英文、数字、标点)
+      asciiChars++;
+    } else {
+      otherChars++;
+    }
+  }
+
+  // 更精确的估算系数 (基于 tiktoken 统计数据):
+  // - ASCII/英文: ~0.25 tokens/char (4 chars per token)
+  // - 中文: ~0.5 tokens/char (2 chars per token, 中文字符是 3 字节 UTF-8 但 tokenizer 输出约 0.5)
+  // - 其他 (emoji, 特殊符号): ~0.25 tokens/char
+  const asciiTokens = asciiChars * 0.25;
+  const cjkTokens = cjkChars * 0.5;
+  const otherTokens = otherChars * 0.25;
+
+  return Math.ceil(asciiTokens + cjkTokens + otherTokens);
+}
+
+/**
+ * 旧版 encode（兼容）
+ * @deprecated 使用新版 encode 替代
+ */
+function encodeOld(text, bytesPerToken = 4) {
   if (!text) return 0;
   const bytes = Buffer.byteLength(String(text), 'utf8');
   return Math.ceil(bytes / bytesPerToken);
@@ -25,64 +88,66 @@ function safeJsonStringify(value) {
   }
 }
 
+/**
+ * 估算单条消息的 token 数（使用改进的编码方法）
+ */
 function estimateTokens(message) {
-  let chars = 0;
+  if (!message) return 0;
+
+  let total = 4; // message overhead
 
   switch (message.role) {
-    case 'user': {
+    case 'user':
+    case 'assistant':
+    case 'custom':
+    case 'toolResult': {
       const content = message.content;
       if (typeof content === 'string') {
-        chars = content.length;
+        total += encode(content);
       } else if (Array.isArray(content)) {
         for (const block of content) {
           if (block.type === 'text' && block.text) {
-            chars += block.text.length;
+            total += encode(block.text);
+          } else if (block.type === 'tool-call' || block.type === 'toolCall') {
+            total += encode(block.name || '') + encode(safeJsonStringify(block.arguments || {}));
+          } else if (block.type === 'tool-result' || block.type === 'toolResult') {
+            total += encode(typeof block.content === 'string' ? block.content : safeJsonStringify(block.content));
+          } else if (block.type === 'image') {
+            total += 85;
+          } else if (block.type === 'thinking') {
+            total += encode(block.thinking || '');
           }
         }
       }
-      return Math.ceil(chars / 4);
-    }
-    case 'assistant': {
-      const assistant = message;
-      for (const block of assistant.content || []) {
-        if (block.type === 'text') {
-          chars += block.text.length;
-        } else if (block.type === 'thinking') {
-          chars += block.thinking.length;
-        } else if (block.type === 'toolCall') {
-          chars += block.name.length + safeJsonStringify(block.arguments).length;
-        }
-      }
-      return Math.ceil(chars / 4);
-    }
-    case 'custom':
-    case 'toolResult': {
-      if (typeof message.content === 'string') {
-        chars = message.content.length;
-      } else {
-        for (const block of message.content || []) {
-          if (block.type === 'text' && block.text) {
-            chars += block.text.length;
-          }
-          if (block.type === 'image') {
-            chars += 4800;
-          }
-        }
-      }
-      return Math.ceil(chars / 4);
+      break;
     }
     case 'bashExecution': {
-      chars = message.command.length + (message.output?.length || 0);
-      return Math.ceil(chars / 4);
+      total += encode(message.command || '');
+      total += encode(message.output || '');
+      break;
     }
     case 'branchSummary':
     case 'compactionSummary': {
-      chars = message.summary.length;
-      return Math.ceil(chars / 4);
+      total += encode(message.summary || '');
+      break;
+    }
+  }
+
+  // tool_calls overhead
+  if (message.tool_calls && Array.isArray(message.tool_calls)) {
+    total += 15; // tool_calls overhead
+    for (const tc of message.tool_calls) {
+      if (tc.function) {
+        total += encode(tc.function.name || '') + encode(safeJsonStringify(tc.function.arguments || {}));
+      }
     }
   }
 
-  return 0;
+  if (message.tool_call_id) {
+    total += 15;
+  }
+
+  return total;
 }
 
 function calculateContextTokens(usage) {
@@ -157,8 +222,8 @@ class TokenCounter {
     this.toolSchema = config.toolSchema || null;
   }
 
-  countText(text, bytesPerToken = 4) {
-    return encode(text, bytesPerToken);
+  countText(text) {
+    return encode(text);
   }
 
   countMessages(messages) {