foliko 1.1.67 → 1.1.69

package/src/core/token-counter.js

@@ -1,10 +1,6 @@
 /**
  * TokenCounter - Token 计算工具
- *
- * 职责：
- * 1. 计算文本的 token 数量
- * 2. 计算消息数组的 token 总数
- * 3. 计算工具定义的 token 总数
+ * Ported from pi's compaction/compaction.ts with enhancements
  */
 
 /**
@@ -22,56 +18,191 @@ function encode(text, bytesPerToken = 4) {
 
 /**
  * 计算 JSON 字符串的 token 数量
- * @param {string} text - JSON 字符串
- * @returns {number} token 数量
  */
 function encodeForJSON(text) {
   if (!text) return 0;
-  // JSON 字符串需要额外计算引号和转义
   const encoded = encode(text);
   return encoded + Math.ceil(Buffer.byteLength(JSON.stringify(text), 'utf8') / 100);
 }
 
+/**
+ * Safe JSON stringify that handles unserializable values
+ */
+function safeJsonStringify(value) {
+  try {
+    return JSON.stringify(value) ?? 'undefined';
+  } catch {
+    return '[unserializable]';
+  }
+}
+
+/**
+ * Estimate token count for one message using a conservative character heuristic
+ * Ported from pi
+ */
+function estimateTokens(message) {
+  let chars = 0;
+
+  switch (message.role) {
+    case 'user': {
+      const content = message.content;
+      if (typeof content === 'string') {
+        chars = content.length;
+      } else if (Array.isArray(content)) {
+        for (const block of content) {
+          if (block.type === 'text' && block.text) {
+            chars += block.text.length;
+          }
+        }
+      }
+      return Math.ceil(chars / 4);
+    }
+    case 'assistant': {
+      const assistant = message;
+      for (const block of assistant.content || []) {
+        if (block.type === 'text') {
+          chars += block.text.length;
+        } else if (block.type === 'thinking') {
+          chars += block.thinking.length;
+        } else if (block.type === 'toolCall') {
+          chars += block.name.length + safeJsonStringify(block.arguments).length;
+        }
+      }
+      return Math.ceil(chars / 4);
+    }
+    case 'custom':
+    case 'toolResult': {
+      if (typeof message.content === 'string') {
+        chars = message.content.length;
+      } else {
+        for (const block of message.content || []) {
+          if (block.type === 'text' && block.text) {
+            chars += block.text.length;
+          }
+          if (block.type === 'image') {
+            chars += 4800;
+          }
+        }
+      }
+      return Math.ceil(chars / 4);
+    }
+    case 'bashExecution': {
+      chars = message.command.length + (message.output?.length || 0);
+      return Math.ceil(chars / 4);
+    }
+    case 'branchSummary':
+    case 'compactionSummary': {
+      chars = message.summary.length;
+      return Math.ceil(chars / 4);
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Calculate total context tokens from provider usage
+ */
+function calculateContextTokens(usage) {
+  return usage.totalTokens || (usage.input || 0) + (usage.output || 0) + (usage.cacheRead || 0) + (usage.cacheWrite || 0);
+}
+
+/**
+ * Get assistant usage from message
+ */
+function getAssistantUsage(msg) {
+  if (msg.role === 'assistant' && msg.usage) {
+    if (msg.stopReason !== 'aborted' && msg.stopReason !== 'error' && msg.usage) {
+      return msg.usage;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Return usage from the last successful assistant message in session entries
+ */
+function getLastAssistantUsage(entries) {
+  for (let i = entries.length - 1; i >= 0; i--) {
+    const entry = entries[i];
+    if (entry.type === 'message') {
+      const usage = getAssistantUsage(entry.message);
+      if (usage) return usage;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Get last assistant usage info from messages
+ */
+function getLastAssistantUsageInfo(messages) {
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const usage = getAssistantUsage(messages[i]);
+    if (usage) return { usage, index: i };
+  }
+  return undefined;
+}
+
+/**
+ * Estimate context tokens for messages using provider usage when available
+ */
+function estimateContextTokens(messages) {
+  const usageInfo = getLastAssistantUsageInfo(messages);
+
+  if (!usageInfo) {
+    let estimated = 0;
+    for (const message of messages) {
+      estimated += estimateTokens(message);
+    }
+    return {
+      tokens: estimated,
+      usageTokens: 0,
+      trailingTokens: estimated,
+      lastUsageIndex: null
+    };
+  }
+
+  const usageTokens = calculateContextTokens(usageInfo.usage);
+  let trailingTokens = 0;
+  for (let i = usageInfo.index + 1; i < messages.length; i++) {
+    trailingTokens += estimateTokens(messages[i]);
+  }
+
+  return {
+    tokens: usageTokens + trailingTokens,
+    usageTokens,
+    trailingTokens,
+    lastUsageIndex: usageInfo.index
+  };
+}
+
+/**
+ * Return whether context usage exceeds the configured compaction threshold
+ */
+function shouldCompact(contextTokens, contextWindow, settings) {
+  if (!settings.enabled) return false;
+  return contextTokens > contextWindow - settings.reserveTokens;
+}
+
 class TokenCounter {
-  /**
-   * @param {Object} config - 配置
-   * @param {Object} config.toolSchema - 工具 schema（用于工具 token 计算）
-   */
   constructor(config = {}) {
     this.toolSchema = config.toolSchema || null;
   }
 
-  /**
-   * 计算文本 token
-   * @param {string} text - 文本
-   * @param {number} bytesPerToken - 每 token 字节数
-   * @returns {number} token 数量
-   */
   countText(text, bytesPerToken = 4) {
     return encode(text, bytesPerToken);
   }
 
-  /**
-   * 计算消息数组的 token 总数
-   * @param {Array} messages - 消息数组
-   * @returns {number} token 总数
-   */
   countMessages(messages) {
     if (!Array.isArray(messages)) return 0;
     return messages.reduce((sum, msg) => sum + this.countMessage(msg), 0);
   }
 
-  /**
-   * 计算单条消息的 token
-   * @param {Object} msg - 消息
-   * @returns {number} token 数量
-   */
   countMessage(msg) {
     if (!msg) return 0;
 
     let total = 0;
-
-    // 角色和格式开销
     total += 4;
 
     if (typeof msg.content === 'string') {
@@ -82,9 +213,8 @@ class TokenCounter {
       }
     }
 
-    // tool_calls 开销
     if (msg.tool_calls && Array.isArray(msg.tool_calls)) {
-      total += 15; // overhead per tool_calls block
+      total += 15;
       for (const tc of msg.tool_calls) {
         if (tc.function) {
           total += this.countText(tc.function.name) + this.countText(tc.function.arguments);
@@ -92,7 +222,6 @@ class TokenCounter {
       }
     }
 
-    // tool_call_id 开销
     if (msg.tool_call_id) {
       total += 15;
     }
@@ -100,11 +229,6 @@ class TokenCounter {
     return total;
   }
 
-  /**
-   * 计算 content block 的 token
-   * @param {Object} block - content block
-   * @returns {number} token 数量
-   */
   countContentBlock(block) {
     if (!block) return 0;
 
@@ -126,34 +250,23 @@ class TokenCounter {
         }
         return 0;
       case 'image':
-        // 图片按 token 估算
         return 85;
       default:
         return this.countText(JSON.stringify(block));
     }
   }
 
-  /**
-   * 计算工具定义的 token 总数
-   * @param {Array} tools - 工具数组
-   * @returns {number} token 总数
-   */
   countTools(tools) {
     if (!tools || !Array.isArray(tools)) return 0;
 
     let total = 0;
     for (const tool of tools) {
-      // 工具名和描述
-      total += 20; // overhead
-
+      total += 20;
       if (tool.description) {
         total += this.countText(tool.description);
       }
-
-      // 参数
       if (tool.inputSchema) {
-        const schema =
-          tool.inputSchema.jsonSchema || tool.inputSchema.inputSchema || tool.inputSchema;
+        const schema = tool.inputSchema.jsonSchema || tool.inputSchema.inputSchema || tool.inputSchema;
         if (schema.properties) {
           for (const [name, prop] of Object.entries(schema.properties)) {
             total += this.countText(name) + 10;
@@ -168,11 +281,6 @@ class TokenCounter {
     return total;
   }
 
-  /**
-   * 估算完整请求的 token（消息 + 工具 + 系统提示）
-   * @param {Object} params - 请求参数
-   * @returns {Object} - { messagesTokens, toolsTokens, systemPromptTokens, total }
-   */
   estimateRequest({ messages, tools, systemPrompt }) {
     const messagesTokens = this.countMessages(messages);
     const toolsTokens = this.countTools(tools);
@@ -182,9 +290,20 @@ class TokenCounter {
       messagesTokens,
       toolsTokens,
       systemPromptTokens,
-      total: messagesTokens + toolsTokens + systemPromptTokens,
+      total: messagesTokens + toolsTokens + systemPromptTokens
     };
   }
 }
 
-module.exports = { TokenCounter, encode, encodeForJSON };
+module.exports = {
+  TokenCounter,
+  encode,
+  encodeForJSON,
+  estimateTokens,
+  calculateContextTokens,
+  getAssistantUsage,
+  getLastAssistantUsage,
+  estimateContextTokens,
+  shouldCompact,
+  safeJsonStringify
+};

package/src/core/tool-executor.js

@@ -9,6 +9,7 @@
 
 const { EventEmitter } = require('../utils/event-emitter');
 const { logger } = require('../utils/logger');
+const { validateToolCalls } = require('../utils/message-validator');
 
 class ToolExecutor extends EventEmitter {
   /**
@@ -183,76 +184,7 @@ class ToolExecutor extends EventEmitter {
    * @returns {Array} 验证后的消息
    */
   validateToolCalls(messages) {
-    let fixedCount = 0;
-    // 收集被跳过的 toolCallId，用于清理对应的 tool-result
-    const invalidatedToolCallIds = new Set();
-
-    for (const msg of messages) {
-      // 清理 assistant 消息中的不完整 tool-call
-      if (msg.role === 'assistant' && Array.isArray(msg.content)) {
-        for (const item of msg.content) {
-          // 兼容 tool-call 和 tool-use 两种类型
-          if (item.type !== 'tool-call' && item.type !== 'tool-use') {
-            continue;
-          }
-
-          const input = item.input;
-          if (typeof input !== 'string') {
-            continue;
-          }
-
-          // 检查 input 是否是有效的 JSON（不是不完整的）
-          const trimmed = input.trim();
-          if (trimmed === '{' || trimmed === '' || !trimmed.startsWith('{')) {
-            // 不完整的 JSON，移除这个 tool-call
-            // 记录 toolCallId，以便后续清理对应的 tool-result
-            if (item.toolCallId) {
-              invalidatedToolCallIds.add(item.toolCallId);
-            }
-            logger.warn(
-              `_validateToolCalls: invalid tool-call input="${input}", toolCallId=${item.toolCallId}, converting to text`
-            );
-            item.type = 'text';
-            item.text = `（工具调用 ${item.toolName} 参数不完整，已跳过）`;
-            delete item.toolCallId;
-            delete item.toolName;
-            delete item.input;
-            fixedCount++;
-          }
-        }
-      }
-    }
-
-    // 如果有无效的 tool-call，清理对应的 tool-result
-    if (invalidatedToolCallIds.size > 0) {
-      logger.warn(
-        `_validateToolCalls: removing ${invalidatedToolCallIds.size} tool-results with invalidated toolCallIds`
-      );
-      for (const msg of messages) {
-        if (msg.role === 'tool' && Array.isArray(msg.content)) {
-          // 过滤掉引用了无效 toolCallId 的 tool-result
-          const oldLen = msg.content.length;
-          msg.content = msg.content.filter((item) => {
-            if (item.type !== 'tool-result' && item.type !== 'tool_result') {
-              return true;
-            }
-            // 如果 tool-result 引用的 toolCallId 已被标记为无效，则移除
-            if (item.toolCallId && invalidatedToolCallIds.has(item.toolCallId)) {
-              logger.warn(
-                `_validateToolCalls: removing orphaned tool-result with toolCallId=${item.toolCallId}`
-              );
-              fixedCount++;
-              return false;
-            }
-            return true;
-          });
-        }
-      }
-    }
-
-    if (fixedCount > 0) {
-      logger.info(`_validateToolCalls: Fixed ${fixedCount} incomplete tool calls/results`);
-    }
+    return validateToolCalls(messages);
   }
 
   /**

package/src/core/ui-extension-context.js

@@ -0,0 +1,174 @@
+/**
+ * ExtensionUIContext - UI Extension Context for Plugins
+ * 允许插件扩展 TUI 界面
+ */
+
+const { logger } = require('../utils/logger');
+
+class ExtensionUIContext {
+  /**
+   * @param {Object} chatUI - ChatUI 实例
+   * @param {string} pluginName - 插件名称
+   */
+  constructor(chatUI, pluginName) {
+    this._ui = chatUI;
+    this._pluginName = pluginName;
+    this._components = new Map();
+  }
+
+  /**
+   * 显示选择对话框
+   * @param {string} title - 对话框标题
+   * @param {Array<{label: string, value: any}>} options - 选项
+   * @returns {Promise<any>} 选中的值
+   */
+  async select(title, options) {
+    return new Promise((resolve, reject) => {
+      // 简化实现：使用 TUI 的 prompt 机制
+      if (!this._ui || !this._ui.tui) {
+        reject(new Error('ChatUI not available'));
+        return;
+      }
+
+      // 构建选项文本
+      let prompt = `${title}\n`;
+      options.forEach((opt, i) => {
+        prompt += `${i + 1}. ${opt.label}\n`;
+      });
+      prompt += '请选择: ';
+
+      // 创建一个简单的选择机制
+      this._ui.editor.setPrompt(prompt);
+      this._ui.editor.once('submit', (value) => {
+        const idx = parseInt(value, 10) - 1;
+        if (idx >= 0 && idx < options.length) {
+          resolve(options[idx].value);
+        } else {
+          resolve(options[0]?.value);
+        }
+      });
+
+      this._ui.tui.setFocus(this._ui.editor);
+    });
+  }
+
+  /**
+   * 显示确认对话框
+   * @param {string} message - 确认消息
+   * @returns {Promise<boolean>} 是否确认
+   */
+  async confirm(message) {
+    const result = await this.select(message, [
+      { label: '是', value: true },
+      { label: '否', value: false },
+    ]);
+    return result;
+  }
+
+  /**
+   * 获取文本输入
+   * @param {string} prompt - 输入提示
+   * @returns {Promise<string>} 用户输入
+   */
+  async input(prompt) {
+    return new Promise((resolve, reject) => {
+      if (!this._ui || !this._ui.editor) {
+        reject(new Error('ChatUI not available'));
+        return;
+      }
+
+      const originalPrompt = this._ui.editor.getPrompt?.() || '';
+      this._ui.editor.setPrompt(prompt);
+
+      const handler = (value) => {
+        this._ui.editor.off('submit', handler);
+        if (originalPrompt) {
+          this._ui.editor.setPrompt(originalPrompt);
+        }
+        resolve(value);
+      };
+
+      this._ui.editor.once('submit', handler);
+      this._ui.tui.setFocus(this._ui.editor);
+    });
+  }
+
+  /**
+   * 显示通知
+   * @param {string} title - 通知标题
+   * @param {string} message - 通知消息
+   * @param {number} [duration=3000] - 显示时长(ms)
+   */
+  notify(title, message, duration = 3000) {
+    if (this._ui && this._ui.statusBar) {
+      this._ui.statusBar.tooler?.show(`🔔 ${title}: ${message}`);
+      setTimeout(() => {
+        this._ui.statusBar.tooler?.hide();
+      }, duration);
+    } else {
+      logger.info(`[Notification] ${title}: ${message}`);
+    }
+  }
+
+  /**
+   * 设置状态栏
+   * @param {string} status - 状态文本
+   */
+  setStatus(status) {
+    if (this._ui && this._ui.statusBar) {
+      this._ui.statusBar.tooler?.show(status);
+    } else {
+      logger.debug(`[Status] ${status}`);
+    }
+  }
+
+  /**
+   * 添加自定义组件
+   * @param {string} id - 组件标识
+   * @param {Object} component - TUI 组件
+   */
+  addComponent(id, component) {
+    if (this._ui && this._ui.tui) {
+      this._ui.tui.addChild(component);
+      this._components.set(id, component);
+    }
+    return this;
+  }
+
+  /**
+   * 移除自定义组件
+   */
+  removeComponent(id) {
+    const comp = this._components.get(id);
+    if (comp && this._ui && this._ui.tui) {
+      this._ui.tui.removeChild(comp);
+      this._components.delete(id);
+    }
+    return this;
+  }
+
+  /**
+   * 获取 TUI 实例（用于更高级的定制）
+   */
+  getTUI() {
+    return this._ui?.tui || null;
+  }
+
+  /**
+   * 在消息区域显示消息
+   */
+  showMessage(message, type = 'info') {
+    if (this._ui) {
+      const colors = {
+        info: 'cyan',
+        success: 'green',
+        warning: 'yellow',
+        error: 'red',
+      };
+      const color = colors[type] || 'cyan';
+      this._ui.create_message(`[${this._pluginName}] ${message}\n`, chalk?.[color] || ((t) => t));
+    }
+  }
+}
+
+module.exports = { ExtensionUIContext };