@cicctencent/agent-core 1.0.17 → 1.0.18

package/dist/esm/utils/token-counter.js

@@ -1,18 +1,24 @@
 // ============================================================
-// Token 计数器 — 基于 tiktoken 的精确 token 计数
+// Token 计数器 - 基于 tiktoken 的精确 token 计数
 // ============================================================
 import { Tiktoken, encodingForModel } from 'js-tiktoken';
 /** 默认编码器（cl100k_base，兼容 GPT-4 / Claude 等主流模型） */
 let encoder = null;
+/** 字符估算伪编码器（tiktoken 不可用时兜底，约 4 字符 ≈ 1 token） */
+function createFallbackEncoder() {
+    return { encode: (text) => new Array(Math.ceil(text.length / 4)) };
+}
 /**
  * LRU 缓存：避免对相同文本重复计算 token
  * 缓存容量 2000 条，命中率高时可显著减少 encode 调用
+ * lastInsertedKey 追踪最近写入的 key，命中时若为同 key 则跳过 delete+set
  */
 const TOKEN_CACHE_MAX = 2000;
 const tokenCache = new Map();
+let lastInsertedKey;
 /**
  * 获取 tiktoken 编码器实例（延迟初始化）
- * P2 #13: 兼容 ESM 环境 — require 可能未定义，回退到字符估算
+ * 兼容 ESM 环境 - require 可能未定义，回退到字符估算
  */
 function getEncoder() {
     if (encoder)
@@ -22,21 +28,13 @@ function getEncoder() {
     }
     catch {
         try {
-            // P2 #13: ESM 兼容 — 尝试 require，失败则回退到字符估算
             const ranks = (typeof require !== 'undefined')
                 ? require('js-tiktoken/ranks/cl100k_base')
                 : undefined;
-            if (ranks) {
-                encoder = new Tiktoken(ranks);
-            }
-            else {
-                // ESM 环境无法加载 ranks — 使用伪 encoder 走字符估算
-                encoder = { encode: (text) => new Array(Math.ceil(text.length / 4)) };
-            }
+            encoder = ranks ? new Tiktoken(ranks) : createFallbackEncoder();
         }
         catch {
-            // 完全无法加载 tiktoken — 使用伪 encoder 走字符估算
-            encoder = { encode: (text) => new Array(Math.ceil(text.length / 4)) };
+            encoder = createFallbackEncoder();
         }
     }
     return encoder;
@@ -52,9 +50,12 @@ export function countTokens(text) {
     // 缓存查找
     const cached = tokenCache.get(text);
     if (cached !== undefined) {
-        // LRU: 删除旧位置并在末尾重新插入
-        tokenCache.delete(text);
-        tokenCache.set(text, cached);
+        // LRU: 仅当不是最近插入的条目时才移动到末尾（O(1) 判断，避免展开全部 keys）
+        if (lastInsertedKey !== text) {
+            tokenCache.delete(text);
+            tokenCache.set(text, cached);
+            lastInsertedKey = text;
+        }
         return cached;
     }
     const count = getEncoder().encode(text).length;
@@ -66,6 +67,7 @@ export function countTokens(text) {
             tokenCache.delete(oldestKey);
     }
     tokenCache.set(text, count);
+    lastInsertedKey = text;
     return count;
 }
 /**
@@ -75,6 +77,8 @@ export function countTokens(text) {
  * @returns 总 token 数量
  */
 export function countMessageTokens(messages) {
+    if (!messages || messages.length === 0)
+        return 0;
     let total = 0;
     for (const msg of messages) {
         // 每条消息 overhead: role + content 分隔
@@ -114,7 +118,7 @@ export function countMessageTokens(messages) {
  * - TextPart: 精确计数
  * - ImagePart: 按详情级别估算（参考 OpenAI Vision 定价）
  *   - low: ~85 tokens（512x512 固定）
- *   - high: ~170 tokens/tile（最多 4 tiles → ~680）
+ *   - high: ~170 tokens/tile（最多 4 tiles -> ~680）
  *   - auto: 默认按 low 估算
  * - FilePart: 计数文本内容
  */

package/dist/esm/utils/token-counter.js.map

@@ -1 +1 @@
-{"version":3,"file":"token-counter.js","sourceRoot":"","sources":["../../../src/utils/token-counter.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,uCAAuC;AACvC,+DAA+D;AAE/D,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGzD,iDAAiD;AACjD,IAAI,OAAO,GAAoB,IAAI,CAAC;AAEpC;;;GAGG;AACH,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE7C;;;GAGG;AACH,SAAS,UAAU;IACjB,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,IAAI,CAAC;QACH,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,yCAAyC;YACzC,MAAM,KAAK,GAAG,CAAC,OAAO,OAAO,KAAK,WAAW,CAAC;gBAC5C,CAAC,CAAC,OAAO,CAAC,+BAA+B,CAAC;gBAC1C,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,KAAK,EAAE,CAAC;gBACV,OAAO,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;iBAAM,CAAC;gBACN,uCAAuC;gBACvC,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAyB,CAAC;YACvG,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,sCAAsC;YACtC,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAyB,CAAC;QACvG,CAAC;IACH,CAAC;IACD,OAAO,OAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,OAAO;IACP,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,qBAAqB;QACrB,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACxB,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC7B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAE/C,cAAc;IACd,IAAI,UAAU,CAAC,IAAI,IAAI,eAAe,EAAE,CAAC;QACvC,2BAA2B;QAC3B,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACjD,IAAI,SAAS,KAAK,SAAS;YAAE,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IACD,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAE5B,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAmB;IACpD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,mCAAmC;QACnC,KAAK,IAAI,CAAC,CAAC;QACX,OAAO;QACP,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACrC,UAAU;QACV,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACpC,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sBAAsB;gBACtB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;oBAC/B,KAAK,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;QACH,CAAC;QACD,6BAA6B;QAC7B,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,KAAK,MAAM,EAAE,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;gBAChC,KAAK,IAAI,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;gBAC9C,KAAK,IAAI,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QACD,+BAA+B;QAC/B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACb,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,2BAA2B;IAC3B,KAAK,IAAI,CAAC,CAAC;IACX,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,sBAAsB,CAAC,IAAiB;IAC/C,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,MAAM;YACT,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,KAAK,OAAO;YACV,sDAAsD;YACtD,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,GAAG,CAAC;YACvC,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM;YACT,OAAO,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChE;YACE,OAAO,CAAC,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}
+{"version":3,"file":"token-counter.js","sourceRoot":"","sources":["../../../src/utils/token-counter.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,uCAAuC;AACvC,+DAA+D;AAE/D,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAGzD,iDAAiD;AACjD,IAAI,OAAO,GAAoB,IAAI,CAAC;AAEpC,iDAAiD;AACjD,SAAS,qBAAqB;IAC5B,OAAO,EAAE,MAAM,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAyB,CAAC;AACpG,CAAC;AAED;;;;GAIG;AACH,MAAM,eAAe,GAAG,IAAI,CAAC;AAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;AAC7C,IAAI,eAAmC,CAAC;AAExC;;;GAGG;AACH,SAAS,UAAU;IACjB,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,IAAI,CAAC;QACH,OAAO,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,CAAC,OAAO,OAAO,KAAK,WAAW,CAAC;gBAC5C,CAAC,CAAC,OAAO,CAAC,+BAA+B,CAAC;gBAC1C,CAAC,CAAC,SAAS,CAAC;YACd,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,qBAAqB,EAAE,CAAC;QAClE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,qBAAqB,EAAE,CAAC;QACpC,CAAC;IACH,CAAC;IACD,OAAO,OAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,OAAO;IACP,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACpC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,+CAA+C;QAC/C,IAAI,eAAe,KAAK,IAAI,EAAE,CAAC;YAC7B,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACxB,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7B,eAAe,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,MAAM,KAAK,GAAG,UAAU,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAE/C,cAAc;IACd,IAAI,UAAU,CAAC,IAAI,IAAI,eAAe,EAAE,CAAC;QACvC,2BAA2B;QAC3B,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC;QACjD,IAAI,SAAS,KAAK,SAAS;YAAE,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IAC5D,CAAC;IACD,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC5B,eAAe,GAAG,IAAI,CAAC;IAEvB,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAmB;IACpD,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;QAC3B,mCAAmC;QACnC,KAAK,IAAI,CAAC,CAAC;QACX,OAAO;QACP,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QACrC,UAAU;QACV,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACpC,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,sBAAsB;gBACtB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;oBAC/B,KAAK,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;QACH,CAAC;QACD,6BAA6B;QAC7B,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YACnB,KAAK,MAAM,EAAE,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;gBAChC,KAAK,IAAI,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;gBAC9C,KAAK,IAAI,WAAW,CAAC,EAAE,CAAC,QAAQ,EAAE,SAAS,IAAI,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QACD,+BAA+B;QAC/B,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACb,KAAK,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,2BAA2B;IAC3B,KAAK,IAAI,CAAC,CAAC;IACX,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,sBAAsB,CAAC,IAAiB;IAC/C,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,MAAM;YACT,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,KAAK,OAAO;YACV,sDAAsD;YACtD,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,GAAG,CAAC;YACvC,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM;YACT,OAAO,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChE;YACE,OAAO,CAAC,CAAC;IACb,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cicctencent/agent-core",
-  "version": "1.0.17",
+  "version": "1.0.18",
   "description": "Core engine for AI agent — LLM orchestration, tool execution, MCP, skill routing, security, and memory pipeline.",
   "type": "module",
   "main": "./dist/cjs/index.js",
@@ -24,16 +24,19 @@
     "build:cjs": "tsc -p tsconfig.cjs.json",
     "build:fixup": "node scripts/fixup-cjs.js",
     "typecheck": "tsc -p tsconfig.esm.json --noEmit",
-    "test": "npm run build && node --test scripts/test-dsml.mjs",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "axios": "^1.18.1",
+    "axios": "^1.19.0",
     "js-tiktoken": "^1.0.21"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",
-    "typescript": "~5.8.3"
+    "typescript": "~5.8.3",
+    "vitest": "^2.1.9"
   },
   "engines": {
     "node": ">=22.17.0"