@octopi-agent/engine 0.98.4 → 0.102.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-schema/agent.d.ts +35 -5
- package/dist/config-schema/agent.js +24 -4
- package/dist/config-schema/context.d.ts +6 -36
- package/dist/config-schema/context.js +4 -28
- package/dist/config-schema/engine.d.ts +57 -160
- package/dist/config-schema/engine.js +1 -5
- package/dist/config-schema/governance.d.ts +8 -1
- package/dist/config-schema/governance.js +8 -1
- package/dist/config-schema/models.d.ts +2 -8
- package/dist/config-schema/models.js +0 -2
- package/dist/config-schema/run.d.ts +2 -2
- package/dist/config-schema/run.js +2 -2
- package/dist/config-schema/shared.d.ts +0 -3
- package/dist/config-schema/shared.js +0 -2
- package/dist/config-schema/substrate.d.ts +14 -88
- package/dist/config-schema/substrate.js +22 -107
- package/dist/config.d.ts +45 -113
- package/dist/config.js +8 -2
- package/dist/harness/activation/dispatcher.js +7 -0
- package/dist/harness/activation/inbox.d.ts +3 -3
- package/dist/harness/activation/inbox.js +3 -3
- package/dist/harness/activation/runtime.js +13 -5
- package/dist/harness/activation/types.d.ts +2 -2
- package/dist/harness/agent/builder.d.ts +24 -3
- package/dist/harness/agent/builder.js +131 -27
- package/dist/harness/agent/config-bridge.js +9 -2
- package/dist/harness/agent/config-integrity.js +19 -2
- package/dist/harness/agent/wire-agent-builder.d.ts +11 -3
- package/dist/harness/agent/wire-agent-builder.js +26 -7
- package/dist/harness/capabilities/document/factory.d.ts +5 -0
- package/dist/harness/capabilities/document/factory.js +1 -0
- package/dist/harness/capabilities/document/port.js +6 -3
- package/dist/harness/capabilities/document/types.d.ts +5 -0
- package/dist/harness/collaboration/autonomous-subsystem/loader.js +1 -1
- package/dist/harness/collaboration/autonomous-subsystem/sense/engine.d.ts +1 -1
- package/dist/harness/collaboration/autonomous-subsystem/sense/engine.js +2 -2
- package/dist/harness/collaboration/autonomous-subsystem/sense/input-builder.js +4 -4
- package/dist/harness/collaboration/autonomous-subsystem/sense/metrics.d.ts +2 -2
- package/dist/harness/collaboration/autonomous-subsystem/sense/metrics.js +2 -2
- package/dist/harness/collaboration/autonomous-subsystem/signal/bus.d.ts +6 -0
- package/dist/harness/collaboration/autonomous-subsystem/signal/bus.js +25 -4
- package/dist/harness/collaboration/autonomous-subsystem/think/executor.js +1 -1
- package/dist/harness/collaboration/autonomous-subsystem/types.d.ts +3 -3
- package/dist/harness/collaboration/delegation/child-run.d.ts +1 -1
- package/dist/harness/collaboration/delegation/child-run.js +4 -4
- package/dist/harness/collaboration/delegation/index.d.ts +1 -1
- package/dist/harness/collaboration/delegation/index.js +1 -1
- package/dist/harness/collaboration/delegation/notifier.d.ts +6 -1
- package/dist/harness/collaboration/delegation/notifier.js +8 -1
- package/dist/harness/collaboration/delegation/service.d.ts +2 -0
- package/dist/harness/collaboration/delegation/service.js +2 -1
- package/dist/harness/collaboration/delegation/store.d.ts +2 -0
- package/dist/harness/collaboration/delegation/store.js +2 -1
- package/dist/harness/collaboration/delegation/types.d.ts +23 -2
- package/dist/harness/collaboration/delegation/types.js +22 -0
- package/dist/harness/collaboration/delegation/wiring.js +11 -1
- package/dist/harness/context/budget-allocator.d.ts +26 -18
- package/dist/harness/context/budget-allocator.js +67 -32
- package/dist/harness/context/capabilities/compact/engine.d.ts +0 -6
- package/dist/harness/context/capabilities/compact/engine.js +169 -121
- package/dist/harness/context/capabilities/compact/handover.d.ts +5 -3
- package/dist/harness/context/capabilities/compact/handover.js +7 -5
- package/dist/harness/context/capabilities/compact/index.d.ts +5 -1
- package/dist/harness/context/capabilities/compact/index.js +4 -0
- package/dist/harness/context/capabilities/compact/mechanical.d.ts +130 -0
- package/dist/harness/context/capabilities/compact/mechanical.js +268 -0
- package/dist/harness/context/capabilities/compact/messages.d.ts +60 -0
- package/dist/harness/context/capabilities/compact/messages.js +91 -0
- package/dist/harness/context/capabilities/compact/types.d.ts +76 -11
- package/dist/harness/context/capabilities/compact/types.js +11 -1
- package/dist/harness/context/capabilities/compact/view-split.d.ts +36 -0
- package/dist/harness/context/capabilities/compact/view-split.js +72 -0
- package/dist/harness/context/capabilities/summary/port.js +0 -1
- package/dist/harness/context/capabilities/summary/resolver.d.ts +1 -1
- package/dist/harness/context/capabilities/summary/resolver.js +1 -6
- package/dist/harness/context/capabilities/summary/tool-binding.d.ts +1 -1
- package/dist/harness/context/capabilities/summary/types.d.ts +3 -6
- package/dist/harness/context/context-size-ledger.d.ts +143 -0
- package/dist/harness/context/context-size-ledger.js +236 -0
- package/dist/harness/context/default-context-engine.d.ts +195 -119
- package/dist/harness/context/default-context-engine.js +1241 -743
- package/dist/harness/context/index.d.ts +2 -2
- package/dist/harness/context/index.js +1 -1
- package/dist/harness/context/layer-types.d.ts +15 -0
- package/dist/harness/context/layer-types.js +14 -1
- package/dist/harness/context/layers.d.ts +14 -15
- package/dist/harness/context/layers.js +41 -11
- package/dist/harness/context/message-selector.d.ts +10 -8
- package/dist/harness/context/message-selector.js +49 -27
- package/dist/harness/context/summarize.d.ts +2 -2
- package/dist/harness/context/summarize.js +9 -2
- package/dist/harness/context/system-prompt-assembler.d.ts +2 -0
- package/dist/harness/context/system-prompt-assembler.js +2 -1
- package/dist/harness/context/token-constants.d.ts +8 -6
- package/dist/harness/context/token-constants.js +8 -7
- package/dist/harness/context/token-estimate-fns.d.ts +37 -27
- package/dist/harness/context/token-estimate-fns.js +85 -66
- package/dist/harness/context/token-estimator.d.ts +27 -12
- package/dist/harness/context/token-estimator.js +120 -66
- package/dist/harness/context/token-segments.d.ts +88 -0
- package/dist/harness/context/token-segments.js +103 -0
- package/dist/harness/context/tool-result-aging.d.ts +6 -1
- package/dist/harness/context/tool-result-aging.js +16 -5
- package/dist/harness/context/tool-result-body.js +2 -0
- package/dist/harness/context/transform/anchors.d.ts +66 -0
- package/dist/harness/context/transform/anchors.js +125 -0
- package/dist/harness/context/transform/fold.d.ts +56 -0
- package/dist/harness/context/transform/fold.js +135 -0
- package/dist/harness/context/transform/migrate.d.ts +41 -0
- package/dist/harness/context/transform/migrate.js +69 -0
- package/dist/harness/context/transform/types.d.ts +129 -0
- package/dist/harness/context/transform/types.js +19 -0
- package/dist/harness/context/turn-envelope.d.ts +6 -3
- package/dist/harness/context/types.d.ts +127 -117
- package/dist/harness/extension/execution-environment/ops-audit.d.ts +1 -1
- package/dist/harness/extension/execution-environment/resource-limits.d.ts +1 -1
- package/dist/harness/extension/execution-environment/resource-limits.js +1 -1
- package/dist/harness/extension/plugin-ecosystem/channel-types.d.ts +1 -1
- package/dist/harness/extension/plugin-ecosystem/commands/index.d.ts +3 -1
- package/dist/harness/extension/plugin-ecosystem/commands/index.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/commands/plugin-bridge.d.ts +12 -1
- package/dist/harness/extension/plugin-ecosystem/commands/plugin-bridge.js +14 -4
- package/dist/harness/extension/plugin-ecosystem/commands/router.d.ts +5 -4
- package/dist/harness/extension/plugin-ecosystem/commands/router.js +23 -14
- package/dist/harness/extension/plugin-ecosystem/commands/types.d.ts +8 -1
- package/dist/harness/extension/plugin-ecosystem/commands/types.js +2 -1
- package/dist/harness/extension/plugin-ecosystem/hooks/adapter.d.ts +54 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/adapter.js +397 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/compaction-integration.d.ts +16 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/compaction-integration.js +66 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/index.d.ts +4 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/index.js +2 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/iteration-snapshot.d.ts +11 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/iteration-snapshot.js +54 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/journal.d.ts +9 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/journal.js +22 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/policy-error.d.ts +12 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/policy-error.js +19 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/run-integration.d.ts +36 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/run-integration.js +338 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/services.d.ts +24 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/services.js +139 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/types.d.ts +229 -0
- package/dist/harness/extension/plugin-ecosystem/hooks/types.js +1 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/api.d.ts +83 -249
- package/dist/harness/extension/plugin-ecosystem/plugins/api.js +88 -214
- package/dist/harness/extension/plugin-ecosystem/plugins/contribution-registry.d.ts +43 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/contribution-registry.js +61 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/engine-compatibility.d.ts +4 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/engine-compatibility.js +18 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/entry.d.ts +7 -85
- package/dist/harness/extension/plugin-ecosystem/plugins/entry.js +8 -50
- package/dist/harness/extension/plugin-ecosystem/plugins/index.d.ts +6 -23
- package/dist/harness/extension/plugin-ecosystem/plugins/index.js +3 -23
- package/dist/harness/extension/plugin-ecosystem/plugins/loader.d.ts +50 -117
- package/dist/harness/extension/plugin-ecosystem/plugins/loader.js +316 -285
- package/dist/harness/extension/plugin-ecosystem/plugins/manager.d.ts +23 -490
- package/dist/harness/extension/plugin-ecosystem/plugins/manager.js +85 -353
- package/dist/harness/extension/plugin-ecosystem/plugins/manifest.d.ts +41 -223
- package/dist/harness/extension/plugin-ecosystem/plugins/manifest.js +138 -19
- package/dist/harness/extension/plugin-ecosystem/plugins/staging.d.ts +15 -0
- package/dist/harness/extension/plugin-ecosystem/plugins/staging.js +15 -0
- package/dist/harness/extension/plugin-ecosystem/skills/manager.d.ts +6 -1
- package/dist/harness/extension/plugin-ecosystem/skills/manager.js +17 -1
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.d.ts +3 -0
- package/dist/harness/extension/plugin-ecosystem/tools/ask-user.js +46 -21
- package/dist/harness/extension/plugin-ecosystem/tools/delegation.js +18 -8
- package/dist/harness/extension/plugin-ecosystem/tools/plugin-tool.d.ts +11 -0
- package/dist/harness/extension/plugin-ecosystem/tools/plugin-tool.js +39 -0
- package/dist/harness/extension/plugin-ecosystem/tools/tool-bus.d.ts +1 -1
- package/dist/harness/extension/plugin-ecosystem/tools/tool-bus.js +11 -2
- package/dist/harness/governance/accounting/usage-ledger.d.ts +13 -1
- package/dist/harness/governance/accounting/usage-ledger.js +23 -1
- package/dist/harness/governance/human-in-the-loop/decision-cache.d.ts +3 -3
- package/dist/harness/governance/human-in-the-loop/decision-cache.js +2 -2
- package/dist/harness/governance/security/capability-enforcer.js +1 -2
- package/dist/harness/index.d.ts +12 -12
- package/dist/harness/index.js +7 -7
- package/dist/harness/knowledge/boot-config.d.ts +55 -0
- package/dist/harness/knowledge/boot-config.js +50 -0
- package/dist/harness/knowledge/catalog-types.d.ts +2 -0
- package/dist/harness/knowledge/client.d.ts +24 -2
- package/dist/harness/knowledge/client.js +19 -5
- package/dist/harness/knowledge/db.js +5 -0
- package/dist/harness/knowledge/describe-sample.d.ts +23 -0
- package/dist/harness/knowledge/describe-sample.js +58 -0
- package/dist/harness/knowledge/describe.d.ts +8 -0
- package/dist/harness/knowledge/describe.js +18 -2
- package/dist/harness/knowledge/engine-thread.js +6 -0
- package/dist/harness/knowledge/grounding.js +3 -1
- package/dist/harness/knowledge/http-app.js +35 -4
- package/dist/harness/knowledge/http-params.d.ts +18 -0
- package/dist/harness/knowledge/http-params.js +24 -0
- package/dist/harness/knowledge/index.d.ts +2 -0
- package/dist/harness/knowledge/index.js +1 -0
- package/dist/harness/knowledge/knowledge-serve-child.js +3 -0
- package/dist/harness/knowledge/purge.js +5 -2
- package/dist/harness/knowledge/query-service.d.ts +13 -2
- package/dist/harness/knowledge/query-service.js +4 -2
- package/dist/harness/knowledge/query-worker-client.d.ts +9 -1
- package/dist/harness/knowledge/query-worker-client.js +4 -2
- package/dist/harness/knowledge/query-worker.js +4 -2
- package/dist/harness/knowledge/read-http.js +5 -1
- package/dist/harness/knowledge/serve.d.ts +8 -7
- package/dist/harness/knowledge/serve.js +9 -0
- package/dist/harness/knowledge/source-store.js +26 -3
- package/dist/harness/knowledge/start-service-process.d.ts +15 -7
- package/dist/harness/knowledge/start-service-process.js +21 -9
- package/dist/harness/knowledge/types.d.ts +7 -0
- package/dist/harness/knowledge/writer-service.d.ts +37 -9
- package/dist/harness/knowledge/writer-service.js +120 -14
- package/dist/harness/knowledge/writer-worker-client.d.ts +10 -9
- package/dist/harness/knowledge/writer-worker-client.js +20 -2
- package/dist/harness/knowledge/writer-worker.js +9 -1
- package/dist/harness/memory/sqlite/wisdom-store.js +35 -10
- package/dist/harness/memory/types.d.ts +6 -4
- package/dist/harness/memory/wisdom-formation.d.ts +14 -3
- package/dist/harness/memory/wisdom-formation.js +85 -22
- package/dist/harness/memory/wisdom-gates.d.ts +14 -0
- package/dist/harness/memory/wisdom-gates.js +59 -19
- package/dist/harness/memory/wisdom.js +9 -0
- package/dist/harness/observability/observer/hub.js +25 -15
- package/dist/harness/observability/observer/types.d.ts +10 -6
- package/dist/harness/run/agent/agent.d.ts +40 -21
- package/dist/harness/run/agent/agent.js +66 -13
- package/dist/harness/run/agent/index.d.ts +1 -1
- package/dist/harness/run/budget/budget.d.ts +28 -10
- package/dist/harness/run/budget/budget.js +38 -15
- package/dist/harness/run/model/index.d.ts +3 -1
- package/dist/harness/run/model/index.js +2 -1
- package/dist/harness/run/model/resolver.d.ts +4 -1
- package/dist/harness/run/model/resolver.js +36 -2
- package/dist/harness/run/model/run-scope.d.ts +2 -0
- package/dist/harness/run/model/run-scope.js +7 -0
- package/dist/harness/run/model/types.d.ts +22 -5
- package/dist/harness/run/model/types.js +3 -2
- package/dist/harness/run/model/window-truth.d.ts +37 -0
- package/dist/harness/run/model/window-truth.js +39 -0
- package/dist/harness/run/reliability/context-length-feedback.d.ts +77 -0
- package/dist/harness/run/reliability/context-length-feedback.js +128 -0
- package/dist/harness/run/reliability/harness-events.d.ts +3 -3
- package/dist/harness/run/reliability/model-error-policy.js +4 -0
- package/dist/harness/run/reliability/run-agent.d.ts +10 -4
- package/dist/harness/run/reliability/run-agent.js +137 -53
- package/dist/harness/run/reliability/run-metrics-collector.d.ts +19 -8
- package/dist/harness/run/reliability/run-metrics-collector.js +38 -23
- package/dist/harness/run/run-guard/default-run-guard.js +8 -8
- package/dist/harness/run/run-scope.d.ts +37 -4
- package/dist/harness/run/runner.d.ts +44 -29
- package/dist/harness/run/runner.js +320 -102
- package/dist/harness/session/attachments/index.d.ts +1 -0
- package/dist/harness/session/attachments/index.js +1 -0
- package/dist/harness/session/attachments/inject.d.ts +21 -10
- package/dist/harness/session/attachments/inject.js +76 -24
- package/dist/harness/session/attachments/recall-index.d.ts +70 -0
- package/dist/harness/session/attachments/recall-index.js +219 -0
- package/dist/harness/session/attachments/service.d.ts +21 -0
- package/dist/harness/session/attachments/service.js +258 -139
- package/dist/harness/session/attachments/types.d.ts +6 -0
- package/dist/harness/session/transform-store.d.ts +136 -0
- package/dist/harness/session/transform-store.js +302 -0
- package/dist/harness/session/types.d.ts +2 -2
- package/dist/harness/shared/events/agent-event-map.d.ts +46 -13
- package/dist/harness/shared/events/agent-event-map.js +3 -2
- package/dist/harness/shared/events/scenario-events.d.ts +6 -6
- package/dist/harness/shared/types/agent-definition.d.ts +1 -2
- package/dist/index.d.ts +14 -17
- package/dist/index.js +7 -9
- package/dist/integration/observability/metrics.d.ts +6 -6
- package/dist/integration/observability/metrics.js +13 -13
- package/dist/integration/observability/observer-bridge.js +2 -2
- package/dist/integration/observability/trace-collector.d.ts +5 -5
- package/dist/integration/observability/trace-collector.js +34 -35
- package/dist/integration/observability/trace-events.d.ts +3 -3
- package/dist/integration/observability/trace-events.js +3 -3
- package/dist/integration/providers/anthropic.js +14 -1
- package/dist/integration/providers/openai.js +7 -1
- package/dist/integration/providers/usage.d.ts +1 -1
- package/dist/integration/providers/wire-dump.d.ts +1 -1
- package/dist/integration/storage/archive-manager.d.ts +6 -0
- package/dist/integration/storage/archive-manager.js +10 -0
- package/dist/integration/storage/jsonl.d.ts +1 -1
- package/dist/integration/storage/jsonl.js +18 -16
- package/dist/subsystems/wisdom-steward/formation/SUBSYSTEM.md +8 -1
- package/dist/subsystems/wisdom-steward/formation/config.yaml +1 -0
- package/dist/subsystems/wisdom-steward/formation/handler.js +27 -5
- package/package.json +11 -14
- package/dist/harness/context/hybrid-compressor.d.ts +0 -41
- package/dist/harness/context/hybrid-compressor.js +0 -159
- package/dist/harness/context/llm-summarizer.d.ts +0 -46
- package/dist/harness/context/llm-summarizer.js +0 -249
- package/dist/harness/context/smart-router.d.ts +0 -75
- package/dist/harness/context/smart-router.js +0 -118
- package/dist/harness/context/truncate-compressor.d.ts +0 -28
- package/dist/harness/context/truncate-compressor.js +0 -107
- package/dist/harness/extension/plugin-ecosystem/plugins/capability.d.ts +0 -97
- package/dist/harness/extension/plugin-ecosystem/plugins/capability.js +0 -155
- package/dist/harness/extension/plugin-ecosystem/plugins/hooks.d.ts +0 -7
- package/dist/harness/extension/plugin-ecosystem/plugins/hooks.js +0 -7
- package/dist/harness/extension/plugin-ecosystem/plugins/lifecycle.d.ts +0 -86
- package/dist/harness/extension/plugin-ecosystem/plugins/lifecycle.js +0 -139
|
@@ -1,115 +1,134 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Token
|
|
2
|
+
* Token 估算 adapter(内容形状 → Segment → 唯一计数)
|
|
3
3
|
*
|
|
4
|
-
* 不依赖 Message / LLMMessage
|
|
5
|
-
*
|
|
4
|
+
* 不依赖 Message / LLMMessage 等领域类型,只做「内容形状 → token 数」。
|
|
5
|
+
* 计数逻辑全部委托 {@link estimateSegments}(P1-c:单实现 + 两 adapter);
|
|
6
|
+
* 本模块是「松散形状」adapter,领域/LLM 两入口的形状归一在 `token-estimator.ts`。
|
|
6
7
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* 2. 次选:专用 tokenizer(如 tiktoken)- 未来扩展
|
|
10
|
-
* 3. 兜底:启发式估算,按内容类型使用不同比率
|
|
8
|
+
* R4 红线:不得返回常数兜底——未知内容块按载荷长度计,且**告警**(分类表缺失必须可见,
|
|
9
|
+
* 每 type 限频 1 条/小时,防高频估算路径刷屏)。
|
|
11
10
|
*/
|
|
12
|
-
import {
|
|
13
|
-
|
|
14
|
-
// 非拉丁字符检测,覆盖完整东亚字符范围
|
|
15
|
-
const NON_LATIN_RE = /[\u2E80-\u9FFF\uA000-\uA4FF\uAC00-\uD7AF\uF900-\uFAFF\uFF01-\uFF60\uFFE0-\uFFE6]/gu;
|
|
16
|
-
// CJK 扩展 B+ 的 surrogate pairs(U+20000-U+2FA1F)
|
|
17
|
-
const CJK_SURROGATE_HIGH_RE = /[\uD840-\uD87E][\uDC00-\uDFFF]/g;
|
|
11
|
+
import { JSON_CHARS_PER_TOKEN, IMAGE_TOKEN_ESTIMATE, AUDIO_TOKEN_ESTIMATE, VIDEO_TOKEN_ESTIMATE, } from './token-constants.js';
|
|
12
|
+
import { estimateSegments, textSegment } from './token-segments.js';
|
|
18
13
|
/**
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* 每个 CJK 字符计为 CHARS_PER_TOKEN 个等效字符,
|
|
22
|
-
* 这样 `adjustedChars / CHARS_PER_TOKEN` 能得到准确的 token 估算。
|
|
14
|
+
* 估算单段文本的 token 数
|
|
23
15
|
*
|
|
24
16
|
* @param text 原始文本
|
|
25
|
-
* @returns
|
|
17
|
+
* @returns 估算 token 数
|
|
26
18
|
*/
|
|
27
|
-
export function
|
|
28
|
-
|
|
29
|
-
return 0;
|
|
30
|
-
const nonLatinCount = (text.match(NON_LATIN_RE) ?? []).length;
|
|
31
|
-
// 修正 surrogate pairs(CJK 扩展 B+ 占 2 个 UTF-16 单元但算 1 个字符)
|
|
32
|
-
const cjkSurrogates = (text.match(CJK_SURROGATE_HIGH_RE) ?? []).length;
|
|
33
|
-
const codePointLength = text.length - cjkSurrogates;
|
|
34
|
-
// CJK 字符本身已计入 codePointLength,额外加 (CHARS_PER_TOKEN - 1) 的权重
|
|
35
|
-
return codePointLength + nonLatinCount * (CHARS_PER_TOKEN - 1);
|
|
19
|
+
export function estimateTextTokens(text) {
|
|
20
|
+
return estimateSegments([textSegment(text)]);
|
|
36
21
|
}
|
|
22
|
+
/** 未知块告警限频:每 type 每小时至多 1 条(估算在热路径高频调用,防刷屏) */
|
|
23
|
+
const UNKNOWN_BLOCK_WARN_INTERVAL_MS = 3_600_000;
|
|
24
|
+
/** 记账上限:超出后新 type 不再告警(64 种未分类形状已足够说明分类表崩了) */
|
|
25
|
+
const UNKNOWN_BLOCK_WARN_MAX_TYPES = 64;
|
|
26
|
+
const unknownBlockWarnedAt = new Map();
|
|
37
27
|
/**
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
28
|
+
* 未分类内容块告警:分类表缺失必须可见(R4 后半),但计数不阻塞(按载荷长度计)。
|
|
29
|
+
* 覆盖两种形态:真正未知的 type,以及已知 type 的畸形载荷(如 text 缺正文)——
|
|
30
|
+
* 后者同样说明分类表没接住,文案用 unclassified 不断言 type 本身未知。
|
|
41
31
|
*
|
|
42
|
-
* @param
|
|
43
|
-
* @
|
|
32
|
+
* @param type 未被分类表接住的块类型
|
|
33
|
+
* @param chars 该块被计入估算的载荷字符数
|
|
44
34
|
*/
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
35
|
+
function warnUnknownBlock(type, chars) {
|
|
36
|
+
const key = type ?? '(undefined)';
|
|
37
|
+
const now = Date.now();
|
|
38
|
+
const last = unknownBlockWarnedAt.get(key);
|
|
39
|
+
if (last !== undefined && now - last < UNKNOWN_BLOCK_WARN_INTERVAL_MS)
|
|
40
|
+
return;
|
|
41
|
+
if (last === undefined && unknownBlockWarnedAt.size >= UNKNOWN_BLOCK_WARN_MAX_TYPES)
|
|
42
|
+
return;
|
|
43
|
+
unknownBlockWarnedAt.set(key, now);
|
|
44
|
+
console.warn(`[token-est] unclassified content block type=${JSON.stringify(key)} counted by payload length (chars=${chars}) — add a category in contentBlockSegment if this is a legitimate shape`);
|
|
53
45
|
}
|
|
54
46
|
/**
|
|
55
|
-
*
|
|
47
|
+
* 内容块 → Segment(分类归一;media 固定成本继承旧常量)
|
|
56
48
|
*
|
|
57
|
-
*
|
|
49
|
+
* 图像接受三种同价形态:原生 `image` 与转换后的 `image_url` / `input_image`
|
|
50
|
+
* (convertMessage 产出 image_url——旧版漏认导致图像按 0 计)。
|
|
51
|
+
* 未知块类型按实际载荷长度计(R4:无常数兜底):正文走 CJK 分类文本口径;
|
|
52
|
+
* 无正文按指针字段(name/url/src/path/filename)长度计——不序列化内嵌 base64
|
|
53
|
+
* 载荷(会虚高),无载荷即 0(不猜)。
|
|
58
54
|
*
|
|
59
55
|
* @param block ContentBlock(至少含 type,text 块需含 text)
|
|
60
|
-
* @returns
|
|
56
|
+
* @returns 归一化片段
|
|
61
57
|
*/
|
|
62
|
-
export function
|
|
58
|
+
export function contentBlockSegment(block) {
|
|
63
59
|
if (block.type === 'text' && typeof block.text === 'string') {
|
|
64
|
-
return
|
|
60
|
+
return textSegment(block.text);
|
|
61
|
+
}
|
|
62
|
+
if (block.type === 'image' || block.type === 'image_url' || block.type === 'input_image') {
|
|
63
|
+
return { kind: 'image', fixed: IMAGE_TOKEN_ESTIMATE };
|
|
65
64
|
}
|
|
66
|
-
if (block.type === 'image')
|
|
67
|
-
return IMAGE_TOKEN_ESTIMATE;
|
|
68
65
|
if (block.type === 'audio' || block.type === 'input_audio')
|
|
69
|
-
return AUDIO_TOKEN_ESTIMATE;
|
|
66
|
+
return { kind: 'audio', fixed: AUDIO_TOKEN_ESTIMATE };
|
|
70
67
|
if (block.type === 'video')
|
|
71
|
-
return VIDEO_TOKEN_ESTIMATE;
|
|
72
|
-
|
|
68
|
+
return { kind: 'video', fixed: VIDEO_TOKEN_ESTIMATE };
|
|
69
|
+
if (block.type === 'file') {
|
|
70
|
+
// FileBlock(core 声明类型):按指针字段计数——`data` 是 base64 内嵌载荷,
|
|
71
|
+
// 不序列化(会虚高);`sizeBytes` 不作估算输入(保持"长度计"单一口径)
|
|
72
|
+
return textSegment([block.name, block.url, block.mimeType].filter((v) => typeof v === 'string').join(' '));
|
|
73
|
+
}
|
|
74
|
+
const body = typeof block.text === 'string'
|
|
75
|
+
? block.text
|
|
76
|
+
: [block.name, block.url, block.src, block.path, block.filename]
|
|
77
|
+
.filter((v) => typeof v === 'string')
|
|
78
|
+
.join(' ');
|
|
79
|
+
warnUnknownBlock(block.type, body.length);
|
|
80
|
+
return textSegment(body);
|
|
73
81
|
}
|
|
74
82
|
/**
|
|
75
|
-
*
|
|
83
|
+
* 估算单个 content block 的 token 数
|
|
84
|
+
*
|
|
85
|
+
* @param block ContentBlock(至少含 type;text 块需含 text;未知块可带指针字段)
|
|
86
|
+
* @returns 估算 token 数
|
|
87
|
+
*/
|
|
88
|
+
export function estimateContentBlock(block) {
|
|
89
|
+
return estimateSegments([contentBlockSegment(block)]);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* 工具调用 → tool_use 段
|
|
76
93
|
*
|
|
77
|
-
* 工具调用的 JSON 结构符号多,token 密度更高。
|
|
78
94
|
* `arguments` 已是字符串时直接用长度,避免二次 JSON.stringify。
|
|
95
|
+
*/
|
|
96
|
+
export function toolCallSegment(name, args) {
|
|
97
|
+
const argStr = typeof args === 'string' ? args : JSON.stringify(args ?? {});
|
|
98
|
+
return { kind: 'tool_use', nameChars: (name ?? '').length, argChars: argStr.length };
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* 估算工具调用 JSON 的 token 数
|
|
79
102
|
*
|
|
80
103
|
* @param toolCalls `{ name?, arguments? }` 形态的工具调用列表
|
|
81
104
|
* @returns 估算 token 数
|
|
82
105
|
*/
|
|
83
106
|
export function estimateToolCallTokens(toolCalls) {
|
|
84
|
-
|
|
85
|
-
for (const tc of toolCalls) {
|
|
86
|
-
total += estimateTextTokens(tc.name ?? '');
|
|
87
|
-
const argsStr = typeof tc.arguments === 'string' ? tc.arguments : JSON.stringify(tc.arguments ?? {});
|
|
88
|
-
total += Math.ceil(argsStr.length / JSON_CHARS_PER_TOKEN);
|
|
89
|
-
}
|
|
90
|
-
return total;
|
|
107
|
+
return estimateSegments(toolCalls.map((tc) => toolCallSegment(tc.name, tc.arguments)));
|
|
91
108
|
}
|
|
92
109
|
/**
|
|
93
110
|
* 估算松散结构消息列表的 token 数
|
|
94
111
|
*
|
|
95
112
|
* 支持 string 和 ContentBlock[] 两种 content 格式,含消息结构开销。
|
|
96
|
-
* 适用于无强类型 Message / LLMMessage
|
|
113
|
+
* 适用于无强类型 Message / LLMMessage 的调用方(knowledge grounding 等)。
|
|
97
114
|
*
|
|
98
115
|
* @param messages `{ content?, role? }` 形态的消息列表
|
|
99
116
|
* @returns 估算 token 数
|
|
100
117
|
*/
|
|
101
118
|
export function estimateTokens(messages) {
|
|
102
|
-
|
|
119
|
+
const segments = [];
|
|
103
120
|
for (const msg of messages) {
|
|
104
|
-
|
|
105
|
-
if (typeof msg.content === 'string'
|
|
106
|
-
|
|
121
|
+
segments.push({ kind: 'message_overhead', count: 1 });
|
|
122
|
+
if (typeof msg.content === 'string') {
|
|
123
|
+
segments.push(textSegment(msg.content));
|
|
107
124
|
}
|
|
108
125
|
else if (Array.isArray(msg.content)) {
|
|
109
126
|
for (const block of msg.content) {
|
|
110
|
-
|
|
127
|
+
segments.push(contentBlockSegment(block));
|
|
111
128
|
}
|
|
112
129
|
}
|
|
113
130
|
}
|
|
114
|
-
return
|
|
131
|
+
return estimateSegments(segments);
|
|
115
132
|
}
|
|
133
|
+
// JSON_CHARS_PER_TOKEN 仍由本层公开导出(工具 schema 等外部调用方使用);计数已走分类查表。
|
|
134
|
+
export { JSON_CHARS_PER_TOKEN };
|
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* HeuristicTokenEstimator — 启发式 Token 估算器
|
|
3
3
|
*
|
|
4
|
-
* 实现本域 `TokenEstimator` 契约(`./types.ts
|
|
5
|
-
*
|
|
4
|
+
* 实现本域 `TokenEstimator` 契约(`./types.ts`)。**计数逻辑唯一**在
|
|
5
|
+
* `token-segments.ts`(P1-c「单实现 + 两 adapter」);本文件是两个形状 adapter:
|
|
6
|
+
* - {@link fromDomainMessages}:领域 `Message[]`(toolCalls / toolResults 字段)
|
|
7
|
+
* - {@link fromLlmMessages}:provider `LLMMessage[]`(tool_calls / tool_use /
|
|
8
|
+
* tool_result 块 / role:'tool' 消息)
|
|
9
|
+
*
|
|
10
|
+
* 同内容两入口读数一致是测试红线(差 < 1%)。两处旧口径缺陷已随统一修正:
|
|
11
|
+
* 1. LLM 口径把 tool_use / tool_result 块落「常数 10」兜底(R4 违约)→ 分类计数;
|
|
12
|
+
* 2. 领域口径的 CJK 膨胀 hack 挪到 2 比率上高倍(全 CJK 工具结果 2 token/字)→ 分类查表。
|
|
6
13
|
*
|
|
7
14
|
* 三层策略:
|
|
8
15
|
* 1. 优先使用 LLM 返回的实际 token 数(usage reported prompt)- 外部回写
|
|
@@ -12,10 +19,23 @@
|
|
|
12
19
|
import type { Message } from '@octopi-agent/core/types.js';
|
|
13
20
|
import type { TokenEstimator } from './types.js';
|
|
14
21
|
import type { LLMMessage, LLMToolDefinition } from '@octopi-agent/core/interfaces/model-provider.js';
|
|
22
|
+
import { type Segment } from './token-segments.js';
|
|
15
23
|
export { estimateTextTokens } from './token-estimate-fns.js';
|
|
24
|
+
/** 领域消息列表 → Segment[](adapter 公开入口) */
|
|
25
|
+
export declare function fromDomainMessages(messages: Message[]): Segment[];
|
|
26
|
+
/** provider LLM 消息 → Segment[](adapter 公开入口) */
|
|
27
|
+
export declare function fromLlmMessages(messages: LLMMessage[]): Segment[];
|
|
16
28
|
/**
|
|
17
|
-
*
|
|
29
|
+
* 估算 LLM 消息列表的 token 数
|
|
30
|
+
*
|
|
31
|
+
* 适配 provider 侧 `LLMMessage`(OpenAI 风格 `tool_calls` / Anthropic 风格
|
|
32
|
+
* `tool_use` / `tool_result` 内容块 / `role:'tool'` 消息)。
|
|
33
|
+
*
|
|
34
|
+
* @param messages LLM 消息列表
|
|
35
|
+
* @returns 估算 token 数
|
|
18
36
|
*/
|
|
37
|
+
export declare function estimateLLMMessages(messages: LLMMessage[]): number;
|
|
38
|
+
/** 启发式 Token 估算器实现 */
|
|
19
39
|
export declare class HeuristicTokenEstimator implements TokenEstimator {
|
|
20
40
|
/**
|
|
21
41
|
* 估算单条消息的 token 数
|
|
@@ -30,10 +50,14 @@ export declare class HeuristicTokenEstimator implements TokenEstimator {
|
|
|
30
50
|
/**
|
|
31
51
|
* 估算多条消息的总 token 数
|
|
32
52
|
*
|
|
53
|
+
* 总量一次取整(评审 #3):逐消息取整会让读数随消息切分形态漂移。
|
|
54
|
+
*
|
|
33
55
|
* @param messages 领域消息列表
|
|
34
56
|
* @returns 估算 token 数
|
|
35
57
|
*/
|
|
36
58
|
estimateMessages(messages: Message[]): number;
|
|
59
|
+
/** 单条消息的浮点估算(WeakMap 缓存存未取整值,取整只发生在对外读数上) */
|
|
60
|
+
private rawEstimateMessage;
|
|
37
61
|
/**
|
|
38
62
|
* 估算文本的 token 数
|
|
39
63
|
*
|
|
@@ -52,12 +76,3 @@ export declare class HeuristicTokenEstimator implements TokenEstimator {
|
|
|
52
76
|
*/
|
|
53
77
|
estimateTools(tools: LLMToolDefinition[]): number;
|
|
54
78
|
}
|
|
55
|
-
/**
|
|
56
|
-
* 估算 LLM 消息列表的 token 数
|
|
57
|
-
*
|
|
58
|
-
* 适配 provider 侧 `LLMMessage`(OpenAI 风格 `tool_calls` / `input_audio`)。
|
|
59
|
-
*
|
|
60
|
-
* @param messages LLM 消息列表
|
|
61
|
-
* @returns 估算 token 数
|
|
62
|
-
*/
|
|
63
|
-
export declare function estimateLLMMessages(messages: LLMMessage[]): number;
|
|
@@ -1,17 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* HeuristicTokenEstimator — 启发式 Token 估算器
|
|
3
3
|
*
|
|
4
|
-
* 实现本域 `TokenEstimator` 契约(`./types.ts
|
|
5
|
-
*
|
|
4
|
+
* 实现本域 `TokenEstimator` 契约(`./types.ts`)。**计数逻辑唯一**在
|
|
5
|
+
* `token-segments.ts`(P1-c「单实现 + 两 adapter」);本文件是两个形状 adapter:
|
|
6
|
+
* - {@link fromDomainMessages}:领域 `Message[]`(toolCalls / toolResults 字段)
|
|
7
|
+
* - {@link fromLlmMessages}:provider `LLMMessage[]`(tool_calls / tool_use /
|
|
8
|
+
* tool_result 块 / role:'tool' 消息)
|
|
9
|
+
*
|
|
10
|
+
* 同内容两入口读数一致是测试红线(差 < 1%)。两处旧口径缺陷已随统一修正:
|
|
11
|
+
* 1. LLM 口径把 tool_use / tool_result 块落「常数 10」兜底(R4 违约)→ 分类计数;
|
|
12
|
+
* 2. 领域口径的 CJK 膨胀 hack 挪到 2 比率上高倍(全 CJK 工具结果 2 token/字)→ 分类查表。
|
|
6
13
|
*
|
|
7
14
|
* 三层策略:
|
|
8
15
|
* 1. 优先使用 LLM 返回的实际 token 数(usage reported prompt)- 外部回写
|
|
9
16
|
* 2. 次选:专用 tokenizer(如 tiktoken)- 未来扩展
|
|
10
17
|
* 3. 兜底:启发式估算,按内容类型使用不同比率
|
|
11
18
|
*/
|
|
12
|
-
import { estimateTextTokens,
|
|
19
|
+
import { estimateTextTokens, contentBlockSegment, toolCallSegment } from './token-estimate-fns.js';
|
|
20
|
+
import { estimateSegments, sumSegments, textSegment, classifyChars } from './token-segments.js';
|
|
13
21
|
import { toolResultBody } from './tool-result-body.js';
|
|
14
|
-
import { TOOL_RESULT_CHARS_PER_TOKEN, JSON_CHARS_PER_TOKEN, MESSAGE_OVERHEAD_TOKENS, } from './token-constants.js';
|
|
15
22
|
// Re-export for callers that only need text estimation
|
|
16
23
|
export { estimateTextTokens } from './token-estimate-fns.js';
|
|
17
24
|
/**
|
|
@@ -21,9 +28,98 @@ export { estimateTextTokens } from './token-estimate-fns.js';
|
|
|
21
28
|
* 估算对同一消息是纯函数,跨实例结果必须一致。
|
|
22
29
|
*/
|
|
23
30
|
const messageTokensCache = new WeakMap();
|
|
31
|
+
/** 领域消息 → Segment[](adapter:只做形状归一;空串 content 不产生段,保两入口形状对称) */
|
|
32
|
+
function fromDomainMessage(message) {
|
|
33
|
+
// wire 对齐(评审 #4):convertMessage 把带 N 条 toolResults 的消息发成 N 条
|
|
34
|
+
// role:'tool' 消息(各带一条开销);其余消息 1 条开销。
|
|
35
|
+
const overhead = message.toolResults?.length ? message.toolResults.length : 1;
|
|
36
|
+
const segments = [{ kind: 'message_overhead', count: overhead }];
|
|
37
|
+
if (typeof message.content === 'string') {
|
|
38
|
+
if (message.content.length > 0)
|
|
39
|
+
segments.push(textSegment(message.content));
|
|
40
|
+
}
|
|
41
|
+
else if (Array.isArray(message.content)) {
|
|
42
|
+
for (const block of message.content) {
|
|
43
|
+
segments.push(contentBlockSegment(block));
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
if (message.toolCalls) {
|
|
47
|
+
for (const tc of message.toolCalls) {
|
|
48
|
+
segments.push(toolCallSegment(tc.name, tc.arguments));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
if (message.toolResults) {
|
|
52
|
+
for (const tr of message.toolResults) {
|
|
53
|
+
const body = toolResultBody(tr);
|
|
54
|
+
const { chars, cjk } = classifyChars(body);
|
|
55
|
+
segments.push({ kind: 'tool_result', chars, cjk, isError: tr.error !== undefined });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return segments;
|
|
59
|
+
}
|
|
60
|
+
/** 领域消息列表 → Segment[](adapter 公开入口) */
|
|
61
|
+
export function fromDomainMessages(messages) {
|
|
62
|
+
const segments = [];
|
|
63
|
+
for (const message of messages)
|
|
64
|
+
segments.push(...fromDomainMessage(message));
|
|
65
|
+
return segments;
|
|
66
|
+
}
|
|
67
|
+
/** provider 内容块 → Segment(tool_use / tool_result 块专用归一) */
|
|
68
|
+
function llmContentBlockSegment(block) {
|
|
69
|
+
const type = typeof block.type === 'string' ? block.type : undefined;
|
|
70
|
+
if (type === 'tool_use') {
|
|
71
|
+
return toolCallSegment(typeof block.name === 'string' ? block.name : undefined, block.input ?? block.arguments ?? {});
|
|
72
|
+
}
|
|
73
|
+
if (type === 'tool_result') {
|
|
74
|
+
const raw = block.content ?? block.text ?? '';
|
|
75
|
+
const body = typeof raw === 'string' ? raw : JSON.stringify(raw);
|
|
76
|
+
const { chars, cjk } = classifyChars(body);
|
|
77
|
+
return { kind: 'tool_result', chars, cjk, isError: block.is_error === true };
|
|
78
|
+
}
|
|
79
|
+
return contentBlockSegment(block);
|
|
80
|
+
}
|
|
81
|
+
/** provider LLM 消息 → Segment[](adapter 公开入口) */
|
|
82
|
+
export function fromLlmMessages(messages) {
|
|
83
|
+
const segments = [];
|
|
84
|
+
for (const msg of messages) {
|
|
85
|
+
segments.push({ kind: 'message_overhead', count: 1 });
|
|
86
|
+
// OpenAI 风格 role:'tool' 消息即工具结果(旧口径按文本比率低计,P1-c 修正)
|
|
87
|
+
if (msg.role === 'tool') {
|
|
88
|
+
const body = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content ?? '');
|
|
89
|
+
const { chars, cjk } = classifyChars(body);
|
|
90
|
+
segments.push({ kind: 'tool_result', chars, cjk, isError: false });
|
|
91
|
+
continue;
|
|
92
|
+
}
|
|
93
|
+
if (typeof msg.content === 'string') {
|
|
94
|
+
if (msg.content.length > 0)
|
|
95
|
+
segments.push(textSegment(msg.content));
|
|
96
|
+
}
|
|
97
|
+
else if (Array.isArray(msg.content)) {
|
|
98
|
+
for (const block of msg.content) {
|
|
99
|
+
segments.push(llmContentBlockSegment(block));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (msg.tool_calls) {
|
|
103
|
+
for (const tc of msg.tool_calls) {
|
|
104
|
+
segments.push(toolCallSegment(tc.function?.name, tc.function?.arguments ?? ''));
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return segments;
|
|
109
|
+
}
|
|
24
110
|
/**
|
|
25
|
-
*
|
|
111
|
+
* 估算 LLM 消息列表的 token 数
|
|
112
|
+
*
|
|
113
|
+
* 适配 provider 侧 `LLMMessage`(OpenAI 风格 `tool_calls` / Anthropic 风格
|
|
114
|
+
* `tool_use` / `tool_result` 内容块 / `role:'tool'` 消息)。
|
|
115
|
+
*
|
|
116
|
+
* @param messages LLM 消息列表
|
|
117
|
+
* @returns 估算 token 数
|
|
26
118
|
*/
|
|
119
|
+
export function estimateLLMMessages(messages) {
|
|
120
|
+
return estimateSegments(fromLlmMessages(messages));
|
|
121
|
+
}
|
|
122
|
+
/** 启发式 Token 估算器实现 */
|
|
27
123
|
export class HeuristicTokenEstimator {
|
|
28
124
|
/**
|
|
29
125
|
* 估算单条消息的 token 数
|
|
@@ -35,43 +131,31 @@ export class HeuristicTokenEstimator {
|
|
|
35
131
|
* @returns 估算 token 数
|
|
36
132
|
*/
|
|
37
133
|
estimateMessage(message) {
|
|
38
|
-
|
|
39
|
-
if (cached !== undefined)
|
|
40
|
-
return cached;
|
|
41
|
-
let tokens = MESSAGE_OVERHEAD_TOKENS;
|
|
42
|
-
if (typeof message.content === 'string') {
|
|
43
|
-
tokens += estimateTextTokens(message.content);
|
|
44
|
-
}
|
|
45
|
-
else if (Array.isArray(message.content)) {
|
|
46
|
-
for (const block of message.content) {
|
|
47
|
-
tokens += estimateContentBlock(block);
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
if (message.toolCalls) {
|
|
51
|
-
tokens += estimateToolCallTokens(message.toolCalls);
|
|
52
|
-
}
|
|
53
|
-
// 工具结果用更密集的比率(chars/2)
|
|
54
|
-
if (message.toolResults) {
|
|
55
|
-
for (const tr of message.toolResults) {
|
|
56
|
-
const body = toolResultBody(tr);
|
|
57
|
-
tokens += Math.ceil(estimateAdjustedChars(body) / TOOL_RESULT_CHARS_PER_TOKEN);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
messageTokensCache.set(message, tokens);
|
|
61
|
-
return tokens;
|
|
134
|
+
return Math.ceil(this.rawEstimateMessage(message));
|
|
62
135
|
}
|
|
63
136
|
/**
|
|
64
137
|
* 估算多条消息的总 token 数
|
|
65
138
|
*
|
|
139
|
+
* 总量一次取整(评审 #3):逐消息取整会让读数随消息切分形态漂移。
|
|
140
|
+
*
|
|
66
141
|
* @param messages 领域消息列表
|
|
67
142
|
* @returns 估算 token 数
|
|
68
143
|
*/
|
|
69
144
|
estimateMessages(messages) {
|
|
70
145
|
let total = 0;
|
|
71
146
|
for (const msg of messages) {
|
|
72
|
-
total += this.
|
|
147
|
+
total += this.rawEstimateMessage(msg);
|
|
73
148
|
}
|
|
74
|
-
return total;
|
|
149
|
+
return Math.ceil(total);
|
|
150
|
+
}
|
|
151
|
+
/** 单条消息的浮点估算(WeakMap 缓存存未取整值,取整只发生在对外读数上) */
|
|
152
|
+
rawEstimateMessage(message) {
|
|
153
|
+
const cached = messageTokensCache.get(message);
|
|
154
|
+
if (cached !== undefined)
|
|
155
|
+
return cached;
|
|
156
|
+
const tokens = sumSegments(fromDomainMessage(message));
|
|
157
|
+
messageTokensCache.set(message, tokens);
|
|
158
|
+
return tokens;
|
|
75
159
|
}
|
|
76
160
|
/**
|
|
77
161
|
* 估算文本的 token 数
|
|
@@ -92,43 +176,13 @@ export class HeuristicTokenEstimator {
|
|
|
92
176
|
* @returns 估算 token 数
|
|
93
177
|
*/
|
|
94
178
|
estimateTools(tools) {
|
|
95
|
-
|
|
179
|
+
const segments = [];
|
|
96
180
|
for (const tool of tools) {
|
|
97
181
|
const fn = tool.function;
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
total += Math.ceil(paramsStr.length / JSON_CHARS_PER_TOKEN);
|
|
102
|
-
}
|
|
103
|
-
return total;
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* 估算 LLM 消息列表的 token 数
|
|
108
|
-
*
|
|
109
|
-
* 适配 provider 侧 `LLMMessage`(OpenAI 风格 `tool_calls` / `input_audio`)。
|
|
110
|
-
*
|
|
111
|
-
* @param messages LLM 消息列表
|
|
112
|
-
* @returns 估算 token 数
|
|
113
|
-
*/
|
|
114
|
-
export function estimateLLMMessages(messages) {
|
|
115
|
-
let total = 0;
|
|
116
|
-
for (const msg of messages) {
|
|
117
|
-
total += MESSAGE_OVERHEAD_TOKENS;
|
|
118
|
-
if (typeof msg.content === 'string') {
|
|
119
|
-
total += estimateTextTokens(msg.content);
|
|
120
|
-
}
|
|
121
|
-
else if (Array.isArray(msg.content)) {
|
|
122
|
-
for (const block of msg.content) {
|
|
123
|
-
total += estimateContentBlock(block);
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
if (msg.tool_calls) {
|
|
127
|
-
total += estimateToolCallTokens(msg.tool_calls.map((tc) => ({
|
|
128
|
-
name: tc.function?.name,
|
|
129
|
-
arguments: tc.function?.arguments ?? '',
|
|
130
|
-
})));
|
|
182
|
+
segments.push(textSegment(fn.name));
|
|
183
|
+
segments.push(textSegment(fn.description));
|
|
184
|
+
segments.push({ kind: 'json', chars: JSON.stringify(fn.parameters ?? {}).length });
|
|
131
185
|
}
|
|
186
|
+
return estimateSegments(segments);
|
|
132
187
|
}
|
|
133
|
-
return total;
|
|
134
188
|
}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Token 估算统一计数核心(P1-c:单实现 + 两 adapter)
|
|
3
|
+
*
|
|
4
|
+
* 「计数与入口形状无关」(arch/context-size-accounting.md §4.2):
|
|
5
|
+
* 入口形状(领域 `Message` / provider `LLMMessage`)只在 adapter 层归一为
|
|
6
|
+
* `Segment`,本模块是**唯一计数逻辑**。同内容两入口读数一致是测试红线(差 < 1%)。
|
|
7
|
+
*
|
|
8
|
+
* 比率策略:**先分类、再查表**({@link CHARS_PER_TOKEN_BY_KIND})。
|
|
9
|
+
* 取代旧 `estimateAdjustedChars` 的「nonLatin×3 膨胀」hack——那个 hack 是为
|
|
10
|
+
* text(4) 比率设计的,挪到 tool_result(2) 上会让全 CJK 结果变成 2 token/字(错一倍)。
|
|
11
|
+
*
|
|
12
|
+
* R4 红线:估算器不得返回常数兜底;未知内容块按长度计(告警半项归 P2-a)。
|
|
13
|
+
*/
|
|
14
|
+
/** 分类查表:chars/token(单源在 token-constants,此处只做语义命名) */
|
|
15
|
+
export declare const CHARS_PER_TOKEN_BY_KIND: {
|
|
16
|
+
/** 英文散文 */
|
|
17
|
+
readonly text_ascii: 4;
|
|
18
|
+
/** 中文等 CJK(1 token/字) */
|
|
19
|
+
readonly text_cjk: 1;
|
|
20
|
+
/** 工具结果:结构化输出更密集 */
|
|
21
|
+
readonly tool_result: 1.7;
|
|
22
|
+
/** JSON 负载(schema / arguments) */
|
|
23
|
+
readonly json: 2;
|
|
24
|
+
/** 代码 / 标识符 */
|
|
25
|
+
readonly code: 3;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* 归一化内容片段。
|
|
29
|
+
*
|
|
30
|
+
* `message_overhead` 为实现扩展(sketch 未列):消息结构开销(role/分隔符/边界标记)
|
|
31
|
+
* 不属于内容,但必须有唯一归属处——两个 adapter 在形状归一时按消息数生成。
|
|
32
|
+
*/
|
|
33
|
+
export type Segment = {
|
|
34
|
+
kind: 'text';
|
|
35
|
+
chars: number;
|
|
36
|
+
cjk: number;
|
|
37
|
+
} | {
|
|
38
|
+
kind: 'tool_use';
|
|
39
|
+
nameChars: number;
|
|
40
|
+
argChars: number;
|
|
41
|
+
} | {
|
|
42
|
+
kind: 'tool_result';
|
|
43
|
+
chars: number;
|
|
44
|
+
cjk: number;
|
|
45
|
+
isError: boolean;
|
|
46
|
+
} | {
|
|
47
|
+
kind: 'json';
|
|
48
|
+
chars: number;
|
|
49
|
+
} | {
|
|
50
|
+
kind: 'image' | 'audio' | 'video';
|
|
51
|
+
fixed: number;
|
|
52
|
+
} | {
|
|
53
|
+
kind: 'message_overhead';
|
|
54
|
+
count: number;
|
|
55
|
+
};
|
|
56
|
+
/** 消息结构开销(role + 分隔符 + 边界标记),单位 token(常量单源在 token-constants) */
|
|
57
|
+
export { MESSAGE_OVERHEAD_TOKENS } from './token-constants.js';
|
|
58
|
+
/**
|
|
59
|
+
* 字符分类:统计文本的码点数与其中的 CJK 数。
|
|
60
|
+
*
|
|
61
|
+
* 旧 `estimateAdjustedChars` 把 CJK 扩展 B 计成 ASCII(正则不匹配 surrogate),
|
|
62
|
+
* 此处并入 CJK(1 token/字)。其余非 BMP 码点(emoji 等)维持旧口径按 2 单元计,
|
|
63
|
+
* 不在本批静默调参。
|
|
64
|
+
*
|
|
65
|
+
* @param text 原始文本
|
|
66
|
+
* @returns chars=码点数;cjk=CJK 码点数(含扩展 B)
|
|
67
|
+
*/
|
|
68
|
+
export declare function classifyChars(text: string): {
|
|
69
|
+
chars: number;
|
|
70
|
+
cjk: number;
|
|
71
|
+
};
|
|
72
|
+
/** 文本 → text 段(adapter 便捷入口) */
|
|
73
|
+
export declare function textSegment(text: string): Segment;
|
|
74
|
+
/**
|
|
75
|
+
* 分段求和(不取整):两入口共用的浮点累加,供调用方在**总量**上统一取整——
|
|
76
|
+
* 逐消息取整会让读数随消息切分形态漂移(评审 #3)。
|
|
77
|
+
*
|
|
78
|
+
* @param segments 归一化内容片段
|
|
79
|
+
* @returns 估算 token 数(浮点)
|
|
80
|
+
*/
|
|
81
|
+
export declare function sumSegments(segments: Segment[]): number;
|
|
82
|
+
/**
|
|
83
|
+
* 唯一计数逻辑:Segment 列表 → token 估算(总量一次取整)。
|
|
84
|
+
*
|
|
85
|
+
* @param segments 归一化内容片段
|
|
86
|
+
* @returns 估算 token 数
|
|
87
|
+
*/
|
|
88
|
+
export declare function estimateSegments(segments: Segment[]): number;
|