fixo-cli 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +530 -0
- package/dist/agent/agent-client.d.ts +108 -0
- package/dist/agent/agent-client.d.ts.map +1 -0
- package/dist/agent/agent-client.js +1247 -0
- package/dist/agent/agent-client.js.map +1 -0
- package/dist/agent/agent-pool.d.ts +20 -0
- package/dist/agent/agent-pool.d.ts.map +1 -0
- package/dist/agent/agent-pool.js +217 -0
- package/dist/agent/agent-pool.js.map +1 -0
- package/dist/agent/background-awareness.d.ts +55 -0
- package/dist/agent/background-awareness.d.ts.map +1 -0
- package/dist/agent/background-awareness.js +104 -0
- package/dist/agent/background-awareness.js.map +1 -0
- package/dist/agent/command-parser.d.ts +33 -0
- package/dist/agent/command-parser.d.ts.map +1 -0
- package/dist/agent/command-parser.js +120 -0
- package/dist/agent/command-parser.js.map +1 -0
- package/dist/agent/context-budget.d.ts +91 -0
- package/dist/agent/context-budget.d.ts.map +1 -0
- package/dist/agent/context-budget.js +219 -0
- package/dist/agent/context-budget.js.map +1 -0
- package/dist/agent/conversation.d.ts +190 -0
- package/dist/agent/conversation.d.ts.map +1 -0
- package/dist/agent/conversation.js +547 -0
- package/dist/agent/conversation.js.map +1 -0
- package/dist/agent/hooks.d.ts +72 -0
- package/dist/agent/hooks.d.ts.map +1 -0
- package/dist/agent/hooks.js +214 -0
- package/dist/agent/hooks.js.map +1 -0
- package/dist/agent/mcp-bridge.d.ts +13 -0
- package/dist/agent/mcp-bridge.d.ts.map +1 -0
- package/dist/agent/mcp-bridge.js +86 -0
- package/dist/agent/mcp-bridge.js.map +1 -0
- package/dist/agent/mcp-client.d.ts +24 -0
- package/dist/agent/mcp-client.d.ts.map +1 -0
- package/dist/agent/mcp-client.js +146 -0
- package/dist/agent/mcp-client.js.map +1 -0
- package/dist/agent/mcp-manager.d.ts +13 -0
- package/dist/agent/mcp-manager.d.ts.map +1 -0
- package/dist/agent/mcp-manager.js +84 -0
- package/dist/agent/mcp-manager.js.map +1 -0
- package/dist/agent/mcp-registry.d.ts +45 -0
- package/dist/agent/mcp-registry.d.ts.map +1 -0
- package/dist/agent/mcp-registry.js +98 -0
- package/dist/agent/mcp-registry.js.map +1 -0
- package/dist/agent/orchestrator.d.ts +14 -0
- package/dist/agent/orchestrator.d.ts.map +1 -0
- package/dist/agent/orchestrator.js +118 -0
- package/dist/agent/orchestrator.js.map +1 -0
- package/dist/agent/parser-adapter.d.ts +120 -0
- package/dist/agent/parser-adapter.d.ts.map +1 -0
- package/dist/agent/parser-adapter.js +265 -0
- package/dist/agent/parser-adapter.js.map +1 -0
- package/dist/agent/parsers/imports.d.ts +11 -0
- package/dist/agent/parsers/imports.d.ts.map +1 -0
- package/dist/agent/parsers/imports.js +94 -0
- package/dist/agent/parsers/imports.js.map +1 -0
- package/dist/agent/parsers/shell.d.ts +23 -0
- package/dist/agent/parsers/shell.d.ts.map +1 -0
- package/dist/agent/parsers/shell.js +200 -0
- package/dist/agent/parsers/shell.js.map +1 -0
- package/dist/agent/parsers/symbols.d.ts +17 -0
- package/dist/agent/parsers/symbols.d.ts.map +1 -0
- package/dist/agent/parsers/symbols.js +103 -0
- package/dist/agent/parsers/symbols.js.map +1 -0
- package/dist/agent/permissions.d.ts +65 -0
- package/dist/agent/permissions.d.ts.map +1 -0
- package/dist/agent/permissions.js +219 -0
- package/dist/agent/permissions.js.map +1 -0
- package/dist/agent/predictive-gate.d.ts +69 -0
- package/dist/agent/predictive-gate.d.ts.map +1 -0
- package/dist/agent/predictive-gate.js +128 -0
- package/dist/agent/predictive-gate.js.map +1 -0
- package/dist/agent/provider-cooldown.d.ts +144 -0
- package/dist/agent/provider-cooldown.d.ts.map +1 -0
- package/dist/agent/provider-cooldown.js +300 -0
- package/dist/agent/provider-cooldown.js.map +1 -0
- package/dist/agent/providers-manager.d.ts +109 -0
- package/dist/agent/providers-manager.d.ts.map +1 -0
- package/dist/agent/providers-manager.js +464 -0
- package/dist/agent/providers-manager.js.map +1 -0
- package/dist/agent/repo-map.d.ts +6 -0
- package/dist/agent/repo-map.d.ts.map +1 -0
- package/dist/agent/repo-map.js +221 -0
- package/dist/agent/repo-map.js.map +1 -0
- package/dist/agent/retry.d.ts +103 -0
- package/dist/agent/retry.d.ts.map +1 -0
- package/dist/agent/retry.js +276 -0
- package/dist/agent/retry.js.map +1 -0
- package/dist/agent/search/index.d.ts +61 -0
- package/dist/agent/search/index.d.ts.map +1 -0
- package/dist/agent/search/index.js +314 -0
- package/dist/agent/search/index.js.map +1 -0
- package/dist/agent/single-agent.d.ts +76 -0
- package/dist/agent/single-agent.d.ts.map +1 -0
- package/dist/agent/single-agent.js +697 -0
- package/dist/agent/single-agent.js.map +1 -0
- package/dist/agent/skills.d.ts +22 -0
- package/dist/agent/skills.d.ts.map +1 -0
- package/dist/agent/skills.js +139 -0
- package/dist/agent/skills.js.map +1 -0
- package/dist/agent/stream-glue.d.ts +85 -0
- package/dist/agent/stream-glue.d.ts.map +1 -0
- package/dist/agent/stream-glue.js +120 -0
- package/dist/agent/stream-glue.js.map +1 -0
- package/dist/agent/subagent.d.ts +72 -0
- package/dist/agent/subagent.d.ts.map +1 -0
- package/dist/agent/subagent.js +193 -0
- package/dist/agent/subagent.js.map +1 -0
- package/dist/agent/telemetry.d.ts +192 -0
- package/dist/agent/telemetry.d.ts.map +1 -0
- package/dist/agent/telemetry.js +400 -0
- package/dist/agent/telemetry.js.map +1 -0
- package/dist/agent/tokenizer.d.ts +42 -0
- package/dist/agent/tokenizer.d.ts.map +1 -0
- package/dist/agent/tokenizer.js +107 -0
- package/dist/agent/tokenizer.js.map +1 -0
- package/dist/agent/tool-executor.d.ts +289 -0
- package/dist/agent/tool-executor.d.ts.map +1 -0
- package/dist/agent/tool-executor.js +2519 -0
- package/dist/agent/tool-executor.js.map +1 -0
- package/dist/agent/web-impl.d.ts +2 -0
- package/dist/agent/web-impl.d.ts.map +1 -0
- package/dist/agent/web-impl.js +34 -0
- package/dist/agent/web-impl.js.map +1 -0
- package/dist/agent/web.d.ts +8 -0
- package/dist/agent/web.d.ts.map +1 -0
- package/dist/agent/web.js +8 -0
- package/dist/agent/web.js.map +1 -0
- package/dist/agent/worker-agent.d.ts +27 -0
- package/dist/agent/worker-agent.d.ts.map +1 -0
- package/dist/agent/worker-agent.js +503 -0
- package/dist/agent/worker-agent.js.map +1 -0
- package/dist/config.d.ts +162 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +138 -0
- package/dist/config.js.map +1 -0
- package/dist/context/fixo-md-watcher.d.ts +42 -0
- package/dist/context/fixo-md-watcher.d.ts.map +1 -0
- package/dist/context/fixo-md-watcher.js +126 -0
- package/dist/context/fixo-md-watcher.js.map +1 -0
- package/dist/context/fixo-md.d.ts +50 -0
- package/dist/context/fixo-md.d.ts.map +1 -0
- package/dist/context/fixo-md.js +118 -0
- package/dist/context/fixo-md.js.map +1 -0
- package/dist/context/todo.d.ts +65 -0
- package/dist/context/todo.d.ts.map +1 -0
- package/dist/context/todo.js +194 -0
- package/dist/context/todo.js.map +1 -0
- package/dist/git/git-manager.d.ts +33 -0
- package/dist/git/git-manager.d.ts.map +1 -0
- package/dist/git/git-manager.js +293 -0
- package/dist/git/git-manager.js.map +1 -0
- package/dist/git/git-ops.d.ts +10 -0
- package/dist/git/git-ops.d.ts.map +1 -0
- package/dist/git/git-ops.js +131 -0
- package/dist/git/git-ops.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +352 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer.d.ts +30 -0
- package/dist/indexer.d.ts.map +1 -0
- package/dist/indexer.js +273 -0
- package/dist/indexer.js.map +1 -0
- package/dist/lsp/lsp-client.d.ts +24 -0
- package/dist/lsp/lsp-client.d.ts.map +1 -0
- package/dist/lsp/lsp-client.js +205 -0
- package/dist/lsp/lsp-client.js.map +1 -0
- package/dist/lsp/lsp-manager.d.ts +17 -0
- package/dist/lsp/lsp-manager.d.ts.map +1 -0
- package/dist/lsp/lsp-manager.js +154 -0
- package/dist/lsp/lsp-manager.js.map +1 -0
- package/dist/lsp/lsp-pre-save.d.ts +137 -0
- package/dist/lsp/lsp-pre-save.d.ts.map +1 -0
- package/dist/lsp/lsp-pre-save.js +245 -0
- package/dist/lsp/lsp-pre-save.js.map +1 -0
- package/dist/lsp/syntax-fallback.d.ts +83 -0
- package/dist/lsp/syntax-fallback.d.ts.map +1 -0
- package/dist/lsp/syntax-fallback.js +275 -0
- package/dist/lsp/syntax-fallback.js.map +1 -0
- package/dist/model-outcomes.d.ts +12 -0
- package/dist/model-outcomes.d.ts.map +1 -0
- package/dist/model-outcomes.js +46 -0
- package/dist/model-outcomes.js.map +1 -0
- package/dist/planner.d.ts +32 -0
- package/dist/planner.d.ts.map +1 -0
- package/dist/planner.js +163 -0
- package/dist/planner.js.map +1 -0
- package/dist/project-memory.d.ts +29 -0
- package/dist/project-memory.d.ts.map +1 -0
- package/dist/project-memory.js +349 -0
- package/dist/project-memory.js.map +1 -0
- package/dist/review.d.ts +2 -0
- package/dist/review.d.ts.map +1 -0
- package/dist/review.js +61 -0
- package/dist/review.js.map +1 -0
- package/dist/runtime/background-jobs.d.ts +97 -0
- package/dist/runtime/background-jobs.d.ts.map +1 -0
- package/dist/runtime/background-jobs.js +331 -0
- package/dist/runtime/background-jobs.js.map +1 -0
- package/dist/runtime/credential-vault.d.ts +124 -0
- package/dist/runtime/credential-vault.d.ts.map +1 -0
- package/dist/runtime/credential-vault.js +184 -0
- package/dist/runtime/credential-vault.js.map +1 -0
- package/dist/runtime/loop-trap.d.ts +197 -0
- package/dist/runtime/loop-trap.d.ts.map +1 -0
- package/dist/runtime/loop-trap.js +420 -0
- package/dist/runtime/loop-trap.js.map +1 -0
- package/dist/runtime/policy.d.ts +15 -0
- package/dist/runtime/policy.d.ts.map +1 -0
- package/dist/runtime/policy.js +60 -0
- package/dist/runtime/policy.js.map +1 -0
- package/dist/runtime/redaction.d.ts +66 -0
- package/dist/runtime/redaction.d.ts.map +1 -0
- package/dist/runtime/redaction.js +155 -0
- package/dist/runtime/redaction.js.map +1 -0
- package/dist/runtime/session-snapshots.d.ts +76 -0
- package/dist/runtime/session-snapshots.d.ts.map +1 -0
- package/dist/runtime/session-snapshots.js +166 -0
- package/dist/runtime/session-snapshots.js.map +1 -0
- package/dist/runtime/staging.d.ts +205 -0
- package/dist/runtime/staging.d.ts.map +1 -0
- package/dist/runtime/staging.js +526 -0
- package/dist/runtime/staging.js.map +1 -0
- package/dist/runtime/task-session.d.ts +95 -0
- package/dist/runtime/task-session.d.ts.map +1 -0
- package/dist/runtime/task-session.js +263 -0
- package/dist/runtime/task-session.js.map +1 -0
- package/dist/runtime/worktree.d.ts +55 -0
- package/dist/runtime/worktree.d.ts.map +1 -0
- package/dist/runtime/worktree.js +175 -0
- package/dist/runtime/worktree.js.map +1 -0
- package/dist/setup-wizard.d.ts +8 -0
- package/dist/setup-wizard.d.ts.map +1 -0
- package/dist/setup-wizard.js +73 -0
- package/dist/setup-wizard.js.map +1 -0
- package/dist/shared/content.d.ts +43 -0
- package/dist/shared/content.d.ts.map +1 -0
- package/dist/shared/content.js +61 -0
- package/dist/shared/content.js.map +1 -0
- package/dist/shared/types.d.ts +217 -0
- package/dist/shared/types.d.ts.map +1 -0
- package/dist/shared/types.js +3 -0
- package/dist/shared/types.js.map +1 -0
- package/dist/test-runner.d.ts +5 -0
- package/dist/test-runner.d.ts.map +1 -0
- package/dist/test-runner.js +42 -0
- package/dist/test-runner.js.map +1 -0
- package/dist/types.d.ts +85 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/ui/ascii.d.ts +23 -0
- package/dist/ui/ascii.d.ts.map +1 -0
- package/dist/ui/ascii.js +45 -0
- package/dist/ui/ascii.js.map +1 -0
- package/dist/ui/colors.d.ts +111 -0
- package/dist/ui/colors.d.ts.map +1 -0
- package/dist/ui/colors.js +166 -0
- package/dist/ui/colors.js.map +1 -0
- package/dist/ui/image-attach.d.ts +27 -0
- package/dist/ui/image-attach.d.ts.map +1 -0
- package/dist/ui/image-attach.js +100 -0
- package/dist/ui/image-attach.js.map +1 -0
- package/dist/ui/index.d.ts +18 -0
- package/dist/ui/index.d.ts.map +1 -0
- package/dist/ui/index.js +18 -0
- package/dist/ui/index.js.map +1 -0
- package/dist/ui/markdown-stream.d.ts +91 -0
- package/dist/ui/markdown-stream.d.ts.map +1 -0
- package/dist/ui/markdown-stream.js +524 -0
- package/dist/ui/markdown-stream.js.map +1 -0
- package/dist/ui/plan-renderer.d.ts +36 -0
- package/dist/ui/plan-renderer.d.ts.map +1 -0
- package/dist/ui/plan-renderer.js +79 -0
- package/dist/ui/plan-renderer.js.map +1 -0
- package/dist/ui/prompt.d.ts +11 -0
- package/dist/ui/prompt.d.ts.map +1 -0
- package/dist/ui/prompt.js +1960 -0
- package/dist/ui/prompt.js.map +1 -0
- package/dist/ui/render-primitives.d.ts +117 -0
- package/dist/ui/render-primitives.d.ts.map +1 -0
- package/dist/ui/render-primitives.js +322 -0
- package/dist/ui/render-primitives.js.map +1 -0
- package/dist/ui/render.d.ts +133 -0
- package/dist/ui/render.d.ts.map +1 -0
- package/dist/ui/render.js +547 -0
- package/dist/ui/render.js.map +1 -0
- package/dist/ui/session-header.d.ts +30 -0
- package/dist/ui/session-header.d.ts.map +1 -0
- package/dist/ui/session-header.js +74 -0
- package/dist/ui/session-header.js.map +1 -0
- package/dist/workspace-guard.d.ts +68 -0
- package/dist/workspace-guard.d.ts.map +1 -0
- package/dist/workspace-guard.js +168 -0
- package/dist/workspace-guard.js.map +1 -0
- package/dist/workspace-lock.d.ts +27 -0
- package/dist/workspace-lock.d.ts.map +1 -0
- package/dist/workspace-lock.js +95 -0
- package/dist/workspace-lock.js.map +1 -0
- package/package.json +63 -0
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Telemetry — local NDJSON sink + on-demand failure diagnosis.
|
|
3
|
+
*
|
|
4
|
+
* The original `logTelemetry` shipped a remote-only HTTP poster to
|
|
5
|
+
* the FreeLLMAPI server. Pillar 5 (Telemetry) replaces it with a
|
|
6
|
+
* first-class, *local-first* event store:
|
|
7
|
+
*
|
|
8
|
+
* - All events are appended to `~/.fixocli/telemetry.jsonl` in
|
|
9
|
+
* newline-delimited JSON. One event per line, easy to tail,
|
|
10
|
+
* grep, and post-process.
|
|
11
|
+
* - The legacy remote poster is preserved as an opt-in
|
|
12
|
+
* secondary sink (controlled by `preferences.telemetryRemote`).
|
|
13
|
+
* - The on-disk log is rotated at 1 MiB with a single `.1`
|
|
14
|
+
* backup, so disk usage is bounded.
|
|
15
|
+
* - `diagnoseFailures(windowMs)` reads the last N events and
|
|
16
|
+
* produces human-readable remediation hints — useful for
|
|
17
|
+
* `/diagnose` slash commands and for the post-mortem shown
|
|
18
|
+
* when a tool call exhausts its budget.
|
|
19
|
+
*
|
|
20
|
+
* The module is *side-effect free at import time*; the file is
|
|
21
|
+
* opened lazily on first write so the CLI cold-start is unaffected.
|
|
22
|
+
*
|
|
23
|
+
* The previous public surface — `logTelemetry(payload)` and the
|
|
24
|
+
* `TelemetryPayload` interface — is preserved so the 7 existing
|
|
25
|
+
* callsites in `worker-agent.ts` and `agent-pool.ts` keep working
|
|
26
|
+
* without modification. New code should prefer `TelemetrySink`
|
|
27
|
+
* directly.
|
|
28
|
+
*/
|
|
29
|
+
import fs from 'node:fs';
|
|
30
|
+
import path from 'node:path';
|
|
31
|
+
import { loadConfig, getConfigDir } from '../config.js';
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
// Per-type event constructors
|
|
34
|
+
// ---------------------------------------------------------------------------
|
|
35
|
+
function makeEvent(type, fields) {
|
|
36
|
+
return {
|
|
37
|
+
ts: new Date().toISOString(),
|
|
38
|
+
type,
|
|
39
|
+
sid: getSessionId(),
|
|
40
|
+
fields: Object.freeze({ ...fields }),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
export const telemetry = {
|
|
44
|
+
toolCall(fields) {
|
|
45
|
+
return makeEvent('tool_call', fields);
|
|
46
|
+
},
|
|
47
|
+
retry(fields) {
|
|
48
|
+
return makeEvent('retry', fields);
|
|
49
|
+
},
|
|
50
|
+
cooldown(fields) {
|
|
51
|
+
return makeEvent('cooldown', fields);
|
|
52
|
+
},
|
|
53
|
+
streamResume(fields) {
|
|
54
|
+
return makeEvent(fields.ok ? 'stream_resume' : 'stream_resume_exhausted', fields);
|
|
55
|
+
},
|
|
56
|
+
contextBudget(fields) {
|
|
57
|
+
return makeEvent('context_budget', fields);
|
|
58
|
+
},
|
|
59
|
+
providerError(fields) {
|
|
60
|
+
return makeEvent('provider_error', fields);
|
|
61
|
+
},
|
|
62
|
+
sessionStart(fields) {
|
|
63
|
+
return makeEvent('session_start', fields);
|
|
64
|
+
},
|
|
65
|
+
sessionEnd(fields) {
|
|
66
|
+
return makeEvent('session_end', fields);
|
|
67
|
+
},
|
|
68
|
+
surgicalEdit(fields) {
|
|
69
|
+
return makeEvent('tool_surgical_edit', fields);
|
|
70
|
+
},
|
|
71
|
+
glob(fields) {
|
|
72
|
+
return makeEvent('tool_glob', fields);
|
|
73
|
+
},
|
|
74
|
+
fixoMdLoaded(fields) {
|
|
75
|
+
return makeEvent('fixo_md_loaded', fields);
|
|
76
|
+
},
|
|
77
|
+
todoMutation(fields) {
|
|
78
|
+
return makeEvent('todo_mutation', fields);
|
|
79
|
+
},
|
|
80
|
+
sessionSnapshot(fields) {
|
|
81
|
+
return makeEvent('session_snapshot', fields);
|
|
82
|
+
},
|
|
83
|
+
asyncSpawn(fields) {
|
|
84
|
+
return makeEvent('tool_async_spawn', fields);
|
|
85
|
+
},
|
|
86
|
+
subagentSummary(fields) {
|
|
87
|
+
return makeEvent('subagent_summary', fields);
|
|
88
|
+
},
|
|
89
|
+
hookFired(fields) {
|
|
90
|
+
return makeEvent('hook_fired', fields);
|
|
91
|
+
},
|
|
92
|
+
permissionDecision(fields) {
|
|
93
|
+
return makeEvent('permission_decision', fields);
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
// ---------------------------------------------------------------------------
|
|
97
|
+
// Session id
|
|
98
|
+
// ---------------------------------------------------------------------------
|
|
99
|
+
let _sessionId = null;
|
|
100
|
+
function getSessionId() {
|
|
101
|
+
if (_sessionId)
|
|
102
|
+
return _sessionId;
|
|
103
|
+
// 12 chars of randomness, base36. Stable for the lifetime of
|
|
104
|
+
// the process but not personally identifying. The two calls to
|
|
105
|
+
// Math.random are concatenated so the string is always 24 chars
|
|
106
|
+
// from which we take the first 12, even if a single random()
|
|
107
|
+
// happens to return a tiny fraction.
|
|
108
|
+
const a = Math.random().toString(36).slice(2);
|
|
109
|
+
const b = Math.random().toString(36).slice(2);
|
|
110
|
+
_sessionId = (a + b).slice(0, 12);
|
|
111
|
+
return _sessionId;
|
|
112
|
+
}
|
|
113
|
+
// ---------------------------------------------------------------------------
|
|
114
|
+
// Filesystem paths
|
|
115
|
+
// ---------------------------------------------------------------------------
|
|
116
|
+
/** Path to the local NDJSON sink. Exposed for tests and the
|
|
117
|
+
* `/diagnose` slash command. */
|
|
118
|
+
export function getTelemetryPath() {
|
|
119
|
+
return path.join(getConfigDir(), 'telemetry.jsonl');
|
|
120
|
+
}
|
|
121
|
+
const MAX_BYTES = 1_048_576; // 1 MiB
|
|
122
|
+
function ensureDir() {
|
|
123
|
+
const dir = getConfigDir();
|
|
124
|
+
if (!fs.existsSync(dir)) {
|
|
125
|
+
fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function rotateIfNeeded() {
|
|
129
|
+
const file = getTelemetryPath();
|
|
130
|
+
if (!fs.existsSync(file))
|
|
131
|
+
return;
|
|
132
|
+
let stat;
|
|
133
|
+
try {
|
|
134
|
+
stat = fs.statSync(file);
|
|
135
|
+
}
|
|
136
|
+
catch {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
if (stat.size < MAX_BYTES)
|
|
140
|
+
return;
|
|
141
|
+
const backup = file + '.1';
|
|
142
|
+
try {
|
|
143
|
+
fs.renameSync(file, backup);
|
|
144
|
+
}
|
|
145
|
+
catch {
|
|
146
|
+
// Best-effort rotation; if rename fails, the next append will
|
|
147
|
+
// just keep extending the file past MAX_BYTES.
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
// ---------------------------------------------------------------------------
|
|
151
|
+
// Public sink API
|
|
152
|
+
// ---------------------------------------------------------------------------
|
|
153
|
+
/**
|
|
154
|
+
* Append an event to the local NDJSON sink. The file is opened in
|
|
155
|
+
* append mode for every write — the cost of an open/close cycle
|
|
156
|
+
* is negligible compared to the JSON.stringify call.
|
|
157
|
+
*
|
|
158
|
+
* Returns true on success, false on any error. Errors are swallowed
|
|
159
|
+
* because telemetry must never break the calling code.
|
|
160
|
+
*/
|
|
161
|
+
export function recordTelemetry(event) {
|
|
162
|
+
if (!isLocalEnabled())
|
|
163
|
+
return false;
|
|
164
|
+
try {
|
|
165
|
+
ensureDir();
|
|
166
|
+
rotateIfNeeded();
|
|
167
|
+
fs.appendFileSync(getTelemetryPath(), JSON.stringify(event) + '\n', { encoding: 'utf-8', mode: 0o600 });
|
|
168
|
+
return true;
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/** Read the last N events (most recent last). Safe to call when the
|
|
175
|
+
* file is missing or empty — returns []. */
|
|
176
|
+
export function readRecentEvents(limit = 200) {
|
|
177
|
+
const file = getTelemetryPath();
|
|
178
|
+
if (!fs.existsSync(file))
|
|
179
|
+
return [];
|
|
180
|
+
try {
|
|
181
|
+
const raw = fs.readFileSync(file, 'utf-8');
|
|
182
|
+
const lines = raw.split('\n').filter((l) => l.length > 0);
|
|
183
|
+
const slice = lines.slice(Math.max(0, lines.length - limit));
|
|
184
|
+
const out = [];
|
|
185
|
+
for (const line of slice) {
|
|
186
|
+
try {
|
|
187
|
+
out.push(JSON.parse(line));
|
|
188
|
+
}
|
|
189
|
+
catch {
|
|
190
|
+
// Skip malformed lines (rotated half-write etc.).
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
return out;
|
|
194
|
+
}
|
|
195
|
+
catch {
|
|
196
|
+
return [];
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
/** Clear the local sink. Test-only; not exposed in the UI. */
|
|
200
|
+
export function clearTelemetry() {
|
|
201
|
+
try {
|
|
202
|
+
const file = getTelemetryPath();
|
|
203
|
+
if (fs.existsSync(file))
|
|
204
|
+
fs.unlinkSync(file);
|
|
205
|
+
}
|
|
206
|
+
catch {
|
|
207
|
+
// ignore
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* Scan the recent event window and return a prioritised list of
|
|
212
|
+
* remediation hints. Pure function — does not modify state.
|
|
213
|
+
*
|
|
214
|
+
* Rules implemented:
|
|
215
|
+
* - 3+ retries in the window → "Flaky network or rate-limit" (warn)
|
|
216
|
+
* - 1+ cooldown event → "Provider X is rate-limited" (warn)
|
|
217
|
+
* - 1+ stream_resume_exhausted → "Stream cuts not recovering" (error)
|
|
218
|
+
* - 1+ context_budget with compact → "Context is filling up" (info)
|
|
219
|
+
* - 3+ tool_call failures of same tool → "Tool X keeps failing" (warn)
|
|
220
|
+
* - 5+ provider_error in window → "Provider outage" (error)
|
|
221
|
+
*/
|
|
222
|
+
export function diagnoseFailures(windowMs = 60 * 60_000) {
|
|
223
|
+
const events = readRecentEvents(2_000);
|
|
224
|
+
if (events.length === 0)
|
|
225
|
+
return [];
|
|
226
|
+
const cutoff = Date.now() - windowMs;
|
|
227
|
+
const recent = events.filter((e) => {
|
|
228
|
+
const t = Date.parse(e.ts);
|
|
229
|
+
return Number.isFinite(t) && t >= cutoff;
|
|
230
|
+
});
|
|
231
|
+
if (recent.length === 0)
|
|
232
|
+
return [];
|
|
233
|
+
const hints = [];
|
|
234
|
+
// Retry storm
|
|
235
|
+
const retries = recent.filter((e) => e.type === 'retry');
|
|
236
|
+
if (retries.length >= 3) {
|
|
237
|
+
hints.push({
|
|
238
|
+
summary: `${retries.length} retries in the last ${Math.round(windowMs / 60_000)} min`,
|
|
239
|
+
severity: 'warn',
|
|
240
|
+
count: retries.length,
|
|
241
|
+
suggestion: 'This often indicates a flaky network, a rate-limited provider, or a proxy timeout. ' +
|
|
242
|
+
'If the trend persists, switch providers with /model.',
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
// Provider cooldowns
|
|
246
|
+
const cooldowns = recent.filter((e) => e.type === 'cooldown');
|
|
247
|
+
if (cooldowns.length > 0) {
|
|
248
|
+
const providers = new Set(cooldowns.map((e) => String(e.fields.providerId ?? '?')));
|
|
249
|
+
hints.push({
|
|
250
|
+
summary: `Provider cooldown: ${[...providers].join(', ')}`,
|
|
251
|
+
severity: 'warn',
|
|
252
|
+
count: cooldowns.length,
|
|
253
|
+
suggestion: 'One or more providers are rate-limiting requests. The cooldown manager will ' +
|
|
254
|
+
'prefer other providers automatically. Add more API keys at the FreeLLMAPI dashboard.',
|
|
255
|
+
});
|
|
256
|
+
}
|
|
257
|
+
// Stream resume exhausted
|
|
258
|
+
const exhausted = recent.filter((e) => e.type === 'stream_resume_exhausted');
|
|
259
|
+
if (exhausted.length > 0) {
|
|
260
|
+
hints.push({
|
|
261
|
+
summary: `${exhausted.length} stream(s) failed to recover after ${exhausted.length} resume attempts`,
|
|
262
|
+
severity: 'error',
|
|
263
|
+
count: exhausted.length,
|
|
264
|
+
suggestion: 'Mid-stream cuts are exceeding the resume budget. Check your network stability, ' +
|
|
265
|
+
'or raise `preferences.resilience.maxResumeAttempts` in the config.',
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
// Context budget compaction
|
|
269
|
+
const compactions = recent.filter((e) => {
|
|
270
|
+
if (e.type !== 'context_budget')
|
|
271
|
+
return false;
|
|
272
|
+
return e.fields.markedForCompaction === true;
|
|
273
|
+
});
|
|
274
|
+
if (compactions.length > 0) {
|
|
275
|
+
hints.push({
|
|
276
|
+
summary: `Context window filling up (${compactions.length} compaction(s) requested)`,
|
|
277
|
+
severity: 'info',
|
|
278
|
+
count: compactions.length,
|
|
279
|
+
suggestion: 'The agent is summarising old turns to stay within the model window. This is normal ' +
|
|
280
|
+
'for long sessions; consider /clear between unrelated tasks.',
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
// Tool-call failure clustering
|
|
284
|
+
const toolFailures = new Map();
|
|
285
|
+
for (const e of recent) {
|
|
286
|
+
if (e.type !== 'tool_call')
|
|
287
|
+
continue;
|
|
288
|
+
if (e.fields.status !== 'failed')
|
|
289
|
+
continue;
|
|
290
|
+
const tool = String(e.fields.tool ?? '?');
|
|
291
|
+
toolFailures.set(tool, (toolFailures.get(tool) ?? 0) + 1);
|
|
292
|
+
}
|
|
293
|
+
for (const [tool, count] of toolFailures) {
|
|
294
|
+
if (count >= 3) {
|
|
295
|
+
hints.push({
|
|
296
|
+
summary: `Tool "${tool}" failed ${count} times in the window`,
|
|
297
|
+
severity: 'warn',
|
|
298
|
+
count,
|
|
299
|
+
suggestion: `The ${tool} tool keeps failing. Check its inputs (paths, arguments) and ` +
|
|
300
|
+
'consider whether the workspace permissions allow the operation.',
|
|
301
|
+
});
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
// Provider outage
|
|
305
|
+
const providerErrors = recent.filter((e) => e.type === 'provider_error');
|
|
306
|
+
if (providerErrors.length >= 5) {
|
|
307
|
+
hints.push({
|
|
308
|
+
summary: `${providerErrors.length} provider errors in the window`,
|
|
309
|
+
severity: 'error',
|
|
310
|
+
count: providerErrors.length,
|
|
311
|
+
suggestion: 'A provider is likely experiencing an outage. The agent will try to fall back to ' +
|
|
312
|
+
'other providers, but the session may be slow until the issue clears.',
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
return hints;
|
|
316
|
+
}
|
|
317
|
+
// ---------------------------------------------------------------------------
|
|
318
|
+
// Helpers
|
|
319
|
+
// ---------------------------------------------------------------------------
|
|
320
|
+
function isLocalEnabled() {
|
|
321
|
+
try {
|
|
322
|
+
const config = loadConfig();
|
|
323
|
+
if (config.preferences?.telemetry === false)
|
|
324
|
+
return false;
|
|
325
|
+
// Default ON when telemetry is on. The `telemetryLocal` flag
|
|
326
|
+
// gives the user a per-sink opt-out.
|
|
327
|
+
if (config.preferences?.telemetryLocal === false)
|
|
328
|
+
return false;
|
|
329
|
+
return true;
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
return false;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
function isRemoteEnabled() {
|
|
336
|
+
try {
|
|
337
|
+
const config = loadConfig();
|
|
338
|
+
if (config.preferences?.telemetry === false)
|
|
339
|
+
return false;
|
|
340
|
+
if (config.preferences?.telemetryRemote === true)
|
|
341
|
+
return true;
|
|
342
|
+
return false;
|
|
343
|
+
}
|
|
344
|
+
catch {
|
|
345
|
+
return false;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
// ---------------------------------------------------------------------------
|
|
349
|
+
// Legacy public surface — preserved for the 7 existing callsites
|
|
350
|
+
// ---------------------------------------------------------------------------
|
|
351
|
+
/**
|
|
352
|
+
* @deprecated Prefer the typed helpers on the `telemetry` object and
|
|
353
|
+
* `recordTelemetry()` directly. This wrapper maps a `TelemetryPayload`
|
|
354
|
+
* to a `tool_call` event.
|
|
355
|
+
*/
|
|
356
|
+
export async function logTelemetry(payload) {
|
|
357
|
+
// Respect user opt-out
|
|
358
|
+
if (!isLocalEnabled() && !isRemoteEnabled())
|
|
359
|
+
return;
|
|
360
|
+
// Prevent async activity errors in tests
|
|
361
|
+
if (process.env.NODE_ENV === 'test' ||
|
|
362
|
+
process.argv.some((arg) => arg.includes('jest') || arg.includes('vitest') || arg.includes('mocha'))) {
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
const event = telemetry.toolCall({
|
|
366
|
+
tool: payload.tool,
|
|
367
|
+
status: payload.status,
|
|
368
|
+
error: payload.error,
|
|
369
|
+
});
|
|
370
|
+
recordTelemetry(event);
|
|
371
|
+
if (!isRemoteEnabled())
|
|
372
|
+
return;
|
|
373
|
+
try {
|
|
374
|
+
const config = loadConfig();
|
|
375
|
+
const baseUrl = config.apiUrl || 'http://localhost:3001/v1';
|
|
376
|
+
let logUrl = 'http://localhost:3001/api/mcp/log';
|
|
377
|
+
try {
|
|
378
|
+
const url = new URL(baseUrl);
|
|
379
|
+
logUrl = `${url.protocol}//${url.host}/api/mcp/log`;
|
|
380
|
+
}
|
|
381
|
+
catch (err) {
|
|
382
|
+
if (process.env.DEBUG || process.env.VERBOSE || process.argv.includes('--verbose')) {
|
|
383
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
384
|
+
console.warn(`[Debug Warning] Telemetry failed to parse baseUrl ${baseUrl}: ${msg}`);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
await fetch(logUrl, {
|
|
388
|
+
method: 'POST',
|
|
389
|
+
headers: { 'Content-Type': 'application/json' },
|
|
390
|
+
body: JSON.stringify(payload),
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
catch (error) {
|
|
394
|
+
if (process.env.DEBUG || process.env.VERBOSE || process.argv.includes('--verbose')) {
|
|
395
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
396
|
+
console.warn(`[Debug Warning] Telemetry submission failed: ${msg}`);
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
//# sourceMappingURL=telemetry.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../../src/agent/telemetry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAkDxD,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E,SAAS,SAAS,CAAC,IAAwB,EAAE,MAA+B;IAC1E,OAAO;QACL,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QAC5B,IAAI;QACJ,GAAG,EAAE,YAAY,EAAE;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,MAAM,EAAE,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,QAAQ,CAAC,MAAyG;QAChH,OAAO,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,CAAC,MAAuE;QAC3E,OAAO,SAAS,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpC,CAAC;IACD,QAAQ,CAAC,MAA2F;QAClG,OAAO,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC;IACD,YAAY,CAAC,MAAsF;QACjG,OAAO,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAC;IACpF,CAAC;IACD,aAAa,CAAC,MAAsG;QAClH,OAAO,SAAS,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,aAAa,CAAC,MAA+D;QAC3E,OAAO,SAAS,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,YAAY,CAAC,MAAsC;QACjD,OAAO,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,UAAU,CAAC,MAAsE;QAC/E,OAAO,SAAS,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IACD,YAAY,CAAC,MAA0E;QACrF,OAAO,SAAS,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,CAAC,MAAiE;QACpE,OAAO,SAAS,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IACxC,CAAC;IACD,YAAY,CAAC,MAAyC;QACpD,OAAO,SAAS,CAAC,gBAAgB,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,YAAY,CAAC,MAAkD;QAC7D,OAAO,SAAS,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,eAAe,CAAC,MAA0E;QACxF,OAAO,SAAS,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IACD,UAAU,CAAC,MAAoD;QAC7D,OAAO,SAAS,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IACD,eAAe,CAAC,MAA2F;QACzG,OAAO,SAAS,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IACD,SAAS,CAAC,MAAqF;QAC7F,OAAO,SAAS,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,kBAAkB,CAAC,MAA2D;QAC5E,OAAO,SAAS,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;CACF,CAAC;AAEF,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,IAAI,UAAU,GAAkB,IAAI,CAAC;AACrC,SAAS,YAAY;IACnB,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAClC,6DAA6D;IAC7D,+DAA+D;IAC/D,gEAAgE;IAChE,6DAA6D;IAC7D,qCAAqC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9C,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAClC,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;iCACiC;AACjC,MAAM,UAAU,gBAAgB;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,iBAAiB,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,SAAS,GAAG,SAAS,CAAC,CAAC,QAAQ;AAErC,SAAS,SAAS;IAChB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,SAAS,cAAc;IACrB,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;IAChC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO;IACjC,IAAI,IAAc,CAAC;IACnB,IAAI,CAAC;QACH,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,GAAG,SAAS;QAAE,OAAO;IAClC,MAAM,MAAM,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3B,IAAI,CAAC;QACH,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,8DAA8D;QAC9D,+CAA+C;IACjD,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,KAAqB;IACnD,IAAI,CAAC,cAAc,EAAE;QAAE,OAAO,KAAK,CAAC;IACpC,IAAI,CAAC;QACH,SAAS,EAAE,CAAC;QACZ,cAAc,EAAE,CAAC;QACjB,EAAE,CAAC,cAAc,CACf,gBAAgB,EAAE,EAClB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,EAC5B,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CACnC,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;6CAC6C;AAC7C,MAAM,UAAU,gBAAgB,CAAC,QAAgB,GAAG;IAClD,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;IAChC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAqB,EAAE,CAAC;QACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmB,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,kDAAkD;YACpD,CAAC;QACH,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,8DAA8D;AAC9D,MAAM,UAAU,cAAc;IAC5B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,gBAAgB,EAAE,CAAC;QAChC,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,SAAS;IACX,CAAC;AACH,CAAC;AAiBD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,gBAAgB,CAAC,WAAmB,EAAE,GAAG,MAAM;IAC7D,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;IACrC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3B,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC;IAC3C,CAAC,CAAC,CAAC;IACH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,MAAM,KAAK,GAAoB,EAAE,CAAC;IAElC,cAAc;IACd,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC;IACzD,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,wBAAwB,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM;YACrF,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,OAAO,CAAC,MAAM;YACrB,UAAU,EACR,qFAAqF;gBACrF,sDAAsD;SACzD,CAAC,CAAC;IACL,CAAC;IAED,qBAAqB;IACrB,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;IAC9D,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC;QACpF,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,sBAAsB,CAAC,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;YAC1D,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,SAAS,CAAC,MAAM;YACvB,UAAU,EACR,8EAA8E;gBAC9E,sFAAsF;SACzF,CAAC,CAAC;IACL,CAAC;IAED,0BAA0B;IAC1B,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,yBAAyB,CAAC,CAAC;IAC7E,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,GAAG,SAAS,CAAC,MAAM,sCAAsC,SAAS,CAAC,MAAM,kBAAkB;YACpG,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,SAAS,CAAC,MAAM;YACvB,UAAU,EACR,iFAAiF;gBACjF,oEAAoE;SACvE,CAAC,CAAC;IACL,CAAC;IAED,4BAA4B;IAC5B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACtC,IAAI,CAAC,CAAC,IAAI,KAAK,gBAAgB;YAAE,OAAO,KAAK,CAAC;QAC9C,OAAO,CAAC,CAAC,MAAM,CAAC,mBAAmB,KAAK,IAAI,CAAC;IAC/C,CAAC,CAAC,CAAC;IACH,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,8BAA8B,WAAW,CAAC,MAAM,2BAA2B;YACpF,QAAQ,EAAE,MAAM;YAChB,KAAK,EAAE,WAAW,CAAC,MAAM;YACzB,UAAU,EACR,qFAAqF;gBACrF,6DAA6D;SAChE,CAAC,CAAC;IACL,CAAC;IAED,+BAA+B;IAC/B,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,IAAI,KAAK,WAAW;YAAE,SAAS;QACrC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ;YAAE,SAAS;QAC3C,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC;QAC1C,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;QACzC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,KAAK,CAAC,IAAI,CAAC;gBACT,OAAO,EAAE,SAAS,IAAI,YAAY,KAAK,sBAAsB;gBAC7D,QAAQ,EAAE,MAAM;gBAChB,KAAK;gBACL,UAAU,EACR,OAAO,IAAI,+DAA+D;oBAC1E,iEAAiE;aACpE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,kBAAkB;IAClB,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,gBAAgB,CAAC,CAAC;IACzE,IAAI,cAAc,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC;YACT,OAAO,EAAE,GAAG,cAAc,CAAC,MAAM,gCAAgC;YACjE,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,cAAc,CAAC,MAAM;YAC5B,UAAU,EACR,kFAAkF;gBAClF,sEAAsE;SACzE,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,UAAU;AACV,8EAA8E;AAE9E,SAAS,cAAc;IACrB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QAC1D,6DAA6D;QAC7D,qCAAqC;QACrC,IAAI,MAAM,CAAC,WAAW,EAAE,cAAc,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QAC/D,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,eAAe;IACtB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,IAAI,MAAM,CAAC,WAAW,EAAE,SAAS,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QAC1D,IAAI,MAAM,CAAC,WAAW,EAAE,eAAe,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAC9D,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,iEAAiE;AACjE,8EAA8E;AAE9E;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAyB;IAC1D,uBAAuB;IACvB,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,eAAe,EAAE;QAAE,OAAO;IAEpD,yCAAyC;IACzC,IACE,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,MAAM;QAC/B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,EACnG,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC;QAC/B,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,KAAK,EAAE,OAAO,CAAC,KAAK;KACrB,CAAC,CAAC;IACH,eAAe,CAAC,KAAK,CAAC,CAAC;IAEvB,IAAI,CAAC,eAAe,EAAE;QAAE,OAAO;IAE/B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,UAAU,EAAE,CAAC;QAC5B,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,IAAI,0BAA0B,CAAC;QAC5D,IAAI,MAAM,GAAG,mCAAmC,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;YAC7B,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,IAAI,cAAc,CAAC;QACtD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;gBACnF,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC7D,OAAO,CAAC,IAAI,CAAC,qDAAqD,OAAO,KAAK,GAAG,EAAE,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QAED,MAAM,KAAK,CAAC,MAAM,EAAE;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;YACnF,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACnE,OAAO,CAAC,IAAI,CAAC,gDAAgD,GAAG,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tokenizer — model-aware BPE token counter built on `gpt-tokenizer`.
|
|
3
|
+
*
|
|
4
|
+
* The FreeLLMAPI proxy fronts many providers (OpenAI, Anthropic, Google,
|
|
5
|
+
* Groq, Cerebras, SambaNova, Mistral, OpenRouter, Cloudflare, Cohere,
|
|
6
|
+
* Zen/NVIDIA). None of them expose their native tokenizer through the
|
|
7
|
+
* proxy, and we do not bundle one for every family. We therefore use
|
|
8
|
+
* OpenAI's public BPE encodings as a *close-enough* proxy:
|
|
9
|
+
*
|
|
10
|
+
* - `cl100k_base` — GPT-4 / GPT-3.5-turbo / most modern BPE families.
|
|
11
|
+
* Used as the default for every model the proxy serves.
|
|
12
|
+
* - `o200k_base` — GPT-4o / GPT-4.1 family. About 15% more efficient
|
|
13
|
+
* (fewer tokens per word) than cl100k.
|
|
14
|
+
*
|
|
15
|
+
* For non-OpenAI providers, our counts will be within ~10-20% of the
|
|
16
|
+
* provider's true bill. That is more than accurate enough for the
|
|
17
|
+
* purpose of *preventing context overflow* (the consequence of an
|
|
18
|
+
* inaccurate count is at worst an early compaction, not a 413).
|
|
19
|
+
*
|
|
20
|
+
* The tokenizer is loaded lazily on first use because the encoding
|
|
21
|
+
* tables are several MB and would otherwise inflate CLI cold-start.
|
|
22
|
+
*/
|
|
23
|
+
import type { ChatContentBlock } from '../shared/types.js';
|
|
24
|
+
/** Names of the encoders we ship. Kept as a closed union for type safety. */
|
|
25
|
+
export type EncoderName = 'cl100k_base' | 'o200k_base';
|
|
26
|
+
/**
|
|
27
|
+
* Pick the right BPE encoding for a given model identifier. The mapping
|
|
28
|
+
* is conservative: any model name we are not sure about gets cl100k_base,
|
|
29
|
+
* which is the universal BPE that every modern OpenAI-adjacent model is
|
|
30
|
+
* based on. GPT-4o / GPT-4.1 use the newer o200k_base vocabulary.
|
|
31
|
+
*/
|
|
32
|
+
export declare function resolveEncoderForModel(model: string | undefined | null): EncoderName;
|
|
33
|
+
/** Count tokens in a single string. */
|
|
34
|
+
export declare function countTokens(text: string, model?: string | null): number;
|
|
35
|
+
/** Count tokens across a list of message-like objects. */
|
|
36
|
+
export declare function countMessagesTokens(messages: ReadonlyArray<{
|
|
37
|
+
content?: string | ChatContentBlock[] | null;
|
|
38
|
+
tool_calls?: unknown;
|
|
39
|
+
}>, model?: string | null): number;
|
|
40
|
+
/** Reset the encoder cache (test-only; not currently used). */
|
|
41
|
+
export declare function _resetTokenizerCache(): void;
|
|
42
|
+
//# sourceMappingURL=tokenizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/agent/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAIH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAO3D,6EAA6E;AAC7E,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,YAAY,CAAC;AAYvD;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI,GAAG,WAAW,CAWpF;AAiBD,uCAAuC;AACvC,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAIvE;AAED,0DAA0D;AAC1D,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,aAAa,CAAC;IACtB,OAAO,CAAC,EAAE,MAAM,GAAG,gBAAgB,EAAE,GAAG,IAAI,CAAC;IAC7C,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,CAAC,EACF,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,GACpB,MAAM,CA6BR;AAED,+DAA+D;AAC/D,wBAAgB,oBAAoB,IAAI,IAAI,CAI3C"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tokenizer — model-aware BPE token counter built on `gpt-tokenizer`.
|
|
3
|
+
*
|
|
4
|
+
* The FreeLLMAPI proxy fronts many providers (OpenAI, Anthropic, Google,
|
|
5
|
+
* Groq, Cerebras, SambaNova, Mistral, OpenRouter, Cloudflare, Cohere,
|
|
6
|
+
* Zen/NVIDIA). None of them expose their native tokenizer through the
|
|
7
|
+
* proxy, and we do not bundle one for every family. We therefore use
|
|
8
|
+
* OpenAI's public BPE encodings as a *close-enough* proxy:
|
|
9
|
+
*
|
|
10
|
+
* - `cl100k_base` — GPT-4 / GPT-3.5-turbo / most modern BPE families.
|
|
11
|
+
* Used as the default for every model the proxy serves.
|
|
12
|
+
* - `o200k_base` — GPT-4o / GPT-4.1 family. About 15% more efficient
|
|
13
|
+
* (fewer tokens per word) than cl100k.
|
|
14
|
+
*
|
|
15
|
+
* For non-OpenAI providers, our counts will be within ~10-20% of the
|
|
16
|
+
* provider's true bill. That is more than accurate enough for the
|
|
17
|
+
* purpose of *preventing context overflow* (the consequence of an
|
|
18
|
+
* inaccurate count is at worst an early compaction, not a 413).
|
|
19
|
+
*
|
|
20
|
+
* The tokenizer is loaded lazily on first use because the encoding
|
|
21
|
+
* tables are several MB and would otherwise inflate CLI cold-start.
|
|
22
|
+
*/
|
|
23
|
+
import { encode as cl100kEncode } from 'gpt-tokenizer/encoding/cl100k_base';
|
|
24
|
+
import { encode as o200kEncode } from 'gpt-tokenizer/encoding/o200k_base';
|
|
25
|
+
import { IMAGE_TOKEN_COST } from '../shared/content.js';
|
|
26
|
+
// ---------------------------------------------------------------------------
|
|
27
|
+
// Encoder resolution
|
|
28
|
+
// ---------------------------------------------------------------------------
|
|
29
|
+
/**
|
|
30
|
+
* Pick the right BPE encoding for a given model identifier. The mapping
|
|
31
|
+
* is conservative: any model name we are not sure about gets cl100k_base,
|
|
32
|
+
* which is the universal BPE that every modern OpenAI-adjacent model is
|
|
33
|
+
* based on. GPT-4o / GPT-4.1 use the newer o200k_base vocabulary.
|
|
34
|
+
*/
|
|
35
|
+
export function resolveEncoderForModel(model) {
|
|
36
|
+
if (!model)
|
|
37
|
+
return 'cl100k_base';
|
|
38
|
+
// GPT-4o and GPT-4.1 use the new o200k_base vocabulary.
|
|
39
|
+
if (/\bgpt-4o\b|\bgpt-4\.1\b|\bo1\b|\bo3\b|\bo4\b/i.test(model)) {
|
|
40
|
+
return 'o200k_base';
|
|
41
|
+
}
|
|
42
|
+
// Everything else — including claude, llama, gemini, mistral, qwen,
|
|
43
|
+
// deepseek, codestral, gpt-3.5, gpt-4 (non-4o), command-r — falls
|
|
44
|
+
// back to cl100k_base. The token count is approximate, but close
|
|
45
|
+
// enough to prevent overflows.
|
|
46
|
+
return 'cl100k_base';
|
|
47
|
+
}
|
|
48
|
+
const ENCODERS = {
|
|
49
|
+
cl100k_base: {
|
|
50
|
+
name: 'cl100k_base',
|
|
51
|
+
encode: (input) => cl100kEncode(input),
|
|
52
|
+
},
|
|
53
|
+
o200k_base: {
|
|
54
|
+
name: 'o200k_base',
|
|
55
|
+
encode: (input) => o200kEncode(input),
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// Public API
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
/** Count tokens in a single string. */
|
|
62
|
+
export function countTokens(text, model) {
|
|
63
|
+
if (!text)
|
|
64
|
+
return 0;
|
|
65
|
+
const encoder = ENCODERS[resolveEncoderForModel(model)];
|
|
66
|
+
return encoder.encode(text).length;
|
|
67
|
+
}
|
|
68
|
+
/** Count tokens across a list of message-like objects. */
|
|
69
|
+
export function countMessagesTokens(messages, model) {
|
|
70
|
+
let total = 0;
|
|
71
|
+
// Each message carries a small per-message framing overhead (role label,
|
|
72
|
+
// separators). 4 tokens is the OpenAI cookbook figure.
|
|
73
|
+
const PER_MESSAGE_OVERHEAD = 4;
|
|
74
|
+
for (const m of messages) {
|
|
75
|
+
total += PER_MESSAGE_OVERHEAD;
|
|
76
|
+
const c = m.content;
|
|
77
|
+
if (typeof c === 'string' && c.length > 0) {
|
|
78
|
+
total += countTokens(c, model);
|
|
79
|
+
}
|
|
80
|
+
else if (Array.isArray(c)) {
|
|
81
|
+
for (const block of c) {
|
|
82
|
+
if (block.type === 'text' && block.text.length > 0) {
|
|
83
|
+
total += countTokens(block.text, model);
|
|
84
|
+
}
|
|
85
|
+
else if (block.type === 'image') {
|
|
86
|
+
// Fixed estimate per the Phase 2 plan. See shared/content.ts.
|
|
87
|
+
total += IMAGE_TOKEN_COST;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
if (m.tool_calls) {
|
|
92
|
+
// We do not have a dedicated BPE for tool-call JSON; the per-message
|
|
93
|
+
// overhead and the content (if any) are usually enough. Add a
|
|
94
|
+
// conservative flat cost per tool call for safety.
|
|
95
|
+
total += JSON.stringify(m.tool_calls).length / 3;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
// The conversation as a whole carries a final 2-token framing cost.
|
|
99
|
+
return Math.ceil(total) + 2;
|
|
100
|
+
}
|
|
101
|
+
/** Reset the encoder cache (test-only; not currently used). */
|
|
102
|
+
export function _resetTokenizerCache() {
|
|
103
|
+
// The encoder table is module-level and immutable, so there is nothing
|
|
104
|
+
// to flush. This stub is here for symmetry with future cache layers
|
|
105
|
+
// and to give tests an obvious hook.
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=tokenizer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/agent/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,MAAM,IAAI,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAC5E,OAAO,EAAE,MAAM,IAAI,WAAW,EAAE,MAAM,mCAAmC,CAAC;AAE1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAexD,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,KAAgC;IACrE,IAAI,CAAC,KAAK;QAAE,OAAO,aAAa,CAAC;IACjC,wDAAwD;IACxD,IAAI,+CAA+C,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAChE,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,oEAAoE;IACpE,kEAAkE;IAClE,iEAAiE;IACjE,+BAA+B;IAC/B,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,MAAM,QAAQ,GAAiC;IAC7C,WAAW,EAAE;QACX,IAAI,EAAE,aAAa;QACnB,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC;KACvC;IACD,UAAU,EAAE;QACV,IAAI,EAAE,YAAY;QAClB,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC;KACtC;CACF,CAAC;AAEF,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,uCAAuC;AACvC,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAAqB;IAC7D,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IACpB,MAAM,OAAO,GAAG,QAAQ,CAAC,sBAAsB,CAAC,KAAK,CAAC,CAAC,CAAC;IACxD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACrC,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,mBAAmB,CACjC,QAGE,EACF,KAAqB;IAErB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,yEAAyE;IACzE,uDAAuD;IACvD,MAAM,oBAAoB,GAAG,CAAC,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,KAAK,IAAI,oBAAoB,CAAC;QAC9B,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC;QACpB,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,KAAK,IAAI,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACjC,CAAC;aAAM,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5B,KAAK,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC;gBACtB,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACnD,KAAK,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC1C,CAAC;qBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;oBAClC,8DAA8D;oBAC9D,KAAK,IAAI,gBAAgB,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;YACjB,qEAAqE;YACrE,8DAA8D;YAC9D,mDAAmD;YACnD,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QACnD,CAAC;IACH,CAAC;IACD,oEAAoE;IACpE,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,oBAAoB;IAClC,uEAAuE;IACvE,oEAAoE;IACpE,qCAAqC;AACvC,CAAC"}
|