@xalia/agent 0.6.9 → 0.6.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.development +6 -1
- package/.env.test +7 -0
- package/README.md +11 -0
- package/context_system.md +498 -0
- package/dist/agent/src/agent/agent.js +77 -18
- package/dist/agent/src/agent/agentUtils.js +3 -2
- package/dist/agent/src/agent/documentSummarizer.js +126 -0
- package/dist/agent/src/agent/dummyLLM.js +25 -22
- package/dist/agent/src/agent/imageGenLLM.js +22 -19
- package/dist/agent/src/agent/llm.js +1 -1
- package/dist/agent/src/agent/openAILLM.js +15 -12
- package/dist/agent/src/agent/openAILLMStreaming.js +68 -37
- package/dist/agent/src/agent/repeatLLM.js +16 -7
- package/dist/agent/src/agent/tokenCounter.js +390 -0
- package/dist/agent/src/agent/tokenCounter.test.js +206 -0
- package/dist/agent/src/agent/toolSettings.js +17 -0
- package/dist/agent/src/agent/tools/calculatorTool.js +45 -0
- package/dist/agent/src/agent/tools/contentExtractors/pdfToText.js +55 -0
- package/dist/agent/src/agent/tools/datetimeTool.js +38 -0
- package/dist/agent/src/agent/tools/fileManager/fileManagerTool.js +156 -0
- package/dist/agent/src/agent/tools/fileManager/index.js +31 -0
- package/dist/agent/src/agent/tools/fileManager/memoryFileManager.js +102 -0
- package/dist/agent/src/{chat/data → agent/tools/fileManager}/mimeTypes.js +3 -1
- package/dist/agent/src/agent/tools/fileManager/prompt.js +33 -0
- package/dist/agent/src/{chat/data/dbSessionFileModels.js → agent/tools/fileManager/types.js} +7 -0
- package/dist/agent/src/agent/tools/index.js +64 -0
- package/dist/agent/src/agent/tools/openUrlTool.js +57 -0
- package/dist/agent/src/agent/tools/renderTool.js +89 -0
- package/dist/agent/src/agent/tools/utils.js +61 -0
- package/dist/agent/src/{chat/utils/search.js → agent/tools/webSearch.js} +1 -2
- package/dist/agent/src/agent/tools/webSearchTool.js +40 -0
- package/dist/agent/src/chat/client/chatClient.js +28 -0
- package/dist/agent/src/chat/client/index.js +4 -1
- package/dist/agent/src/chat/client/sessionClient.js +28 -2
- package/dist/agent/src/chat/constants.js +8 -0
- package/dist/agent/src/chat/data/dbSessionFiles.js +11 -6
- package/dist/agent/src/chat/protocol/messages.js +5 -0
- package/dist/agent/src/chat/server/chatContextManager.js +45 -25
- package/dist/agent/src/chat/server/conversation.js +3 -0
- package/dist/agent/src/chat/server/imageGeneratorTools.js +20 -8
- package/dist/agent/src/chat/server/openAIRouterLLM.js +0 -3
- package/dist/agent/src/chat/server/openSession.js +218 -55
- package/dist/agent/src/chat/server/promptRefiner.js +86 -0
- package/dist/agent/src/chat/server/server.js +5 -1
- package/dist/agent/src/chat/server/sessionFileManager.js +22 -221
- package/dist/agent/src/chat/server/sessionRegistry.js +87 -0
- package/dist/agent/src/chat/server/titleGenerator.js +112 -0
- package/dist/agent/src/chat/server/titleGenerator.test.js +113 -0
- package/dist/agent/src/chat/server/tools.js +63 -287
- package/dist/agent/src/chat/utils/approvalManager.js +6 -3
- package/dist/agent/src/chat/utils/multiAsyncQueue.js +3 -0
- package/dist/agent/src/test/agent.test.js +16 -17
- package/dist/agent/src/test/chatContextManager.test.js +15 -3
- package/dist/agent/src/test/dbMcpServerConfigs.test.js +4 -4
- package/dist/agent/src/test/dbSessionFiles.test.js +17 -17
- package/dist/agent/src/test/testTools.js +6 -1
- package/dist/agent/src/test/tools.test.js +27 -9
- package/dist/agent/src/tool/agentChat.js +5 -2
- package/dist/agent/src/tool/chatMain.js +34 -7
- package/dist/agent/src/tool/commandPrompt.js +2 -2
- package/dist/agent/src/tool/files.js +7 -8
- package/package.json +4 -1
- package/scripts/test_chat +195 -176
- package/src/agent/agent.ts +98 -23
- package/src/agent/agentUtils.ts +3 -2
- package/src/agent/documentSummarizer.ts +157 -0
- package/src/agent/dummyLLM.ts +27 -23
- package/src/agent/imageGenLLM.ts +28 -24
- package/src/agent/llm.ts +2 -2
- package/src/agent/openAILLM.ts +17 -13
- package/src/agent/openAILLMStreaming.ts +80 -41
- package/src/agent/repeatLLM.ts +19 -7
- package/src/agent/test_data/harrypotter.txt +6065 -0
- package/src/agent/tokenCounter.test.ts +243 -0
- package/src/agent/tokenCounter.ts +483 -0
- package/src/agent/toolSettings.ts +24 -0
- package/src/agent/tools/calculatorTool.ts +50 -0
- package/src/agent/tools/contentExtractors/pdfToText.ts +60 -0
- package/src/agent/tools/datetimeTool.ts +41 -0
- package/src/agent/tools/fileManager/fileManagerTool.ts +199 -0
- package/src/agent/tools/fileManager/index.ts +50 -0
- package/src/agent/tools/fileManager/memoryFileManager.ts +120 -0
- package/src/{chat/data → agent/tools/fileManager}/mimeTypes.ts +3 -1
- package/src/agent/tools/fileManager/prompt.ts +38 -0
- package/src/{chat/data/dbSessionFileModels.ts → agent/tools/fileManager/types.ts} +76 -0
- package/src/agent/tools/index.ts +49 -0
- package/src/agent/tools/openUrlTool.ts +62 -0
- package/src/agent/tools/renderTool.ts +92 -0
- package/src/agent/tools/utils.ts +74 -0
- package/src/{chat/utils/search.ts → agent/tools/webSearch.ts} +0 -1
- package/src/agent/tools/webSearchTool.ts +44 -0
- package/src/chat/client/chatClient.ts +45 -0
- package/src/chat/client/index.ts +3 -0
- package/src/chat/client/sessionClient.ts +40 -3
- package/src/chat/client/sessionFiles.ts +1 -1
- package/src/chat/constants.ts +6 -0
- package/src/chat/data/dataModels.ts +6 -0
- package/src/chat/data/dbSessionFiles.ts +12 -4
- package/src/chat/protocol/messages.ts +60 -7
- package/src/chat/server/chatContextManager.ts +58 -37
- package/src/chat/server/conversation.ts +3 -0
- package/src/chat/server/imageGeneratorTools.ts +31 -12
- package/src/chat/server/openAIRouterLLM.ts +1 -4
- package/src/chat/server/openSession.ts +323 -67
- package/src/chat/server/promptRefiner.ts +106 -0
- package/src/chat/server/server.ts +4 -1
- package/src/chat/server/sessionFileManager.ts +35 -306
- package/src/chat/server/sessionRegistry.ts +128 -0
- package/src/chat/server/titleGenerator.test.ts +103 -0
- package/src/chat/server/titleGenerator.ts +143 -0
- package/src/chat/server/tools.ts +77 -304
- package/src/chat/utils/approvalManager.ts +9 -3
- package/src/chat/utils/multiAsyncQueue.ts +4 -0
- package/src/test/agent.test.ts +17 -23
- package/src/test/chatContextManager.test.ts +29 -4
- package/src/test/dbMcpServerConfigs.test.ts +4 -4
- package/src/test/dbSessionFiles.test.ts +16 -16
- package/src/test/testTools.ts +8 -3
- package/src/test/tools.test.ts +30 -5
- package/src/tool/agentChat.ts +12 -3
- package/src/tool/chatMain.ts +33 -6
- package/src/tool/commandPrompt.ts +2 -2
- package/src/tool/files.ts +1 -3
- package/dist/agent/src/agent/tools.js +0 -44
- package/src/agent/tools.ts +0 -57
- /package/dist/agent/src/{chat/utils → agent/tools/contentExtractors}/htmlToText.js +0 -0
- /package/src/{chat/utils → agent/tools/contentExtractors}/htmlToText.ts +0 -0
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.TokenCounter = exports.ModelDetector = void 0;
|
|
7
|
+
exports.createTokenCounter = createTokenCounter;
|
|
8
|
+
exports.countTokensQuick = countTokensQuick;
|
|
9
|
+
exports.getContextWindowSize = getContextWindowSize;
|
|
10
|
+
const tiktoken_1 = require("tiktoken");
|
|
11
|
+
const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
|
|
12
|
+
/**
|
|
13
|
+
* Model context window sizes (from OpenRouter API)
|
|
14
|
+
* Only includes models currently used in DEFAULT_MODEL_MAP
|
|
15
|
+
*/
|
|
16
|
+
const MODEL_CONTEXT_WINDOWS = {
|
|
17
|
+
// OpenAI models
|
|
18
|
+
"gpt-4o-mini": 128000,
|
|
19
|
+
"gpt-4o": 128000,
|
|
20
|
+
"openai/gpt-4o-mini": 128000,
|
|
21
|
+
"openai/gpt-4o": 128000,
|
|
22
|
+
// Google models
|
|
23
|
+
"google/gemini-2.5-flash": 1048576,
|
|
24
|
+
"google/gemini-2.5-pro": 1048576,
|
|
25
|
+
"google/gemini-2.5-flash-image-preview": 32768,
|
|
26
|
+
// Anthropic models
|
|
27
|
+
"anthropic/claude-3.7-sonnet": 200000,
|
|
28
|
+
"anthropic/claude-sonnet-4": 1000000,
|
|
29
|
+
"anthropic/claude-sonnet-4.5": 1000000,
|
|
30
|
+
"claude-3-7-sonnet-20250219": 200000,
|
|
31
|
+
};
|
|
32
|
+
const DEFAULT_CONTEXT_WINDOW = 128000;
|
|
33
|
+
/**
|
|
34
|
+
* Model provider detection and normalization
|
|
35
|
+
*/
|
|
36
|
+
exports.ModelDetector = {
|
|
37
|
+
/**
|
|
38
|
+
* Detect the provider from a model string
|
|
39
|
+
* Supports formats like:
|
|
40
|
+
* - "gpt-4o" -> openai
|
|
41
|
+
* - "openai/gpt-4o" -> openai
|
|
42
|
+
* - "anthropic/claude-sonnet-4.5" -> anthropic
|
|
43
|
+
* - "claude-3-7-sonnet-20250219" -> anthropic
|
|
44
|
+
* - "gemini-2.0-flash-exp" -> google
|
|
45
|
+
* - "google/gemini-2.0-flash-exp" -> google
|
|
46
|
+
*/
|
|
47
|
+
detectProvider(model) {
|
|
48
|
+
const lowerModel = model.toLowerCase();
|
|
49
|
+
// Check for provider prefix (e.g., "anthropic/", "openai/")
|
|
50
|
+
if (lowerModel.startsWith("openai/")) {
|
|
51
|
+
return "openai";
|
|
52
|
+
}
|
|
53
|
+
if (lowerModel.startsWith("anthropic/")) {
|
|
54
|
+
return "anthropic";
|
|
55
|
+
}
|
|
56
|
+
if (lowerModel.startsWith("google/")) {
|
|
57
|
+
return "google";
|
|
58
|
+
}
|
|
59
|
+
// Check for model name patterns
|
|
60
|
+
if (lowerModel.includes("gpt")) {
|
|
61
|
+
return "openai";
|
|
62
|
+
}
|
|
63
|
+
if (lowerModel.includes("claude")) {
|
|
64
|
+
return "anthropic";
|
|
65
|
+
}
|
|
66
|
+
if (lowerModel.includes("gemini")) {
|
|
67
|
+
return "google";
|
|
68
|
+
}
|
|
69
|
+
if (lowerModel.includes("o1") || lowerModel.includes("o3")) {
|
|
70
|
+
return "openai";
|
|
71
|
+
}
|
|
72
|
+
return "unknown";
|
|
73
|
+
},
|
|
74
|
+
/**
|
|
75
|
+
* Normalize model string for tokenizer
|
|
76
|
+
* Primary purpose: Remove provider prefixes
|
|
77
|
+
*
|
|
78
|
+
* Note: tiktoken supports most OpenAI model variants natively
|
|
79
|
+
* (gpt-4o-mini, gpt-4o-2024-11-20, o1-preview, etc.),
|
|
80
|
+
* so we don't need to map them.
|
|
81
|
+
*/
|
|
82
|
+
normalizeModel(model, _provider) {
|
|
83
|
+
// Remove provider prefix (e.g., "anthropic/" -> "")
|
|
84
|
+
// This is the main normalization needed for OpenRouter-style
|
|
85
|
+
// model strings
|
|
86
|
+
const normalized = model.replace(/^[^/]+\//, "");
|
|
87
|
+
// For OpenAI: tiktoken handles variants natively,
|
|
88
|
+
// just return normalized
|
|
89
|
+
// For Anthropic/Google: return normalized for their
|
|
90
|
+
// respective tokenizers
|
|
91
|
+
return normalized;
|
|
92
|
+
},
|
|
93
|
+
};
|
|
94
|
+
/**
|
|
95
|
+
* Multi-model token counter supporting OpenAI, Anthropic (Claude),
|
|
96
|
+
* and Google (Gemini)
|
|
97
|
+
*/
|
|
98
|
+
class TokenCounter {
|
|
99
|
+
constructor(model, options) {
|
|
100
|
+
this.encoder = null;
|
|
101
|
+
this.anthropicClient = null;
|
|
102
|
+
this.model = model;
|
|
103
|
+
this.provider = exports.ModelDetector.detectProvider(model);
|
|
104
|
+
this.normalizedModel = exports.ModelDetector.normalizeModel(model, this.provider);
|
|
105
|
+
// Initialize encoder based on provider
|
|
106
|
+
if (this.provider === "openai") {
|
|
107
|
+
try {
|
|
108
|
+
this.encoder = (0, tiktoken_1.encoding_for_model)(this.normalizedModel);
|
|
109
|
+
}
|
|
110
|
+
catch (_error) {
|
|
111
|
+
// Fallback to cl100k_base for unknown OpenAI models
|
|
112
|
+
this.encoder = (0, tiktoken_1.encoding_for_model)("gpt-4");
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
else if (this.provider === "anthropic") {
|
|
116
|
+
// Use p50k_base encoding for Claude (approximation)
|
|
117
|
+
this.encoder = (0, tiktoken_1.get_encoding)("p50k_base");
|
|
118
|
+
}
|
|
119
|
+
// Initialize Anthropic client for dev mode sanity checks
|
|
120
|
+
if (options?.enableDevMode && this.provider === "anthropic") {
|
|
121
|
+
const apiKey = process.env.ANTHROPIC_API_KEY;
|
|
122
|
+
if (apiKey) {
|
|
123
|
+
this.anthropicClient = new sdk_1.default({ apiKey });
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
getModel() {
|
|
128
|
+
return this.model;
|
|
129
|
+
}
|
|
130
|
+
getProvider() {
|
|
131
|
+
return this.provider;
|
|
132
|
+
}
|
|
133
|
+
getContextWindow() {
|
|
134
|
+
return MODEL_CONTEXT_WINDOWS[this.model] ?? DEFAULT_CONTEXT_WINDOW;
|
|
135
|
+
}
|
|
136
|
+
countTokens(text) {
|
|
137
|
+
if (!text || text.length === 0) {
|
|
138
|
+
return 0;
|
|
139
|
+
}
|
|
140
|
+
switch (this.provider) {
|
|
141
|
+
case "openai":
|
|
142
|
+
return this.countTokensOpenAI(text);
|
|
143
|
+
case "anthropic":
|
|
144
|
+
return this.countTokensAnthropic(text);
|
|
145
|
+
case "google":
|
|
146
|
+
return this.countTokensGoogle(text);
|
|
147
|
+
default:
|
|
148
|
+
return this.countTokensApproximation(text);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
countMessageTokens(message) {
|
|
152
|
+
const contentTokens = this.countMessageContent(message);
|
|
153
|
+
const overhead = this.getMessageOverhead(message);
|
|
154
|
+
return contentTokens + overhead;
|
|
155
|
+
}
|
|
156
|
+
countMessagesTokens(messages) {
|
|
157
|
+
if (messages.length === 0) {
|
|
158
|
+
return 0;
|
|
159
|
+
}
|
|
160
|
+
let total = 0;
|
|
161
|
+
// Count each message
|
|
162
|
+
for (const message of messages) {
|
|
163
|
+
total += this.countMessageTokens(message);
|
|
164
|
+
}
|
|
165
|
+
// Add conversation-level overhead
|
|
166
|
+
total += this.getConversationOverhead(messages.length);
|
|
167
|
+
return total;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Free resources (for encoders)
|
|
171
|
+
*/
|
|
172
|
+
free() {
|
|
173
|
+
if (this.encoder) {
|
|
174
|
+
this.encoder.free();
|
|
175
|
+
this.encoder = null;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* DEV ONLY: Get accurate token count from Anthropic API
|
|
180
|
+
* for sanity checking
|
|
181
|
+
* This requires enableDevMode: true in constructor and
|
|
182
|
+
* ANTHROPIC_API_KEY environment variable
|
|
183
|
+
* Use this to compare against p50k_base approximation
|
|
184
|
+
*/
|
|
185
|
+
async countTokensAccurate(text) {
|
|
186
|
+
if (this.provider !== "anthropic") {
|
|
187
|
+
// For non-Anthropic models, just use regular counting
|
|
188
|
+
return this.countTokens(text);
|
|
189
|
+
}
|
|
190
|
+
if (!this.anthropicClient) {
|
|
191
|
+
throw new Error("Accurate token counting requires enableDevMode: true " +
|
|
192
|
+
"and ANTHROPIC_API_KEY environment variable");
|
|
193
|
+
}
|
|
194
|
+
try {
|
|
195
|
+
const result = await this.anthropicClient.messages.countTokens({
|
|
196
|
+
model: this.normalizedModel,
|
|
197
|
+
messages: [{ role: "user", content: text }],
|
|
198
|
+
});
|
|
199
|
+
return result.input_tokens;
|
|
200
|
+
}
|
|
201
|
+
catch (_error) {
|
|
202
|
+
const errorMsg = _error instanceof Error ? _error.message : String(_error);
|
|
203
|
+
throw new Error(`Failed to get accurate token count from Anthropic API: ${errorMsg}`);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* OpenAI token counting using tiktoken
|
|
208
|
+
*/
|
|
209
|
+
countTokensOpenAI(text) {
|
|
210
|
+
if (!this.encoder) {
|
|
211
|
+
return this.countTokensApproximation(text);
|
|
212
|
+
}
|
|
213
|
+
try {
|
|
214
|
+
const tokens = this.encoder.encode(text);
|
|
215
|
+
return tokens.length;
|
|
216
|
+
}
|
|
217
|
+
catch (_error) {
|
|
218
|
+
// Fallback to approximation on error
|
|
219
|
+
return this.countTokensApproximation(text);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/**
|
|
223
|
+
* Anthropic (Claude) token counting using p50k_base approximation
|
|
224
|
+
* This is an approximation - for accurate counts,
|
|
225
|
+
* use countTokensAccurate() in dev mode
|
|
226
|
+
*
|
|
227
|
+
* Note: We add a fixed overhead of 7 tokens based on empirical
|
|
228
|
+
* testing, which improves accuracy from ~70% to ~95% for typical
|
|
229
|
+
* text lengths.
|
|
230
|
+
*/
|
|
231
|
+
countTokensAnthropic(text) {
|
|
232
|
+
if (!this.encoder) {
|
|
233
|
+
return this.countTokensApproximation(text);
|
|
234
|
+
}
|
|
235
|
+
try {
|
|
236
|
+
const tokens = this.encoder.encode(text);
|
|
237
|
+
// Add fixed overhead of 7 tokens to improve approximation
|
|
238
|
+
// accuracy
|
|
239
|
+
return tokens.length + 7;
|
|
240
|
+
}
|
|
241
|
+
catch (_error) {
|
|
242
|
+
// Fallback to approximation on error
|
|
243
|
+
return this.countTokensApproximation(text);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Google (Gemini) token counting
|
|
248
|
+
* Note: Gemini doesn't have an official tokenizer library yet,
|
|
249
|
+
* so we use an approximation based on Gemini's documentation:
|
|
250
|
+
* ~4 characters per token for English text
|
|
251
|
+
*/
|
|
252
|
+
countTokensGoogle(text) {
|
|
253
|
+
// Gemini approximation: ~4 chars per token
|
|
254
|
+
// More accurate than generic approximation due to Gemini-specific analysis
|
|
255
|
+
const charCount = text.length;
|
|
256
|
+
return Math.ceil(charCount / 4);
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* Generic approximation for unknown models
|
|
260
|
+
* Rule of thumb: ~4 characters per token for English text
|
|
261
|
+
* This is less accurate but works as a fallback
|
|
262
|
+
*/
|
|
263
|
+
countTokensApproximation(text) {
|
|
264
|
+
const charCount = text.length;
|
|
265
|
+
// Add some overhead for whitespace and punctuation
|
|
266
|
+
return Math.ceil(charCount / 3.5);
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Count tokens in message content (handles multimodal content)
|
|
270
|
+
*/
|
|
271
|
+
countMessageContent(message) {
|
|
272
|
+
if (typeof message.content === "string") {
|
|
273
|
+
return this.countTokens(message.content);
|
|
274
|
+
}
|
|
275
|
+
if (Array.isArray(message.content)) {
|
|
276
|
+
let total = 0;
|
|
277
|
+
for (const part of message.content) {
|
|
278
|
+
if ("text" in part && typeof part.text === "string") {
|
|
279
|
+
total += this.countTokens(part.text);
|
|
280
|
+
}
|
|
281
|
+
else if ("image_url" in part) {
|
|
282
|
+
// Image tokens are provider-specific
|
|
283
|
+
total += this.getImageTokens();
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
return total;
|
|
287
|
+
}
|
|
288
|
+
return 0;
|
|
289
|
+
}
|
|
290
|
+
/**
|
|
291
|
+
* Get per-message overhead tokens
|
|
292
|
+
* Based on OpenAI's documentation: each message has ~3-4 tokens of overhead
|
|
293
|
+
*/
|
|
294
|
+
getMessageOverhead(message) {
|
|
295
|
+
// Base overhead for role
|
|
296
|
+
let overhead = 4;
|
|
297
|
+
// Additional overhead for tool calls
|
|
298
|
+
if ("tool_calls" in message && message.tool_calls) {
|
|
299
|
+
// Each tool call adds overhead
|
|
300
|
+
overhead += message.tool_calls.length * 3;
|
|
301
|
+
// Count tokens in tool call arguments
|
|
302
|
+
for (const toolCall of message.tool_calls) {
|
|
303
|
+
const func = toolCall.function;
|
|
304
|
+
if (func.arguments) {
|
|
305
|
+
overhead += this.countTokens(func.arguments);
|
|
306
|
+
}
|
|
307
|
+
if (func.name) {
|
|
308
|
+
overhead += this.countTokens(func.name);
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
// Additional overhead for tool message
|
|
313
|
+
if ("tool_call_id" in message && message.tool_call_id) {
|
|
314
|
+
overhead += 2;
|
|
315
|
+
}
|
|
316
|
+
return overhead;
|
|
317
|
+
}
|
|
318
|
+
/**
|
|
319
|
+
* Get conversation-level overhead
|
|
320
|
+
* For OpenAI: 3 tokens per request
|
|
321
|
+
* For Claude: Similar overhead
|
|
322
|
+
*/
|
|
323
|
+
getConversationOverhead(messageCount) {
|
|
324
|
+
if (messageCount === 0) {
|
|
325
|
+
return 0;
|
|
326
|
+
}
|
|
327
|
+
switch (this.provider) {
|
|
328
|
+
case "openai":
|
|
329
|
+
return 3; // Per OpenAI docs
|
|
330
|
+
case "anthropic":
|
|
331
|
+
return 3; // Similar to OpenAI
|
|
332
|
+
case "google":
|
|
333
|
+
return 2; // Gemini has lower overhead
|
|
334
|
+
default:
|
|
335
|
+
return 3;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
/**
|
|
339
|
+
* Get estimated tokens for an image
|
|
340
|
+
* This is a rough estimate - actual tokens depend on image size
|
|
341
|
+
* and detail level
|
|
342
|
+
*
|
|
343
|
+
* Sources:
|
|
344
|
+
* - OpenAI: https://platform.openai.com/docs/guides/vision
|
|
345
|
+
* "low" detail: 85 tokens,
|
|
346
|
+
* "high" detail: 85 base + tiles (170-765+ tokens)
|
|
347
|
+
* - Anthropic: https://docs.anthropic.com/claude/docs/vision
|
|
348
|
+
* Standard images: ~1600 tokens
|
|
349
|
+
* - Google: No official token count documentation available
|
|
350
|
+
*/
|
|
351
|
+
getImageTokens() {
|
|
352
|
+
switch (this.provider) {
|
|
353
|
+
case "openai":
|
|
354
|
+
// Using mid-range estimate between low (85) and typical
|
|
355
|
+
// high detail (~255)
|
|
356
|
+
// Most images use "auto" which chooses based on content
|
|
357
|
+
return 170;
|
|
358
|
+
case "anthropic":
|
|
359
|
+
// Claude documentation: ~1600 tokens for standard images
|
|
360
|
+
return 1600;
|
|
361
|
+
case "google":
|
|
362
|
+
// Gemini: No official documentation, using conservative estimate
|
|
363
|
+
return 1000;
|
|
364
|
+
default:
|
|
365
|
+
return 1000;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
exports.TokenCounter = TokenCounter;
|
|
370
|
+
/**
|
|
371
|
+
* Create a token counter for a given model
|
|
372
|
+
*/
|
|
373
|
+
function createTokenCounter(model) {
|
|
374
|
+
return new TokenCounter(model);
|
|
375
|
+
}
|
|
376
|
+
/**
|
|
377
|
+
* Utility function to count tokens in text without creating a counter instance
|
|
378
|
+
*/
|
|
379
|
+
function countTokensQuick(text, model) {
|
|
380
|
+
const counter = createTokenCounter(model);
|
|
381
|
+
const count = counter.countTokens(text);
|
|
382
|
+
counter.free();
|
|
383
|
+
return count;
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* Get context window size for a model without creating a counter instance
|
|
387
|
+
*/
|
|
388
|
+
function getContextWindowSize(model) {
|
|
389
|
+
return MODEL_CONTEXT_WINDOWS[model] ?? DEFAULT_CONTEXT_WINDOW;
|
|
390
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const vitest_1 = require("vitest");
|
|
4
|
+
const tokenCounter_1 = require("./tokenCounter");
|
|
5
|
+
const fs_1 = require("fs");
|
|
6
|
+
const path_1 = require("path");
|
|
7
|
+
(0, vitest_1.describe)("ModelDetector", () => {
|
|
8
|
+
(0, vitest_1.describe)("detectProvider", () => {
|
|
9
|
+
vitest_1.it.each([
|
|
10
|
+
["gpt-4o", "openai"],
|
|
11
|
+
["gpt-4", "openai"],
|
|
12
|
+
["openai/gpt-4o", "openai"],
|
|
13
|
+
["o1-preview", "openai"],
|
|
14
|
+
["claude-3-7-sonnet-20250219", "anthropic"],
|
|
15
|
+
["anthropic/claude-sonnet-4.5", "anthropic"],
|
|
16
|
+
["gemini-2.0-flash-exp", "google"],
|
|
17
|
+
["google/gemini-pro", "google"],
|
|
18
|
+
["unknown-model", "unknown"],
|
|
19
|
+
["", "unknown"],
|
|
20
|
+
])("should detect %s as %s", (model, expected) => {
|
|
21
|
+
(0, vitest_1.expect)(tokenCounter_1.ModelDetector.detectProvider(model)).toBe(expected);
|
|
22
|
+
});
|
|
23
|
+
});
|
|
24
|
+
(0, vitest_1.describe)("normalizeModel", () => {
|
|
25
|
+
vitest_1.it.each([
|
|
26
|
+
["openai/gpt-4o", "openai", "gpt-4o"],
|
|
27
|
+
["anthropic/claude-3-opus", "anthropic", "claude-3-opus"],
|
|
28
|
+
["google/gemini-pro", "google", "gemini-pro"],
|
|
29
|
+
["gpt-4o-mini", "openai", "gpt-4o-mini"], // Preserves variants
|
|
30
|
+
["o1-preview", "openai", "o1-preview"],
|
|
31
|
+
])("should normalize %s to %s", (input, provider, expected) => {
|
|
32
|
+
(0, vitest_1.expect)(tokenCounter_1.ModelDetector.normalizeModel(input, provider)).toBe(expected);
|
|
33
|
+
});
|
|
34
|
+
});
|
|
35
|
+
});
|
|
36
|
+
(0, vitest_1.describe)("TokenCounter", () => {
|
|
37
|
+
let counter;
|
|
38
|
+
(0, vitest_1.afterEach)(() => {
|
|
39
|
+
if (counter) {
|
|
40
|
+
counter.free();
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
(0, vitest_1.describe)("Basic counting", () => {
|
|
44
|
+
(0, vitest_1.it)("should count tokens for OpenAI models", () => {
|
|
45
|
+
counter = new tokenCounter_1.TokenCounter("gpt-4o");
|
|
46
|
+
(0, vitest_1.expect)(counter.getProvider()).toBe("openai");
|
|
47
|
+
(0, vitest_1.expect)(counter.countTokens("Hello, world!")).toBeGreaterThan(0);
|
|
48
|
+
(0, vitest_1.expect)(counter.countTokens("")).toBe(0);
|
|
49
|
+
});
|
|
50
|
+
(0, vitest_1.it)("should count tokens for Anthropic models with +7 overhead", () => {
|
|
51
|
+
counter = new tokenCounter_1.TokenCounter("anthropic/claude-sonnet-4.5");
|
|
52
|
+
(0, vitest_1.expect)(counter.getProvider()).toBe("anthropic");
|
|
53
|
+
// With +7 overhead, "Hello, world!" should be ~11 tokens
|
|
54
|
+
(0, vitest_1.expect)(counter.countTokens("Hello, world!")).toBeLessThan(15);
|
|
55
|
+
});
|
|
56
|
+
(0, vitest_1.it)("should handle provider prefixes", () => {
|
|
57
|
+
counter = new tokenCounter_1.TokenCounter("openai/gpt-4o");
|
|
58
|
+
(0, vitest_1.expect)(counter.getProvider()).toBe("openai");
|
|
59
|
+
(0, vitest_1.expect)(counter.getModel()).toBe("openai/gpt-4o");
|
|
60
|
+
});
|
|
61
|
+
(0, vitest_1.it)("should fall back to approximation for unknown models", () => {
|
|
62
|
+
counter = new tokenCounter_1.TokenCounter("unknown-model");
|
|
63
|
+
(0, vitest_1.expect)(counter.getProvider()).toBe("unknown");
|
|
64
|
+
const text = "Hello, world!";
|
|
65
|
+
const expected = Math.ceil(text.length / 3.5);
|
|
66
|
+
(0, vitest_1.expect)(counter.countTokens(text)).toBe(expected);
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
(0, vitest_1.describe)("Message counting", () => {
|
|
70
|
+
(0, vitest_1.beforeEach)(() => {
|
|
71
|
+
counter = new tokenCounter_1.TokenCounter("gpt-4o");
|
|
72
|
+
});
|
|
73
|
+
(0, vitest_1.it)("should count simple message with overhead", () => {
|
|
74
|
+
if (!counter)
|
|
75
|
+
throw new Error("Counter not initialized");
|
|
76
|
+
const message = {
|
|
77
|
+
role: "user",
|
|
78
|
+
content: "Hello, how are you?",
|
|
79
|
+
};
|
|
80
|
+
(0, vitest_1.expect)(counter.countMessageTokens(message)).toBeGreaterThan(4);
|
|
81
|
+
});
|
|
82
|
+
(0, vitest_1.it)("should count message with tool calls", () => {
|
|
83
|
+
if (!counter)
|
|
84
|
+
throw new Error("Counter not initialized");
|
|
85
|
+
const message = {
|
|
86
|
+
role: "assistant",
|
|
87
|
+
content: "Let me help you.",
|
|
88
|
+
tool_calls: [
|
|
89
|
+
{
|
|
90
|
+
id: "call_123",
|
|
91
|
+
type: "function",
|
|
92
|
+
function: {
|
|
93
|
+
name: "get_file",
|
|
94
|
+
arguments: JSON.stringify({ name: "test.txt" }),
|
|
95
|
+
},
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
};
|
|
99
|
+
(0, vitest_1.expect)(counter.countMessageTokens(message)).toBeGreaterThan(10);
|
|
100
|
+
});
|
|
101
|
+
(0, vitest_1.it)("should count multimodal message with images", () => {
|
|
102
|
+
if (!counter)
|
|
103
|
+
throw new Error("Counter not initialized");
|
|
104
|
+
const message = {
|
|
105
|
+
role: "user",
|
|
106
|
+
content: [
|
|
107
|
+
{ type: "text", text: "What's in this image?" },
|
|
108
|
+
{
|
|
109
|
+
type: "image_url",
|
|
110
|
+
image_url: { url: "data:image/png;base64,..." },
|
|
111
|
+
},
|
|
112
|
+
],
|
|
113
|
+
};
|
|
114
|
+
// OpenAI images: ~170 tokens + text + overhead
|
|
115
|
+
(0, vitest_1.expect)(counter.countMessageTokens(message)).toBeGreaterThan(170);
|
|
116
|
+
});
|
|
117
|
+
(0, vitest_1.it)("should count conversation with overhead", () => {
|
|
118
|
+
if (!counter)
|
|
119
|
+
throw new Error("Counter not initialized");
|
|
120
|
+
const messages = [
|
|
121
|
+
{ role: "system", content: "You are a helpful assistant." },
|
|
122
|
+
{ role: "user", content: "Hello!" },
|
|
123
|
+
{ role: "assistant", content: "Hi there! How can I help?" },
|
|
124
|
+
];
|
|
125
|
+
(0, vitest_1.expect)(counter.countMessagesTokens(messages)).toBeGreaterThan(20);
|
|
126
|
+
(0, vitest_1.expect)(counter.countMessagesTokens([])).toBe(0);
|
|
127
|
+
});
|
|
128
|
+
});
|
|
129
|
+
(0, vitest_1.describe)("Factory functions", () => {
|
|
130
|
+
(0, vitest_1.it)("should create counter with factory", () => {
|
|
131
|
+
const testCounter = (0, tokenCounter_1.createTokenCounter)("gpt-4o");
|
|
132
|
+
(0, vitest_1.expect)(testCounter.getProvider()).toBe("openai");
|
|
133
|
+
testCounter.free();
|
|
134
|
+
});
|
|
135
|
+
(0, vitest_1.it)("should count tokens quickly", () => {
|
|
136
|
+
const count = (0, tokenCounter_1.countTokensQuick)("Hello, world!", "gpt-4o");
|
|
137
|
+
(0, vitest_1.expect)(count).toBeGreaterThan(0);
|
|
138
|
+
});
|
|
139
|
+
});
|
|
140
|
+
(0, vitest_1.describe)("Edge cases", () => {
|
|
141
|
+
(0, vitest_1.beforeEach)(() => {
|
|
142
|
+
counter = new tokenCounter_1.TokenCounter("gpt-4o");
|
|
143
|
+
});
|
|
144
|
+
(0, vitest_1.it)("should handle long text, special chars, and code", () => {
|
|
145
|
+
if (!counter)
|
|
146
|
+
throw new Error("Counter not initialized");
|
|
147
|
+
(0, vitest_1.expect)(counter.countTokens("Hello ".repeat(1000))).toBeGreaterThan(1000);
|
|
148
|
+
(0, vitest_1.expect)(counter.countTokens("Hello! 你好 🌍")).toBeGreaterThan(0);
|
|
149
|
+
(0, vitest_1.expect)(counter.countTokens('function hello() { return "world"; }')).toBeGreaterThan(5);
|
|
150
|
+
});
|
|
151
|
+
});
|
|
152
|
+
});
|
|
153
|
+
(0, vitest_1.describe)("Dev mode accurate counting", () => {
|
|
154
|
+
(0, vitest_1.it)("should throw error without dev mode", async () => {
|
|
155
|
+
const counter = new tokenCounter_1.TokenCounter("anthropic/claude-sonnet-4.5");
|
|
156
|
+
await (0, vitest_1.expect)(counter.countTokensAccurate("test")).rejects.toThrow("Accurate token counting requires enableDevMode");
|
|
157
|
+
counter.free();
|
|
158
|
+
});
|
|
159
|
+
(0, vitest_1.it)("should compare p50k_base approximation vs Anthropic API", async () => {
|
|
160
|
+
const counter = new tokenCounter_1.TokenCounter("claude-3-7-sonnet-20250219", {
|
|
161
|
+
enableDevMode: true,
|
|
162
|
+
});
|
|
163
|
+
// Test cases covering different scales
|
|
164
|
+
const readmePath = (0, path_1.join)(__dirname, "..", "..", "README.md");
|
|
165
|
+
const harryPotterPath = (0, path_1.join)(__dirname, "test_data", "harrypotter.txt");
|
|
166
|
+
const testCases = [
|
|
167
|
+
{ text: "Hello, world!", label: "short" },
|
|
168
|
+
{ text: "The quick brown fox jumps over the lazy dog.", label: "medium" },
|
|
169
|
+
{ text: (0, fs_1.readFileSync)(readmePath, "utf-8"), label: "README.md" },
|
|
170
|
+
{
|
|
171
|
+
text: (0, fs_1.readFileSync)(harryPotterPath, "utf-8"),
|
|
172
|
+
label: "harrypotter.txt",
|
|
173
|
+
},
|
|
174
|
+
];
|
|
175
|
+
if (process.env.ANTHROPIC_API_KEY) {
|
|
176
|
+
console.log("\n📊 Token Count Comparison (p50k_base+7 vs Anthropic API)\n");
|
|
177
|
+
for (const { text, label } of testCases) {
|
|
178
|
+
const accurate = await counter.countTokensAccurate(text);
|
|
179
|
+
const approximate = counter.countTokens(text);
|
|
180
|
+
const diff = accurate - approximate;
|
|
181
|
+
const accuracy = ((approximate / accurate) * 100).toFixed(1);
|
|
182
|
+
const size = text.length > 1000
|
|
183
|
+
? `${String(text.length)} chars`
|
|
184
|
+
: `${String(text.length)} chars`;
|
|
185
|
+
console.log(`${label.padEnd(20)} [${size}]`);
|
|
186
|
+
console.log(` Accurate: ${accurate.toLocaleString().padStart(8)} tokens`);
|
|
187
|
+
console.log(` Approximate: ${approximate.toLocaleString().padStart(8)} tokens`);
|
|
188
|
+
const diffStr = (diff > 0 ? "+" : "") + String(diff);
|
|
189
|
+
console.log(` Difference: ${diffStr.padStart(8)} (${accuracy}%)\n`);
|
|
190
|
+
(0, vitest_1.expect)(accurate).toBeGreaterThan(0);
|
|
191
|
+
(0, vitest_1.expect)(approximate).toBeGreaterThan(0);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
else {
|
|
195
|
+
await (0, vitest_1.expect)(counter.countTokensAccurate("test")).rejects.toThrow();
|
|
196
|
+
}
|
|
197
|
+
counter.free();
|
|
198
|
+
});
|
|
199
|
+
(0, vitest_1.it)("should return regular count for non-Anthropic models", async () => {
|
|
200
|
+
const counter = new tokenCounter_1.TokenCounter("gpt-4o", { enableDevMode: true });
|
|
201
|
+
const regular = counter.countTokens("Hello, world!");
|
|
202
|
+
const accurate = await counter.countTokensAccurate("Hello, world!");
|
|
203
|
+
(0, vitest_1.expect)(accurate).toBe(regular);
|
|
204
|
+
counter.free();
|
|
205
|
+
});
|
|
206
|
+
});
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Centralized settings for agent tools.
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.OPEN_URL_MAX_LENGTH = exports.MAX_TOOL_CALL_RESPONSE_LENGTH = void 0;
|
|
7
|
+
const env = (key, defaultValue) => process.env[key] || defaultValue;
|
|
8
|
+
/**
|
|
9
|
+
* Maximum length for tool call responses before truncation.
|
|
10
|
+
* Applied as a final sanity check on all tool call results.
|
|
11
|
+
*/
|
|
12
|
+
exports.MAX_TOOL_CALL_RESPONSE_LENGTH = parseInt(env("MAX_TOOL_CALL_RESPONSE_LENGTH", "50000"), 10);
|
|
13
|
+
/**
|
|
14
|
+
* Maximum length for open_url content extraction.
|
|
15
|
+
* Controls how much text is extracted from HTML pages.
|
|
16
|
+
*/
|
|
17
|
+
exports.OPEN_URL_MAX_LENGTH = parseInt(env("OPEN_URL_MAX_LENGTH", "50000"), 10);
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.calculatorTool = void 0;
|
|
4
|
+
exports.calculatorEval = calculatorEval;
|
|
5
|
+
const expr_eval_1 = require("expr-eval");
|
|
6
|
+
const utils_1 = require("./utils");
|
|
7
|
+
const ARITHMETIC_DESC = {
|
|
8
|
+
type: "function",
|
|
9
|
+
function: {
|
|
10
|
+
name: "arithmetic",
|
|
11
|
+
description: "Evaluate arithmetic expression",
|
|
12
|
+
parameters: {
|
|
13
|
+
type: "object",
|
|
14
|
+
properties: {
|
|
15
|
+
expr: {
|
|
16
|
+
type: "string",
|
|
17
|
+
description: "Expression containing +-*/()",
|
|
18
|
+
},
|
|
19
|
+
},
|
|
20
|
+
required: ["expr"],
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
};
|
|
24
|
+
function calculatorEval(args) {
|
|
25
|
+
try {
|
|
26
|
+
return String(expr_eval_1.Parser.evaluate(args));
|
|
27
|
+
}
|
|
28
|
+
catch (e) {
|
|
29
|
+
if (typeof e.message === "string") {
|
|
30
|
+
return e.message;
|
|
31
|
+
}
|
|
32
|
+
return String(e);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
exports.calculatorTool = {
|
|
36
|
+
// eslint-disable-next-line @typescript-eslint/require-await
|
|
37
|
+
setup: async (agent) => {
|
|
38
|
+
const getExpr = (0, utils_1.makeParseArgsFn)(["expr"]);
|
|
39
|
+
const toolFn = async (_, args) => {
|
|
40
|
+
const { expr } = getExpr(args);
|
|
41
|
+
return Promise.resolve({ response: calculatorEval(expr) });
|
|
42
|
+
};
|
|
43
|
+
agent.addAgentTool(ARITHMETIC_DESC, toolFn);
|
|
44
|
+
},
|
|
45
|
+
};
|