@xalia/agent 0.6.9 → 0.6.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. package/.env.development +6 -1
  2. package/.env.test +7 -0
  3. package/README.md +11 -0
  4. package/context_system.md +498 -0
  5. package/dist/agent/src/agent/agent.js +77 -18
  6. package/dist/agent/src/agent/agentUtils.js +3 -2
  7. package/dist/agent/src/agent/documentSummarizer.js +126 -0
  8. package/dist/agent/src/agent/dummyLLM.js +25 -22
  9. package/dist/agent/src/agent/imageGenLLM.js +22 -19
  10. package/dist/agent/src/agent/llm.js +1 -1
  11. package/dist/agent/src/agent/openAILLM.js +15 -12
  12. package/dist/agent/src/agent/openAILLMStreaming.js +68 -37
  13. package/dist/agent/src/agent/repeatLLM.js +16 -7
  14. package/dist/agent/src/agent/tokenCounter.js +390 -0
  15. package/dist/agent/src/agent/tokenCounter.test.js +206 -0
  16. package/dist/agent/src/agent/toolSettings.js +17 -0
  17. package/dist/agent/src/agent/tools/calculatorTool.js +45 -0
  18. package/dist/agent/src/agent/tools/contentExtractors/pdfToText.js +55 -0
  19. package/dist/agent/src/agent/tools/datetimeTool.js +38 -0
  20. package/dist/agent/src/agent/tools/fileManager/fileManagerTool.js +156 -0
  21. package/dist/agent/src/agent/tools/fileManager/index.js +31 -0
  22. package/dist/agent/src/agent/tools/fileManager/memoryFileManager.js +102 -0
  23. package/dist/agent/src/{chat/data → agent/tools/fileManager}/mimeTypes.js +3 -1
  24. package/dist/agent/src/agent/tools/fileManager/prompt.js +33 -0
  25. package/dist/agent/src/{chat/data/dbSessionFileModels.js → agent/tools/fileManager/types.js} +7 -0
  26. package/dist/agent/src/agent/tools/index.js +64 -0
  27. package/dist/agent/src/agent/tools/openUrlTool.js +57 -0
  28. package/dist/agent/src/agent/tools/renderTool.js +89 -0
  29. package/dist/agent/src/agent/tools/utils.js +61 -0
  30. package/dist/agent/src/{chat/utils/search.js → agent/tools/webSearch.js} +1 -2
  31. package/dist/agent/src/agent/tools/webSearchTool.js +40 -0
  32. package/dist/agent/src/chat/client/chatClient.js +28 -0
  33. package/dist/agent/src/chat/client/index.js +4 -1
  34. package/dist/agent/src/chat/client/sessionClient.js +28 -2
  35. package/dist/agent/src/chat/constants.js +8 -0
  36. package/dist/agent/src/chat/data/dbSessionFiles.js +11 -6
  37. package/dist/agent/src/chat/protocol/messages.js +5 -0
  38. package/dist/agent/src/chat/server/chatContextManager.js +45 -25
  39. package/dist/agent/src/chat/server/conversation.js +3 -0
  40. package/dist/agent/src/chat/server/imageGeneratorTools.js +20 -8
  41. package/dist/agent/src/chat/server/openAIRouterLLM.js +0 -3
  42. package/dist/agent/src/chat/server/openSession.js +218 -55
  43. package/dist/agent/src/chat/server/promptRefiner.js +86 -0
  44. package/dist/agent/src/chat/server/server.js +5 -1
  45. package/dist/agent/src/chat/server/sessionFileManager.js +22 -221
  46. package/dist/agent/src/chat/server/sessionRegistry.js +87 -0
  47. package/dist/agent/src/chat/server/titleGenerator.js +112 -0
  48. package/dist/agent/src/chat/server/titleGenerator.test.js +113 -0
  49. package/dist/agent/src/chat/server/tools.js +63 -287
  50. package/dist/agent/src/chat/utils/approvalManager.js +6 -3
  51. package/dist/agent/src/chat/utils/multiAsyncQueue.js +3 -0
  52. package/dist/agent/src/test/agent.test.js +16 -17
  53. package/dist/agent/src/test/chatContextManager.test.js +15 -3
  54. package/dist/agent/src/test/dbMcpServerConfigs.test.js +4 -4
  55. package/dist/agent/src/test/dbSessionFiles.test.js +17 -17
  56. package/dist/agent/src/test/testTools.js +6 -1
  57. package/dist/agent/src/test/tools.test.js +27 -9
  58. package/dist/agent/src/tool/agentChat.js +5 -2
  59. package/dist/agent/src/tool/chatMain.js +34 -7
  60. package/dist/agent/src/tool/commandPrompt.js +2 -2
  61. package/dist/agent/src/tool/files.js +7 -8
  62. package/package.json +4 -1
  63. package/scripts/test_chat +195 -176
  64. package/src/agent/agent.ts +98 -23
  65. package/src/agent/agentUtils.ts +3 -2
  66. package/src/agent/documentSummarizer.ts +157 -0
  67. package/src/agent/dummyLLM.ts +27 -23
  68. package/src/agent/imageGenLLM.ts +28 -24
  69. package/src/agent/llm.ts +2 -2
  70. package/src/agent/openAILLM.ts +17 -13
  71. package/src/agent/openAILLMStreaming.ts +80 -41
  72. package/src/agent/repeatLLM.ts +19 -7
  73. package/src/agent/test_data/harrypotter.txt +6065 -0
  74. package/src/agent/tokenCounter.test.ts +243 -0
  75. package/src/agent/tokenCounter.ts +483 -0
  76. package/src/agent/toolSettings.ts +24 -0
  77. package/src/agent/tools/calculatorTool.ts +50 -0
  78. package/src/agent/tools/contentExtractors/pdfToText.ts +60 -0
  79. package/src/agent/tools/datetimeTool.ts +41 -0
  80. package/src/agent/tools/fileManager/fileManagerTool.ts +199 -0
  81. package/src/agent/tools/fileManager/index.ts +50 -0
  82. package/src/agent/tools/fileManager/memoryFileManager.ts +120 -0
  83. package/src/{chat/data → agent/tools/fileManager}/mimeTypes.ts +3 -1
  84. package/src/agent/tools/fileManager/prompt.ts +38 -0
  85. package/src/{chat/data/dbSessionFileModels.ts → agent/tools/fileManager/types.ts} +76 -0
  86. package/src/agent/tools/index.ts +49 -0
  87. package/src/agent/tools/openUrlTool.ts +62 -0
  88. package/src/agent/tools/renderTool.ts +92 -0
  89. package/src/agent/tools/utils.ts +74 -0
  90. package/src/{chat/utils/search.ts → agent/tools/webSearch.ts} +0 -1
  91. package/src/agent/tools/webSearchTool.ts +44 -0
  92. package/src/chat/client/chatClient.ts +45 -0
  93. package/src/chat/client/index.ts +3 -0
  94. package/src/chat/client/sessionClient.ts +40 -3
  95. package/src/chat/client/sessionFiles.ts +1 -1
  96. package/src/chat/constants.ts +6 -0
  97. package/src/chat/data/dataModels.ts +6 -0
  98. package/src/chat/data/dbSessionFiles.ts +12 -4
  99. package/src/chat/protocol/messages.ts +60 -7
  100. package/src/chat/server/chatContextManager.ts +58 -37
  101. package/src/chat/server/conversation.ts +3 -0
  102. package/src/chat/server/imageGeneratorTools.ts +31 -12
  103. package/src/chat/server/openAIRouterLLM.ts +1 -4
  104. package/src/chat/server/openSession.ts +323 -67
  105. package/src/chat/server/promptRefiner.ts +106 -0
  106. package/src/chat/server/server.ts +4 -1
  107. package/src/chat/server/sessionFileManager.ts +35 -306
  108. package/src/chat/server/sessionRegistry.ts +128 -0
  109. package/src/chat/server/titleGenerator.test.ts +103 -0
  110. package/src/chat/server/titleGenerator.ts +143 -0
  111. package/src/chat/server/tools.ts +77 -304
  112. package/src/chat/utils/approvalManager.ts +9 -3
  113. package/src/chat/utils/multiAsyncQueue.ts +4 -0
  114. package/src/test/agent.test.ts +17 -23
  115. package/src/test/chatContextManager.test.ts +29 -4
  116. package/src/test/dbMcpServerConfigs.test.ts +4 -4
  117. package/src/test/dbSessionFiles.test.ts +16 -16
  118. package/src/test/testTools.ts +8 -3
  119. package/src/test/tools.test.ts +30 -5
  120. package/src/tool/agentChat.ts +12 -3
  121. package/src/tool/chatMain.ts +33 -6
  122. package/src/tool/commandPrompt.ts +2 -2
  123. package/src/tool/files.ts +1 -3
  124. package/dist/agent/src/agent/tools.js +0 -44
  125. package/src/agent/tools.ts +0 -57
  126. /package/dist/agent/src/{chat/utils → agent/tools/contentExtractors}/htmlToText.js +0 -0
  127. /package/src/{chat/utils → agent/tools/contentExtractors}/htmlToText.ts +0 -0
@@ -0,0 +1,390 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.TokenCounter = exports.ModelDetector = void 0;
7
+ exports.createTokenCounter = createTokenCounter;
8
+ exports.countTokensQuick = countTokensQuick;
9
+ exports.getContextWindowSize = getContextWindowSize;
10
+ const tiktoken_1 = require("tiktoken");
11
+ const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
12
+ /**
13
+ * Model context window sizes (from OpenRouter API)
14
+ * Only includes models currently used in DEFAULT_MODEL_MAP
15
+ */
16
+ const MODEL_CONTEXT_WINDOWS = {
17
+ // OpenAI models
18
+ "gpt-4o-mini": 128000,
19
+ "gpt-4o": 128000,
20
+ "openai/gpt-4o-mini": 128000,
21
+ "openai/gpt-4o": 128000,
22
+ // Google models
23
+ "google/gemini-2.5-flash": 1048576,
24
+ "google/gemini-2.5-pro": 1048576,
25
+ "google/gemini-2.5-flash-image-preview": 32768,
26
+ // Anthropic models
27
+ "anthropic/claude-3.7-sonnet": 200000,
28
+ "anthropic/claude-sonnet-4": 1000000,
29
+ "anthropic/claude-sonnet-4.5": 1000000,
30
+ "claude-3-7-sonnet-20250219": 200000,
31
+ };
32
+ const DEFAULT_CONTEXT_WINDOW = 128000;
33
+ /**
34
+ * Model provider detection and normalization
35
+ */
36
+ exports.ModelDetector = {
37
+ /**
38
+ * Detect the provider from a model string
39
+ * Supports formats like:
40
+ * - "gpt-4o" -> openai
41
+ * - "openai/gpt-4o" -> openai
42
+ * - "anthropic/claude-sonnet-4.5" -> anthropic
43
+ * - "claude-3-7-sonnet-20250219" -> anthropic
44
+ * - "gemini-2.0-flash-exp" -> google
45
+ * - "google/gemini-2.0-flash-exp" -> google
46
+ */
47
+ detectProvider(model) {
48
+ const lowerModel = model.toLowerCase();
49
+ // Check for provider prefix (e.g., "anthropic/", "openai/")
50
+ if (lowerModel.startsWith("openai/")) {
51
+ return "openai";
52
+ }
53
+ if (lowerModel.startsWith("anthropic/")) {
54
+ return "anthropic";
55
+ }
56
+ if (lowerModel.startsWith("google/")) {
57
+ return "google";
58
+ }
59
+ // Check for model name patterns
60
+ if (lowerModel.includes("gpt")) {
61
+ return "openai";
62
+ }
63
+ if (lowerModel.includes("claude")) {
64
+ return "anthropic";
65
+ }
66
+ if (lowerModel.includes("gemini")) {
67
+ return "google";
68
+ }
69
+ if (lowerModel.includes("o1") || lowerModel.includes("o3")) {
70
+ return "openai";
71
+ }
72
+ return "unknown";
73
+ },
74
+ /**
75
+ * Normalize model string for tokenizer
76
+ * Primary purpose: Remove provider prefixes
77
+ *
78
+ * Note: tiktoken supports most OpenAI model variants natively
79
+ * (gpt-4o-mini, gpt-4o-2024-11-20, o1-preview, etc.),
80
+ * so we don't need to map them.
81
+ */
82
+ normalizeModel(model, _provider) {
83
+ // Remove provider prefix (e.g., "anthropic/" -> "")
84
+ // This is the main normalization needed for OpenRouter-style
85
+ // model strings
86
+ const normalized = model.replace(/^[^/]+\//, "");
87
+ // For OpenAI: tiktoken handles variants natively,
88
+ // just return normalized
89
+ // For Anthropic/Google: return normalized for their
90
+ // respective tokenizers
91
+ return normalized;
92
+ },
93
+ };
94
+ /**
95
+ * Multi-model token counter supporting OpenAI, Anthropic (Claude),
96
+ * and Google (Gemini)
97
+ */
98
+ class TokenCounter {
99
+ constructor(model, options) {
100
+ this.encoder = null;
101
+ this.anthropicClient = null;
102
+ this.model = model;
103
+ this.provider = exports.ModelDetector.detectProvider(model);
104
+ this.normalizedModel = exports.ModelDetector.normalizeModel(model, this.provider);
105
+ // Initialize encoder based on provider
106
+ if (this.provider === "openai") {
107
+ try {
108
+ this.encoder = (0, tiktoken_1.encoding_for_model)(this.normalizedModel);
109
+ }
110
+ catch (_error) {
111
+ // Fallback to cl100k_base for unknown OpenAI models
112
+ this.encoder = (0, tiktoken_1.encoding_for_model)("gpt-4");
113
+ }
114
+ }
115
+ else if (this.provider === "anthropic") {
116
+ // Use p50k_base encoding for Claude (approximation)
117
+ this.encoder = (0, tiktoken_1.get_encoding)("p50k_base");
118
+ }
119
+ // Initialize Anthropic client for dev mode sanity checks
120
+ if (options?.enableDevMode && this.provider === "anthropic") {
121
+ const apiKey = process.env.ANTHROPIC_API_KEY;
122
+ if (apiKey) {
123
+ this.anthropicClient = new sdk_1.default({ apiKey });
124
+ }
125
+ }
126
+ }
127
+ getModel() {
128
+ return this.model;
129
+ }
130
+ getProvider() {
131
+ return this.provider;
132
+ }
133
+ getContextWindow() {
134
+ return MODEL_CONTEXT_WINDOWS[this.model] ?? DEFAULT_CONTEXT_WINDOW;
135
+ }
136
+ countTokens(text) {
137
+ if (!text || text.length === 0) {
138
+ return 0;
139
+ }
140
+ switch (this.provider) {
141
+ case "openai":
142
+ return this.countTokensOpenAI(text);
143
+ case "anthropic":
144
+ return this.countTokensAnthropic(text);
145
+ case "google":
146
+ return this.countTokensGoogle(text);
147
+ default:
148
+ return this.countTokensApproximation(text);
149
+ }
150
+ }
151
+ countMessageTokens(message) {
152
+ const contentTokens = this.countMessageContent(message);
153
+ const overhead = this.getMessageOverhead(message);
154
+ return contentTokens + overhead;
155
+ }
156
+ countMessagesTokens(messages) {
157
+ if (messages.length === 0) {
158
+ return 0;
159
+ }
160
+ let total = 0;
161
+ // Count each message
162
+ for (const message of messages) {
163
+ total += this.countMessageTokens(message);
164
+ }
165
+ // Add conversation-level overhead
166
+ total += this.getConversationOverhead(messages.length);
167
+ return total;
168
+ }
169
+ /**
170
+ * Free resources (for encoders)
171
+ */
172
+ free() {
173
+ if (this.encoder) {
174
+ this.encoder.free();
175
+ this.encoder = null;
176
+ }
177
+ }
178
+ /**
179
+ * DEV ONLY: Get accurate token count from Anthropic API
180
+ * for sanity checking
181
+ * This requires enableDevMode: true in constructor and
182
+ * ANTHROPIC_API_KEY environment variable
183
+ * Use this to compare against p50k_base approximation
184
+ */
185
+ async countTokensAccurate(text) {
186
+ if (this.provider !== "anthropic") {
187
+ // For non-Anthropic models, just use regular counting
188
+ return this.countTokens(text);
189
+ }
190
+ if (!this.anthropicClient) {
191
+ throw new Error("Accurate token counting requires enableDevMode: true " +
192
+ "and ANTHROPIC_API_KEY environment variable");
193
+ }
194
+ try {
195
+ const result = await this.anthropicClient.messages.countTokens({
196
+ model: this.normalizedModel,
197
+ messages: [{ role: "user", content: text }],
198
+ });
199
+ return result.input_tokens;
200
+ }
201
+ catch (_error) {
202
+ const errorMsg = _error instanceof Error ? _error.message : String(_error);
203
+ throw new Error(`Failed to get accurate token count from Anthropic API: ${errorMsg}`);
204
+ }
205
+ }
206
+ /**
207
+ * OpenAI token counting using tiktoken
208
+ */
209
+ countTokensOpenAI(text) {
210
+ if (!this.encoder) {
211
+ return this.countTokensApproximation(text);
212
+ }
213
+ try {
214
+ const tokens = this.encoder.encode(text);
215
+ return tokens.length;
216
+ }
217
+ catch (_error) {
218
+ // Fallback to approximation on error
219
+ return this.countTokensApproximation(text);
220
+ }
221
+ }
222
+ /**
223
+ * Anthropic (Claude) token counting using p50k_base approximation
224
+ * This is an approximation - for accurate counts,
225
+ * use countTokensAccurate() in dev mode
226
+ *
227
+ * Note: We add a fixed overhead of 7 tokens based on empirical
228
+ * testing, which improves accuracy from ~70% to ~95% for typical
229
+ * text lengths.
230
+ */
231
+ countTokensAnthropic(text) {
232
+ if (!this.encoder) {
233
+ return this.countTokensApproximation(text);
234
+ }
235
+ try {
236
+ const tokens = this.encoder.encode(text);
237
+ // Add fixed overhead of 7 tokens to improve approximation
238
+ // accuracy
239
+ return tokens.length + 7;
240
+ }
241
+ catch (_error) {
242
+ // Fallback to approximation on error
243
+ return this.countTokensApproximation(text);
244
+ }
245
+ }
246
+ /**
247
+ * Google (Gemini) token counting
248
+ * Note: Gemini doesn't have an official tokenizer library yet,
249
+ * so we use an approximation based on Gemini's documentation:
250
+ * ~4 characters per token for English text
251
+ */
252
+ countTokensGoogle(text) {
253
+ // Gemini approximation: ~4 chars per token
254
+ // More accurate than generic approximation due to Gemini-specific analysis
255
+ const charCount = text.length;
256
+ return Math.ceil(charCount / 4);
257
+ }
258
+ /**
259
+ * Generic approximation for unknown models
260
+ * Rule of thumb: ~4 characters per token for English text
261
+ * This is less accurate but works as a fallback
262
+ */
263
+ countTokensApproximation(text) {
264
+ const charCount = text.length;
265
+ // Add some overhead for whitespace and punctuation
266
+ return Math.ceil(charCount / 3.5);
267
+ }
268
+ /**
269
+ * Count tokens in message content (handles multimodal content)
270
+ */
271
+ countMessageContent(message) {
272
+ if (typeof message.content === "string") {
273
+ return this.countTokens(message.content);
274
+ }
275
+ if (Array.isArray(message.content)) {
276
+ let total = 0;
277
+ for (const part of message.content) {
278
+ if ("text" in part && typeof part.text === "string") {
279
+ total += this.countTokens(part.text);
280
+ }
281
+ else if ("image_url" in part) {
282
+ // Image tokens are provider-specific
283
+ total += this.getImageTokens();
284
+ }
285
+ }
286
+ return total;
287
+ }
288
+ return 0;
289
+ }
290
+ /**
291
+ * Get per-message overhead tokens
292
+ * Based on OpenAI's documentation: each message has ~3-4 tokens of overhead
293
+ */
294
+ getMessageOverhead(message) {
295
+ // Base overhead for role
296
+ let overhead = 4;
297
+ // Additional overhead for tool calls
298
+ if ("tool_calls" in message && message.tool_calls) {
299
+ // Each tool call adds overhead
300
+ overhead += message.tool_calls.length * 3;
301
+ // Count tokens in tool call arguments
302
+ for (const toolCall of message.tool_calls) {
303
+ const func = toolCall.function;
304
+ if (func.arguments) {
305
+ overhead += this.countTokens(func.arguments);
306
+ }
307
+ if (func.name) {
308
+ overhead += this.countTokens(func.name);
309
+ }
310
+ }
311
+ }
312
+ // Additional overhead for tool message
313
+ if ("tool_call_id" in message && message.tool_call_id) {
314
+ overhead += 2;
315
+ }
316
+ return overhead;
317
+ }
318
+ /**
319
+ * Get conversation-level overhead
320
+ * For OpenAI: 3 tokens per request
321
+ * For Claude: Similar overhead
322
+ */
323
+ getConversationOverhead(messageCount) {
324
+ if (messageCount === 0) {
325
+ return 0;
326
+ }
327
+ switch (this.provider) {
328
+ case "openai":
329
+ return 3; // Per OpenAI docs
330
+ case "anthropic":
331
+ return 3; // Similar to OpenAI
332
+ case "google":
333
+ return 2; // Gemini has lower overhead
334
+ default:
335
+ return 3;
336
+ }
337
+ }
338
+ /**
339
+ * Get estimated tokens for an image
340
+ * This is a rough estimate - actual tokens depend on image size
341
+ * and detail level
342
+ *
343
+ * Sources:
344
+ * - OpenAI: https://platform.openai.com/docs/guides/vision
345
+ * "low" detail: 85 tokens,
346
+ * "high" detail: 85 base + tiles (170-765+ tokens)
347
+ * - Anthropic: https://docs.anthropic.com/claude/docs/vision
348
+ * Standard images: ~1600 tokens
349
+ * - Google: No official token count documentation available
350
+ */
351
+ getImageTokens() {
352
+ switch (this.provider) {
353
+ case "openai":
354
+ // Using mid-range estimate between low (85) and typical
355
+ // high detail (~255)
356
+ // Most images use "auto" which chooses based on content
357
+ return 170;
358
+ case "anthropic":
359
+ // Claude documentation: ~1600 tokens for standard images
360
+ return 1600;
361
+ case "google":
362
+ // Gemini: No official documentation, using conservative estimate
363
+ return 1000;
364
+ default:
365
+ return 1000;
366
+ }
367
+ }
368
+ }
369
+ exports.TokenCounter = TokenCounter;
370
+ /**
371
+ * Create a token counter for a given model
372
+ */
373
+ function createTokenCounter(model) {
374
+ return new TokenCounter(model);
375
+ }
376
+ /**
377
+ * Utility function to count tokens in text without creating a counter instance
378
+ */
379
+ function countTokensQuick(text, model) {
380
+ const counter = createTokenCounter(model);
381
+ const count = counter.countTokens(text);
382
+ counter.free();
383
+ return count;
384
+ }
385
+ /**
386
+ * Get context window size for a model without creating a counter instance
387
+ */
388
+ function getContextWindowSize(model) {
389
+ return MODEL_CONTEXT_WINDOWS[model] ?? DEFAULT_CONTEXT_WINDOW;
390
+ }
@@ -0,0 +1,206 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ const vitest_1 = require("vitest");
4
+ const tokenCounter_1 = require("./tokenCounter");
5
+ const fs_1 = require("fs");
6
+ const path_1 = require("path");
7
+ (0, vitest_1.describe)("ModelDetector", () => {
8
+ (0, vitest_1.describe)("detectProvider", () => {
9
+ vitest_1.it.each([
10
+ ["gpt-4o", "openai"],
11
+ ["gpt-4", "openai"],
12
+ ["openai/gpt-4o", "openai"],
13
+ ["o1-preview", "openai"],
14
+ ["claude-3-7-sonnet-20250219", "anthropic"],
15
+ ["anthropic/claude-sonnet-4.5", "anthropic"],
16
+ ["gemini-2.0-flash-exp", "google"],
17
+ ["google/gemini-pro", "google"],
18
+ ["unknown-model", "unknown"],
19
+ ["", "unknown"],
20
+ ])("should detect %s as %s", (model, expected) => {
21
+ (0, vitest_1.expect)(tokenCounter_1.ModelDetector.detectProvider(model)).toBe(expected);
22
+ });
23
+ });
24
+ (0, vitest_1.describe)("normalizeModel", () => {
25
+ vitest_1.it.each([
26
+ ["openai/gpt-4o", "openai", "gpt-4o"],
27
+ ["anthropic/claude-3-opus", "anthropic", "claude-3-opus"],
28
+ ["google/gemini-pro", "google", "gemini-pro"],
29
+ ["gpt-4o-mini", "openai", "gpt-4o-mini"], // Preserves variants
30
+ ["o1-preview", "openai", "o1-preview"],
31
+ ])("should normalize %s to %s", (input, provider, expected) => {
32
+ (0, vitest_1.expect)(tokenCounter_1.ModelDetector.normalizeModel(input, provider)).toBe(expected);
33
+ });
34
+ });
35
+ });
36
+ (0, vitest_1.describe)("TokenCounter", () => {
37
+ let counter;
38
+ (0, vitest_1.afterEach)(() => {
39
+ if (counter) {
40
+ counter.free();
41
+ }
42
+ });
43
+ (0, vitest_1.describe)("Basic counting", () => {
44
+ (0, vitest_1.it)("should count tokens for OpenAI models", () => {
45
+ counter = new tokenCounter_1.TokenCounter("gpt-4o");
46
+ (0, vitest_1.expect)(counter.getProvider()).toBe("openai");
47
+ (0, vitest_1.expect)(counter.countTokens("Hello, world!")).toBeGreaterThan(0);
48
+ (0, vitest_1.expect)(counter.countTokens("")).toBe(0);
49
+ });
50
+ (0, vitest_1.it)("should count tokens for Anthropic models with +7 overhead", () => {
51
+ counter = new tokenCounter_1.TokenCounter("anthropic/claude-sonnet-4.5");
52
+ (0, vitest_1.expect)(counter.getProvider()).toBe("anthropic");
53
+ // With +7 overhead, "Hello, world!" should be ~11 tokens
54
+ (0, vitest_1.expect)(counter.countTokens("Hello, world!")).toBeLessThan(15);
55
+ });
56
+ (0, vitest_1.it)("should handle provider prefixes", () => {
57
+ counter = new tokenCounter_1.TokenCounter("openai/gpt-4o");
58
+ (0, vitest_1.expect)(counter.getProvider()).toBe("openai");
59
+ (0, vitest_1.expect)(counter.getModel()).toBe("openai/gpt-4o");
60
+ });
61
+ (0, vitest_1.it)("should fall back to approximation for unknown models", () => {
62
+ counter = new tokenCounter_1.TokenCounter("unknown-model");
63
+ (0, vitest_1.expect)(counter.getProvider()).toBe("unknown");
64
+ const text = "Hello, world!";
65
+ const expected = Math.ceil(text.length / 3.5);
66
+ (0, vitest_1.expect)(counter.countTokens(text)).toBe(expected);
67
+ });
68
+ });
69
+ (0, vitest_1.describe)("Message counting", () => {
70
+ (0, vitest_1.beforeEach)(() => {
71
+ counter = new tokenCounter_1.TokenCounter("gpt-4o");
72
+ });
73
+ (0, vitest_1.it)("should count simple message with overhead", () => {
74
+ if (!counter)
75
+ throw new Error("Counter not initialized");
76
+ const message = {
77
+ role: "user",
78
+ content: "Hello, how are you?",
79
+ };
80
+ (0, vitest_1.expect)(counter.countMessageTokens(message)).toBeGreaterThan(4);
81
+ });
82
+ (0, vitest_1.it)("should count message with tool calls", () => {
83
+ if (!counter)
84
+ throw new Error("Counter not initialized");
85
+ const message = {
86
+ role: "assistant",
87
+ content: "Let me help you.",
88
+ tool_calls: [
89
+ {
90
+ id: "call_123",
91
+ type: "function",
92
+ function: {
93
+ name: "get_file",
94
+ arguments: JSON.stringify({ name: "test.txt" }),
95
+ },
96
+ },
97
+ ],
98
+ };
99
+ (0, vitest_1.expect)(counter.countMessageTokens(message)).toBeGreaterThan(10);
100
+ });
101
+ (0, vitest_1.it)("should count multimodal message with images", () => {
102
+ if (!counter)
103
+ throw new Error("Counter not initialized");
104
+ const message = {
105
+ role: "user",
106
+ content: [
107
+ { type: "text", text: "What's in this image?" },
108
+ {
109
+ type: "image_url",
110
+ image_url: { url: "data:image/png;base64,..." },
111
+ },
112
+ ],
113
+ };
114
+ // OpenAI images: ~170 tokens + text + overhead
115
+ (0, vitest_1.expect)(counter.countMessageTokens(message)).toBeGreaterThan(170);
116
+ });
117
+ (0, vitest_1.it)("should count conversation with overhead", () => {
118
+ if (!counter)
119
+ throw new Error("Counter not initialized");
120
+ const messages = [
121
+ { role: "system", content: "You are a helpful assistant." },
122
+ { role: "user", content: "Hello!" },
123
+ { role: "assistant", content: "Hi there! How can I help?" },
124
+ ];
125
+ (0, vitest_1.expect)(counter.countMessagesTokens(messages)).toBeGreaterThan(20);
126
+ (0, vitest_1.expect)(counter.countMessagesTokens([])).toBe(0);
127
+ });
128
+ });
129
+ (0, vitest_1.describe)("Factory functions", () => {
130
+ (0, vitest_1.it)("should create counter with factory", () => {
131
+ const testCounter = (0, tokenCounter_1.createTokenCounter)("gpt-4o");
132
+ (0, vitest_1.expect)(testCounter.getProvider()).toBe("openai");
133
+ testCounter.free();
134
+ });
135
+ (0, vitest_1.it)("should count tokens quickly", () => {
136
+ const count = (0, tokenCounter_1.countTokensQuick)("Hello, world!", "gpt-4o");
137
+ (0, vitest_1.expect)(count).toBeGreaterThan(0);
138
+ });
139
+ });
140
+ (0, vitest_1.describe)("Edge cases", () => {
141
+ (0, vitest_1.beforeEach)(() => {
142
+ counter = new tokenCounter_1.TokenCounter("gpt-4o");
143
+ });
144
+ (0, vitest_1.it)("should handle long text, special chars, and code", () => {
145
+ if (!counter)
146
+ throw new Error("Counter not initialized");
147
+ (0, vitest_1.expect)(counter.countTokens("Hello ".repeat(1000))).toBeGreaterThan(1000);
148
+ (0, vitest_1.expect)(counter.countTokens("Hello! 你好 🌍")).toBeGreaterThan(0);
149
+ (0, vitest_1.expect)(counter.countTokens('function hello() { return "world"; }')).toBeGreaterThan(5);
150
+ });
151
+ });
152
+ });
153
+ (0, vitest_1.describe)("Dev mode accurate counting", () => {
154
+ (0, vitest_1.it)("should throw error without dev mode", async () => {
155
+ const counter = new tokenCounter_1.TokenCounter("anthropic/claude-sonnet-4.5");
156
+ await (0, vitest_1.expect)(counter.countTokensAccurate("test")).rejects.toThrow("Accurate token counting requires enableDevMode");
157
+ counter.free();
158
+ });
159
+ (0, vitest_1.it)("should compare p50k_base approximation vs Anthropic API", async () => {
160
+ const counter = new tokenCounter_1.TokenCounter("claude-3-7-sonnet-20250219", {
161
+ enableDevMode: true,
162
+ });
163
+ // Test cases covering different scales
164
+ const readmePath = (0, path_1.join)(__dirname, "..", "..", "README.md");
165
+ const harryPotterPath = (0, path_1.join)(__dirname, "test_data", "harrypotter.txt");
166
+ const testCases = [
167
+ { text: "Hello, world!", label: "short" },
168
+ { text: "The quick brown fox jumps over the lazy dog.", label: "medium" },
169
+ { text: (0, fs_1.readFileSync)(readmePath, "utf-8"), label: "README.md" },
170
+ {
171
+ text: (0, fs_1.readFileSync)(harryPotterPath, "utf-8"),
172
+ label: "harrypotter.txt",
173
+ },
174
+ ];
175
+ if (process.env.ANTHROPIC_API_KEY) {
176
+ console.log("\n📊 Token Count Comparison (p50k_base+7 vs Anthropic API)\n");
177
+ for (const { text, label } of testCases) {
178
+ const accurate = await counter.countTokensAccurate(text);
179
+ const approximate = counter.countTokens(text);
180
+ const diff = accurate - approximate;
181
+ const accuracy = ((approximate / accurate) * 100).toFixed(1);
182
+ const size = text.length > 1000
183
+ ? `${String(text.length)} chars`
184
+ : `${String(text.length)} chars`;
185
+ console.log(`${label.padEnd(20)} [${size}]`);
186
+ console.log(` Accurate: ${accurate.toLocaleString().padStart(8)} tokens`);
187
+ console.log(` Approximate: ${approximate.toLocaleString().padStart(8)} tokens`);
188
+ const diffStr = (diff > 0 ? "+" : "") + String(diff);
189
+ console.log(` Difference: ${diffStr.padStart(8)} (${accuracy}%)\n`);
190
+ (0, vitest_1.expect)(accurate).toBeGreaterThan(0);
191
+ (0, vitest_1.expect)(approximate).toBeGreaterThan(0);
192
+ }
193
+ }
194
+ else {
195
+ await (0, vitest_1.expect)(counter.countTokensAccurate("test")).rejects.toThrow();
196
+ }
197
+ counter.free();
198
+ });
199
+ (0, vitest_1.it)("should return regular count for non-Anthropic models", async () => {
200
+ const counter = new tokenCounter_1.TokenCounter("gpt-4o", { enableDevMode: true });
201
+ const regular = counter.countTokens("Hello, world!");
202
+ const accurate = await counter.countTokensAccurate("Hello, world!");
203
+ (0, vitest_1.expect)(accurate).toBe(regular);
204
+ counter.free();
205
+ });
206
+ });
@@ -0,0 +1,17 @@
1
+ "use strict";
2
+ /**
3
+ * Centralized settings for agent tools.
4
+ */
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.OPEN_URL_MAX_LENGTH = exports.MAX_TOOL_CALL_RESPONSE_LENGTH = void 0;
7
+ const env = (key, defaultValue) => process.env[key] || defaultValue;
8
+ /**
9
+ * Maximum length for tool call responses before truncation.
10
+ * Applied as a final sanity check on all tool call results.
11
+ */
12
+ exports.MAX_TOOL_CALL_RESPONSE_LENGTH = parseInt(env("MAX_TOOL_CALL_RESPONSE_LENGTH", "50000"), 10);
13
+ /**
14
+ * Maximum length for open_url content extraction.
15
+ * Controls how much text is extracted from HTML pages.
16
+ */
17
+ exports.OPEN_URL_MAX_LENGTH = parseInt(env("OPEN_URL_MAX_LENGTH", "50000"), 10);
@@ -0,0 +1,45 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.calculatorTool = void 0;
4
+ exports.calculatorEval = calculatorEval;
5
+ const expr_eval_1 = require("expr-eval");
6
+ const utils_1 = require("./utils");
7
+ const ARITHMETIC_DESC = {
8
+ type: "function",
9
+ function: {
10
+ name: "arithmetic",
11
+ description: "Evaluate arithmetic expression",
12
+ parameters: {
13
+ type: "object",
14
+ properties: {
15
+ expr: {
16
+ type: "string",
17
+ description: "Expression containing +-*/()",
18
+ },
19
+ },
20
+ required: ["expr"],
21
+ },
22
+ },
23
+ };
24
+ function calculatorEval(args) {
25
+ try {
26
+ return String(expr_eval_1.Parser.evaluate(args));
27
+ }
28
+ catch (e) {
29
+ if (typeof e.message === "string") {
30
+ return e.message;
31
+ }
32
+ return String(e);
33
+ }
34
+ }
35
+ exports.calculatorTool = {
36
+ // eslint-disable-next-line @typescript-eslint/require-await
37
+ setup: async (agent) => {
38
+ const getExpr = (0, utils_1.makeParseArgsFn)(["expr"]);
39
+ const toolFn = async (_, args) => {
40
+ const { expr } = getExpr(args);
41
+ return Promise.resolve({ response: calculatorEval(expr) });
42
+ };
43
+ agent.addAgentTool(ARITHMETIC_DESC, toolFn);
44
+ },
45
+ };