@tyvm/knowhow 0.0.99 → 0.0.101

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/jest.manual.config.js +19 -0
  2. package/package.json +3 -1
  3. package/scripts/build-for-node.sh +162 -0
  4. package/src/agents/base/base.ts +32 -0
  5. package/src/cli.ts +23 -2
  6. package/src/clients/contextLimits.ts +0 -2
  7. package/src/clients/index.ts +19 -6
  8. package/src/clients/pricing/anthropic.ts +0 -6
  9. package/src/clients/pricing/google.ts +0 -4
  10. package/src/clients/pricing/xai.ts +11 -0
  11. package/src/login.ts +1 -4
  12. package/src/services/Mcp.ts +17 -3
  13. package/src/types.ts +17 -17
  14. package/tests/manual/clients/completions.json +454 -0
  15. package/tests/manual/clients/completions.test.ts +166 -0
  16. package/ts_build/package.json +3 -1
  17. package/ts_build/src/agents/base/base.js +15 -0
  18. package/ts_build/src/agents/base/base.js.map +1 -1
  19. package/ts_build/src/cli.js +12 -2
  20. package/ts_build/src/cli.js.map +1 -1
  21. package/ts_build/src/clients/anthropic.d.ts +0 -6
  22. package/ts_build/src/clients/contextLimits.js +0 -2
  23. package/ts_build/src/clients/contextLimits.js.map +1 -1
  24. package/ts_build/src/clients/index.js +16 -6
  25. package/ts_build/src/clients/index.js.map +1 -1
  26. package/ts_build/src/clients/pricing/anthropic.d.ts +0 -6
  27. package/ts_build/src/clients/pricing/anthropic.js +0 -6
  28. package/ts_build/src/clients/pricing/anthropic.js.map +1 -1
  29. package/ts_build/src/clients/pricing/google.js +0 -4
  30. package/ts_build/src/clients/pricing/google.js.map +1 -1
  31. package/ts_build/src/clients/pricing/xai.d.ts +10 -0
  32. package/ts_build/src/clients/pricing/xai.js +10 -0
  33. package/ts_build/src/clients/pricing/xai.js.map +1 -1
  34. package/ts_build/src/clients/xai.d.ts +13 -4
  35. package/ts_build/src/login.js +1 -4
  36. package/ts_build/src/login.js.map +1 -1
  37. package/ts_build/src/services/Mcp.js +11 -3
  38. package/ts_build/src/services/Mcp.js.map +1 -1
  39. package/ts_build/src/types.d.ts +2 -2
  40. package/ts_build/src/types.js +12 -13
  41. package/ts_build/src/types.js.map +1 -1
  42. package/ts_build/tests/manual/clients/completions.test.d.ts +1 -0
  43. package/ts_build/tests/manual/clients/completions.test.js +135 -0
  44. package/ts_build/tests/manual/clients/completions.test.js.map +1 -0
  45. package/test-ai-completion.ts +0 -39
  46. package/test-mcp-args.ts +0 -71
  47. package/test-tools-service.ts +0 -45
@@ -0,0 +1,454 @@
1
+ {
2
+ "anthropic::claude-haiku-4-5-20251001": {
3
+ "provider": "anthropic",
4
+ "model": "claude-haiku-4-5-20251001",
5
+ "prompt": "Say hello in 5 words.",
6
+ "response": "Hello, how are you today?",
7
+ "usage": {
8
+ "input_tokens": 15,
9
+ "cache_creation_input_tokens": 0,
10
+ "cache_read_input_tokens": 0,
11
+ "cache_creation": {
12
+ "ephemeral_5m_input_tokens": 0,
13
+ "ephemeral_1h_input_tokens": 0
14
+ },
15
+ "output_tokens": 10,
16
+ "service_tier": "standard",
17
+ "inference_geo": "not_available"
18
+ },
19
+ "usd_cost": 0.00006500000000000001,
20
+ "testedAt": "2026-04-14T19:18:23.775Z"
21
+ },
22
+ "anthropic::claude-sonnet-4-6": {
23
+ "provider": "anthropic",
24
+ "model": "claude-sonnet-4-6",
25
+ "prompt": "Say hello in 5 words.",
26
+ "response": "\"Hello there, how are you?\"",
27
+ "usage": {
28
+ "input_tokens": 15,
29
+ "cache_creation_input_tokens": 0,
30
+ "cache_read_input_tokens": 0,
31
+ "cache_creation": {
32
+ "ephemeral_5m_input_tokens": 0,
33
+ "ephemeral_1h_input_tokens": 0
34
+ },
35
+ "output_tokens": 11,
36
+ "service_tier": "standard",
37
+ "inference_geo": "global"
38
+ },
39
+ "usd_cost": 0.00021,
40
+ "testedAt": "2026-04-14T19:18:24.981Z"
41
+ },
42
+ "anthropic::claude-opus-4-6": {
43
+ "provider": "anthropic",
44
+ "model": "claude-opus-4-6",
45
+ "prompt": "Say hello in 5 words.",
46
+ "response": "Hello there, how are you?",
47
+ "usage": {
48
+ "input_tokens": 15,
49
+ "cache_creation_input_tokens": 0,
50
+ "cache_read_input_tokens": 0,
51
+ "cache_creation": {
52
+ "ephemeral_5m_input_tokens": 0,
53
+ "ephemeral_1h_input_tokens": 0
54
+ },
55
+ "output_tokens": 10,
56
+ "service_tier": "standard",
57
+ "inference_geo": "global"
58
+ },
59
+ "usd_cost": 0.000325,
60
+ "testedAt": "2026-04-14T19:18:27.010Z"
61
+ },
62
+ "google::gemini-3.1-flash-lite-preview": {
63
+ "provider": "google",
64
+ "model": "gemini-3.1-flash-lite-preview",
65
+ "prompt": "Say hello in 5 words.",
66
+ "response": "Hello, it is nice meeting.",
67
+ "usage": {
68
+ "promptTokenCount": 8,
69
+ "candidatesTokenCount": 7,
70
+ "totalTokenCount": 15,
71
+ "promptTokensDetails": [
72
+ {
73
+ "modality": "TEXT",
74
+ "tokenCount": 8
75
+ }
76
+ ]
77
+ },
78
+ "usd_cost": 0.000002,
79
+ "testedAt": "2026-04-14T19:18:35.317Z"
80
+ },
81
+ "google::gemini-3.1-flash-image-preview": {
82
+ "provider": "google",
83
+ "model": "gemini-3.1-flash-image-preview",
84
+ "prompt": "Say hello in 5 words.",
85
+ "response": "Hello there, welcome everyone, goodbye. \n",
86
+ "usage": {
87
+ "promptTokenCount": 8,
88
+ "candidatesTokenCount": 10,
89
+ "totalTokenCount": 18,
90
+ "promptTokensDetails": [
91
+ {
92
+ "modality": "TEXT",
93
+ "tokenCount": 8
94
+ }
95
+ ]
96
+ },
97
+ "usd_cost": 0.000004,
98
+ "testedAt": "2026-04-14T19:18:38.564Z"
99
+ },
100
+ "xai::grok-3-mini-fast-beta": {
101
+ "provider": "xai",
102
+ "model": "grok-3-mini-fast-beta",
103
+ "prompt": "Say hello in 5 words.",
104
+ "response": "Hello, this is Grok here.",
105
+ "usage": {
106
+ "prompt_tokens": 14,
107
+ "completion_tokens": 8,
108
+ "total_tokens": 554,
109
+ "prompt_tokens_details": {
110
+ "text_tokens": 14,
111
+ "audio_tokens": 0,
112
+ "image_tokens": 0,
113
+ "cached_tokens": 5
114
+ },
115
+ "completion_tokens_details": {
116
+ "reasoning_tokens": 532,
117
+ "audio_tokens": 0,
118
+ "accepted_prediction_tokens": 0,
119
+ "rejected_prediction_tokens": 0
120
+ },
121
+ "num_sources_used": 0,
122
+ "cost_in_usd_ticks": 2730750
123
+ },
124
+ "usd_cost": 0.0000404,
125
+ "testedAt": "2026-04-14T19:18:52.696Z"
126
+ },
127
+ "xai::grok-3-mini-beta": {
128
+ "provider": "xai",
129
+ "model": "grok-3-mini-beta",
130
+ "prompt": "Say hello in 5 words.",
131
+ "response": "Hello, I am Grok here.",
132
+ "usage": {
133
+ "prompt_tokens": 14,
134
+ "completion_tokens": 8,
135
+ "total_tokens": 549,
136
+ "prompt_tokens_details": {
137
+ "text_tokens": 14,
138
+ "audio_tokens": 0,
139
+ "image_tokens": 0,
140
+ "cached_tokens": 3
141
+ },
142
+ "completion_tokens_details": {
143
+ "reasoning_tokens": 527,
144
+ "audio_tokens": 0,
145
+ "accepted_prediction_tokens": 0,
146
+ "rejected_prediction_tokens": 0
147
+ },
148
+ "num_sources_used": 0,
149
+ "cost_in_usd_ticks": 2710250
150
+ },
151
+ "usd_cost": 0.0000082,
152
+ "testedAt": "2026-04-14T19:19:03.041Z"
153
+ },
154
+ "xai::grok-4-0709": {
155
+ "provider": "xai",
156
+ "model": "grok-4-0709",
157
+ "prompt": "Say hello in 5 words.",
158
+ "response": "Hello, how are you today?",
159
+ "usage": {
160
+ "prompt_tokens": 691,
161
+ "completion_tokens": 7,
162
+ "total_tokens": 849,
163
+ "prompt_tokens_details": {
164
+ "text_tokens": 691,
165
+ "audio_tokens": 0,
166
+ "image_tokens": 0,
167
+ "cached_tokens": 679
168
+ },
169
+ "completion_tokens_details": {
170
+ "reasoning_tokens": 151,
171
+ "audio_tokens": 0,
172
+ "accepted_prediction_tokens": 0,
173
+ "rejected_prediction_tokens": 0
174
+ },
175
+ "num_sources_used": 0,
176
+ "cost_in_usd_ticks": 29152500
177
+ },
178
+ "usd_cost": 0.0021780000000000002,
179
+ "testedAt": "2026-04-14T19:19:06.403Z"
180
+ },
181
+ "xai::grok-4-1-fast-non-reasoning": {
182
+ "provider": "xai",
183
+ "model": "grok-4-1-fast-non-reasoning",
184
+ "prompt": "Say hello in 5 words.",
185
+ "response": "Hello, world! How are you today?",
186
+ "usage": {
187
+ "prompt_tokens": 175,
188
+ "completion_tokens": 9,
189
+ "total_tokens": 184,
190
+ "prompt_tokens_details": {
191
+ "text_tokens": 175,
192
+ "audio_tokens": 0,
193
+ "image_tokens": 0,
194
+ "cached_tokens": 161
195
+ },
196
+ "completion_tokens_details": {
197
+ "reasoning_tokens": 0,
198
+ "audio_tokens": 0,
199
+ "accepted_prediction_tokens": 0,
200
+ "rejected_prediction_tokens": 0
201
+ },
202
+ "num_sources_used": 0,
203
+ "cost_in_usd_ticks": 153500
204
+ },
205
+ "usd_cost": 0.00004755,
206
+ "testedAt": "2026-04-14T19:19:06.825Z"
207
+ },
208
+ "openai::gpt-4o-mini-2024-07-18": {
209
+ "provider": "openai",
210
+ "model": "gpt-4o-mini-2024-07-18",
211
+ "prompt": "Say hello in 5 words.",
212
+ "response": "Hello! How are you today?",
213
+ "usage": {
214
+ "prompt_tokens": 14,
215
+ "completion_tokens": 7,
216
+ "total_tokens": 21,
217
+ "prompt_tokens_details": {
218
+ "cached_tokens": 0,
219
+ "audio_tokens": 0
220
+ },
221
+ "completion_tokens_details": {
222
+ "reasoning_tokens": 0,
223
+ "audio_tokens": 0,
224
+ "accepted_prediction_tokens": 0,
225
+ "rejected_prediction_tokens": 0
226
+ }
227
+ },
228
+ "usd_cost": 0.000006300000000000001,
229
+ "testedAt": "2026-04-14T19:20:03.349Z"
230
+ },
231
+ "openai::gpt-4.1-nano-2025-04-14": {
232
+ "provider": "openai",
233
+ "model": "gpt-4.1-nano-2025-04-14",
234
+ "prompt": "Say hello in 5 words.",
235
+ "response": "Hello! How are you today?",
236
+ "usage": {
237
+ "prompt_tokens": 14,
238
+ "completion_tokens": 7,
239
+ "total_tokens": 21,
240
+ "prompt_tokens_details": {
241
+ "cached_tokens": 0,
242
+ "audio_tokens": 0
243
+ },
244
+ "completion_tokens_details": {
245
+ "reasoning_tokens": 0,
246
+ "audio_tokens": 0,
247
+ "accepted_prediction_tokens": 0,
248
+ "rejected_prediction_tokens": 0
249
+ }
250
+ },
251
+ "usd_cost": 0.0000042000000000000004,
252
+ "testedAt": "2026-04-14T19:20:04.374Z"
253
+ },
254
+ "openai::gpt-5.4": {
255
+ "provider": "openai",
256
+ "model": "gpt-5.4",
257
+ "prompt": "Say hello in 5 words.",
258
+ "response": "Hello! Hope you're doing well.",
259
+ "usage": {
260
+ "prompt_tokens": 13,
261
+ "completion_tokens": 111,
262
+ "total_tokens": 124
263
+ },
264
+ "usd_cost": 0.0016975,
265
+ "testedAt": "2026-04-14T19:20:06.964Z"
266
+ },
267
+ "openai::gpt-5.4-pro": {
268
+ "provider": "openai",
269
+ "model": "gpt-5.4-pro",
270
+ "prompt": "Say hello in 5 words.",
271
+ "response": "Hello, have a wonderful day!",
272
+ "usage": {
273
+ "prompt_tokens": 13,
274
+ "completion_tokens": 157,
275
+ "total_tokens": 170
276
+ },
277
+ "usd_cost": 0.028650000000000002,
278
+ "testedAt": "2026-04-14T19:20:26.803Z"
279
+ },
280
+ "openai::gpt-5.4-mini": {
281
+ "provider": "openai",
282
+ "model": "gpt-5.4-mini",
283
+ "prompt": "Say hello in 5 words.",
284
+ "response": "Hello, how are you today?",
285
+ "usage": {
286
+ "prompt_tokens": 13,
287
+ "completion_tokens": 46,
288
+ "total_tokens": 59
289
+ },
290
+ "usd_cost": 0.00021674999999999998,
291
+ "testedAt": "2026-04-14T19:20:27.900Z"
292
+ },
293
+ "openai::gpt-5.4-nano": {
294
+ "provider": "openai",
295
+ "model": "gpt-5.4-nano",
296
+ "prompt": "Say hello in 5 words.",
297
+ "response": "Hello! How are you doing?",
298
+ "usage": {
299
+ "prompt_tokens": 13,
300
+ "completion_tokens": 11,
301
+ "total_tokens": 24
302
+ },
303
+ "usd_cost": 0.00001635,
304
+ "testedAt": "2026-04-14T19:20:28.644Z"
305
+ },
306
+ "openai::gpt-5.3-chat-latest": {
307
+ "provider": "openai",
308
+ "model": "gpt-5.3-chat-latest",
309
+ "prompt": "Say hello in 5 words.",
310
+ "response": "Hello there, nice to meet.",
311
+ "usage": {
312
+ "prompt_tokens": 13,
313
+ "completion_tokens": 16,
314
+ "total_tokens": 29,
315
+ "prompt_tokens_details": {
316
+ "cached_tokens": 0,
317
+ "audio_tokens": 0
318
+ },
319
+ "completion_tokens_details": {
320
+ "reasoning_tokens": 0,
321
+ "audio_tokens": 0,
322
+ "accepted_prediction_tokens": 0,
323
+ "rejected_prediction_tokens": 0
324
+ }
325
+ },
326
+ "usd_cost": 0.00024675,
327
+ "testedAt": "2026-04-14T19:20:30.469Z"
328
+ },
329
+ "openai::gpt-5.3-codex": {
330
+ "provider": "openai",
331
+ "model": "gpt-5.3-codex",
332
+ "prompt": "Say hello in 5 words.",
333
+ "response": "Hello there, hope you're doing great!",
334
+ "usage": {
335
+ "prompt_tokens": 13,
336
+ "completion_tokens": 12,
337
+ "total_tokens": 25
338
+ },
339
+ "usd_cost": 0.00019075,
340
+ "testedAt": "2026-04-14T19:20:31.376Z"
341
+ },
342
+ "google::gemini-3.1-pro-preview": {
343
+ "provider": "google",
344
+ "model": "gemini-3.1-pro-preview",
345
+ "prompt": "Say hello in 5 words.",
346
+ "response": "Hello,",
347
+ "usage": {
348
+ "promptTokenCount": 8,
349
+ "candidatesTokenCount": 2,
350
+ "totalTokenCount": 54,
351
+ "promptTokensDetails": [
352
+ {
353
+ "modality": "TEXT",
354
+ "tokenCount": 8
355
+ }
356
+ ],
357
+ "thoughtsTokenCount": 44
358
+ },
359
+ "usd_cost": 0.000016,
360
+ "testedAt": "2026-04-14T19:20:41.040Z"
361
+ },
362
+ "anthropic::claude-haiku-4-5": {
363
+ "provider": "anthropic",
364
+ "model": "claude-haiku-4-5",
365
+ "prompt": "Say hello in 5 words.",
366
+ "response": "Hello, how are you today?",
367
+ "usage": {
368
+ "input_tokens": 15,
369
+ "cache_creation_input_tokens": 0,
370
+ "cache_read_input_tokens": 0,
371
+ "cache_creation": {
372
+ "ephemeral_5m_input_tokens": 0,
373
+ "ephemeral_1h_input_tokens": 0
374
+ },
375
+ "output_tokens": 10,
376
+ "service_tier": "standard",
377
+ "inference_geo": "not_available"
378
+ },
379
+ "usd_cost": 0.00006500000000000001,
380
+ "testedAt": "2026-04-14T19:21:44.319Z"
381
+ },
382
+ "google::gemini-2.5-flash": {
383
+ "provider": "google",
384
+ "model": "gemini-2.5-flash",
385
+ "prompt": "Say hello in 5 words.",
386
+ "response": "Hello there",
387
+ "usage": {
388
+ "promptTokenCount": 8,
389
+ "candidatesTokenCount": 2,
390
+ "totalTokenCount": 54,
391
+ "promptTokensDetails": [
392
+ {
393
+ "modality": "TEXT",
394
+ "tokenCount": 8
395
+ }
396
+ ],
397
+ "thoughtsTokenCount": 44
398
+ },
399
+ "usd_cost": 0.0000024,
400
+ "testedAt": "2026-04-14T19:21:50.097Z"
401
+ },
402
+ "xai::grok-4.20-0309-non-reasoning": {
403
+ "provider": "xai",
404
+ "model": "grok-4.20-0309-non-reasoning",
405
+ "prompt": "Say hello in 5 words.",
406
+ "response": "Hello there, how are you?",
407
+ "usage": {
408
+ "prompt_tokens": 129,
409
+ "completion_tokens": 7,
410
+ "total_tokens": 136,
411
+ "prompt_tokens_details": {
412
+ "text_tokens": 129,
413
+ "audio_tokens": 0,
414
+ "image_tokens": 0,
415
+ "cached_tokens": 64
416
+ },
417
+ "completion_tokens_details": {
418
+ "reasoning_tokens": 0,
419
+ "audio_tokens": 0,
420
+ "accepted_prediction_tokens": 0,
421
+ "rejected_prediction_tokens": 0
422
+ },
423
+ "num_sources_used": 0,
424
+ "cost_in_usd_ticks": 1848000
425
+ },
426
+ "testedAt": "2026-04-14T19:28:47.420Z"
427
+ },
428
+ "xai::grok-4.20-0309-reasoning": {
429
+ "provider": "xai",
430
+ "model": "grok-4.20-0309-reasoning",
431
+ "prompt": "Say hello in 5 words.",
432
+ "response": "Hello there how are you?",
433
+ "usage": {
434
+ "prompt_tokens": 131,
435
+ "completion_tokens": 6,
436
+ "total_tokens": 1170,
437
+ "prompt_tokens_details": {
438
+ "text_tokens": 131,
439
+ "audio_tokens": 0,
440
+ "image_tokens": 0,
441
+ "cached_tokens": 64
442
+ },
443
+ "completion_tokens_details": {
444
+ "reasoning_tokens": 1033,
445
+ "audio_tokens": 0,
446
+ "accepted_prediction_tokens": 0,
447
+ "rejected_prediction_tokens": 0
448
+ },
449
+ "num_sources_used": 0,
450
+ "cost_in_usd_ticks": 63808000
451
+ },
452
+ "testedAt": "2026-04-14T19:28:53.889Z"
453
+ }
454
+ }
@@ -0,0 +1,166 @@
1
+ /**
2
+ * Text Completions Manual Test
3
+ *
4
+ * Tests text generation ("say hello in 5 words") across multiple models and providers.
5
+ * Results are persisted to `completions.json` so that re-running the suite skips
6
+ * models that have already been tested (avoiding unnecessary API spend).
7
+ *
8
+ * Run with:
9
+ * npx jest tests/manual/clients/completions.test.ts --testTimeout=60000 --runInBand
10
+ */
11
+
12
+ import * as fs from "fs";
13
+ import * as path from "path";
14
+ import { AIClient } from "../../../src/clients";
15
+ import { Models } from "../../../src/types";
16
+
17
+ const OUTPUT_FILE = path.join(__dirname, "completions.json");
18
+ const PROMPT = "Say hello in 5 words.";
19
+
20
+ // ─── Models to test ──────────────────────────────────────────────────────────
21
+ // One or two representative, cheap/fast models per provider.
22
+
23
+ interface ModelEntry {
24
+ provider: string;
25
+ model: string;
26
+ envKey: string;
27
+ }
28
+
29
+ const TEST_MODELS: ModelEntry[] = [
30
+ // OpenAI
31
+ { provider: "openai", model: Models.openai.GPT_4o_Mini, envKey: "OPENAI_KEY" },
32
+ { provider: "openai", model: Models.openai.GPT_41_Nano, envKey: "OPENAI_KEY" },
33
+ { provider: "openai", model: Models.openai.GPT_54, envKey: "OPENAI_KEY" },
34
+ { provider: "openai", model: Models.openai.GPT_54_Pro, envKey: "OPENAI_KEY" },
35
+ { provider: "openai", model: Models.openai.GPT_54_Mini, envKey: "OPENAI_KEY" },
36
+ { provider: "openai", model: Models.openai.GPT_54_Nano, envKey: "OPENAI_KEY" },
37
+ { provider: "openai", model: Models.openai.GPT_53_Chat, envKey: "OPENAI_KEY" },
38
+ { provider: "openai", model: Models.openai.GPT_53_Codex, envKey: "OPENAI_KEY" },
39
+
40
+ // Anthropic
41
+ { provider: "anthropic", model: Models.anthropic.Haiku4_5, envKey: "ANTHROPIC_API_KEY" },
42
+ { provider: "anthropic", model: Models.anthropic.Sonnet4_6, envKey: "ANTHROPIC_API_KEY" },
43
+ { provider: "anthropic", model: Models.anthropic.Opus4_6, envKey: "ANTHROPIC_API_KEY" },
44
+
45
+ // Google
46
+ { provider: "google", model: Models.google.Gemini_25_Flash, envKey: "GEMINI_API_KEY" },
47
+ { provider: "google", model: Models.google.Gemini_31_Flash_Lite_Preview, envKey: "GEMINI_API_KEY" },
48
+ { provider: "google", model: Models.google.Gemini_31_Flash_Image_Preview, envKey: "GEMINI_API_KEY" },
49
+ { provider: "google", model: Models.google.Gemini_31_Pro_Preview, envKey: "GEMINI_API_KEY" },
50
+
51
+ // XAI
52
+ { provider: "xai", model: Models.xai.Grok3MiniFastBeta, envKey: "XAI_API_KEY" },
53
+ { provider: "xai", model: Models.xai.Grok3MiniBeta, envKey: "XAI_API_KEY" },
54
+ { provider: "xai", model: Models.xai.Grok4, envKey: "XAI_API_KEY" },
55
+ { provider: "xai", model: Models.xai.Grok4_1_Fast_NonReasoning, envKey: "XAI_API_KEY" },
56
+ { provider: "xai", model: Models.xai.Grok4_1_Fast_NonReasoning, envKey: "XAI_API_KEY" },
57
+ { provider: "xai", model: Models.xai.Grok_4_20_NonReasoning, envKey: "XAI_API_KEY" },
58
+ { provider: "xai", model: Models.xai.Grok_4_20_Reasoning, envKey: "XAI_API_KEY" },
59
+ ];
60
+
61
+ // ─── Persistence helpers ──────────────────────────────────────────────────────
62
+
63
+ interface CompletionRecord {
64
+ provider: string;
65
+ model: string;
66
+ prompt: string;
67
+ response: string;
68
+ usage: {
69
+ prompt_tokens?: number;
70
+ completion_tokens?: number;
71
+ total_tokens?: number;
72
+ [key: string]: any;
73
+ };
74
+ usd_cost?: number;
75
+ testedAt: string;
76
+ }
77
+
78
+ type CompletionsJSON = Record<string, CompletionRecord>;
79
+
80
+ function loadResults(): CompletionsJSON {
81
+ if (fs.existsSync(OUTPUT_FILE)) {
82
+ try {
83
+ return JSON.parse(fs.readFileSync(OUTPUT_FILE, "utf-8")) as CompletionsJSON;
84
+ } catch {
85
+ return {};
86
+ }
87
+ }
88
+ return {};
89
+ }
90
+
91
+ function saveResults(results: CompletionsJSON): void {
92
+ fs.writeFileSync(OUTPUT_FILE, JSON.stringify(results, null, 2), "utf-8");
93
+ }
94
+
95
+ function recordKey(provider: string, model: string): string {
96
+ return `${provider}::${model}`;
97
+ }
98
+
99
+ // ─── Test suite ───────────────────────────────────────────────────────────────
100
+
101
+ describe("Text Completions – multi-provider model verification", () => {
102
+ let client: AIClient;
103
+ let results: CompletionsJSON;
104
+
105
+ beforeAll(() => {
106
+ client = new AIClient();
107
+ results = loadResults();
108
+ });
109
+
110
+ afterAll(() => {
111
+ saveResults(results);
112
+ console.log(`\n📄 Results saved to: ${OUTPUT_FILE}`);
113
+ console.log(` Total records: ${Object.keys(results).length}`);
114
+ });
115
+
116
+ for (const { provider, model, envKey } of TEST_MODELS) {
117
+ const key = recordKey(provider, model);
118
+
119
+ test(`${provider} / ${model}`, async () => {
120
+ // ── Skip if already recorded ──────────────────────────────────────────
121
+ if (results[key]) {
122
+ const rec = results[key];
123
+ console.log(`⏭ Skipping (already tested on ${rec.testedAt})`);
124
+ console.log(` Response : "${rec.response}"`);
125
+ console.log(` Cost : $${rec.usd_cost?.toFixed(8) ?? "unknown"}`);
126
+ expect(rec.response).toBeTruthy();
127
+ return;
128
+ }
129
+
130
+ // ── Skip if API key missing ───────────────────────────────────────────
131
+ if (!process.env[envKey]) {
132
+ console.log(`⏭ Skipping: ${envKey} not set`);
133
+ return;
134
+ }
135
+
136
+ // ── Run completion ────────────────────────────────────────────────────
137
+ const response = await client.createCompletion(provider, {
138
+ model,
139
+ messages: [{ role: "user", content: PROMPT }],
140
+ max_tokens: 50,
141
+ });
142
+
143
+ const text = response.choices[0]?.message?.content ?? "";
144
+ expect(text).toBeTruthy();
145
+
146
+ const record: CompletionRecord = {
147
+ provider,
148
+ model,
149
+ prompt: PROMPT,
150
+ response: text,
151
+ usage: response.usage ?? {},
152
+ usd_cost: response.usd_cost,
153
+ testedAt: new Date().toISOString(),
154
+ };
155
+
156
+ results[key] = record;
157
+ // Persist immediately so a mid-run failure doesn't lose earlier results
158
+ saveResults(results);
159
+
160
+ console.log(`✅ ${provider} / ${model}`);
161
+ console.log(` Response : "${text}"`);
162
+ console.log(` Tokens : ${JSON.stringify(response.usage)}`);
163
+ console.log(` Cost : $${response.usd_cost?.toFixed(8) ?? "unknown"}`);
164
+ }, 60_000);
165
+ }
166
+ });
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@tyvm/knowhow",
3
- "version": "0.0.99",
3
+ "version": "0.0.101",
4
4
  "description": "ai cli with plugins and agents",
5
5
  "main": "ts_build/src/index.js",
6
6
  "bin": {
@@ -9,8 +9,10 @@
9
9
  "scripts": {
10
10
  "test": "jest --testTimeout 300000",
11
11
  "test:debug": "node --inspect-brk ../../node_modules/jest/bin/jest.js --detectOpenHandles --forceExit --testTimeout 300000",
12
+ "build": "npm run compile",
12
13
  "compile": "npm run clean && tsc",
13
14
  "clean": "rm -rf ts_build",
15
+ "node:build": "bash scripts/build-for-node.sh",
14
16
  "start": "npm run compile && node --no-node-snapshot ts_build/src/server/index.js",
15
17
  "dataset:diffs:generate": "ts-node src/dataset/diffs/generate.ts",
16
18
  "dataset:diffs:jsonl": "ts-node src/dataset/diffs/jsonl.ts",
@@ -419,6 +419,10 @@ class BaseAgent {
419
419
  tools: this.getEnabledTools(),
420
420
  tool_choice: "auto",
421
421
  });
422
+ if (this.status === this.eventTypes.pause) {
423
+ this.log("Agent was paused after completion, waiting before processing tool calls");
424
+ await this.unpaused();
425
+ }
422
426
  if (response?.usd_cost === undefined) {
423
427
  this.log(`Response cost is undefined: ${JSON.stringify(response, null, 2)}`, "warn");
424
428
  const error = response;
@@ -438,12 +442,23 @@ class BaseAgent {
438
442
  messages = await this.messageProcessor.processMessages(messages, "pre_tools");
439
443
  this.updateCurrentThread(messages);
440
444
  for (const toolCall of toolCalls) {
445
+ if (this.status === this.eventTypes.pause) {
446
+ this.log("Agent was paused before tool call, waiting before processing tool calls");
447
+ await this.unpaused();
448
+ }
441
449
  const toolMessages = await this.processToolMessages(toolCall);
442
450
  messages.push(...toolMessages);
443
451
  const finalMessage = toolMessages.find((called) => this.requiredToolNames.includes(called.name) ||
444
452
  this.requiredToolNames.includes((0, utils_1.mcpToolName)(called.name)) ||
445
453
  this.requiredToolNames.some((required) => called.name.endsWith(required)));
446
454
  if (finalMessage) {
455
+ if (this.pendingUserMessages.length > 0) {
456
+ this.log("finalAnswer called but pending user messages exist, continuing to respond to feedback");
457
+ messages.push(...this.pendingUserMessages);
458
+ this.pendingUserMessages = [];
459
+ this.updateCurrentThread(messages);
460
+ return this.call(userInput, messages);
461
+ }
447
462
  this.events.emit(this.eventTypes.agentTaskComplete, {
448
463
  taskId: this.currentTaskId ||
449
464
  this.startTimeMs?.toString() ||