llms 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +21 -0
  3. data/README.md +160 -0
  4. data/bin/llms-chat +6 -0
  5. data/bin/llms-test-model-access +4 -0
  6. data/bin/llms-test-model-image-support +4 -0
  7. data/bin/llms-test-model-prompt-caching +4 -0
  8. data/bin/llms-test-model-tool-use +5 -0
  9. data/lib/llms/adapters/anthropic_message_adapter.rb +73 -0
  10. data/lib/llms/adapters/anthropic_tool_call_adapter.rb +20 -0
  11. data/lib/llms/adapters/base_message_adapter.rb +60 -0
  12. data/lib/llms/adapters/google_gemini_message_adapter.rb +72 -0
  13. data/lib/llms/adapters/google_gemini_tool_call_adapter.rb +20 -0
  14. data/lib/llms/adapters/open_ai_compatible_message_adapter.rb +88 -0
  15. data/lib/llms/adapters/open_ai_compatible_tool_call_adapter.rb +67 -0
  16. data/lib/llms/adapters.rb +12 -0
  17. data/lib/llms/apis/google_gemini_api.rb +45 -0
  18. data/lib/llms/apis/open_ai_compatible_api.rb +54 -0
  19. data/lib/llms/cli/base.rb +186 -0
  20. data/lib/llms/cli/chat.rb +92 -0
  21. data/lib/llms/cli/test_access.rb +79 -0
  22. data/lib/llms/cli/test_image_support.rb +92 -0
  23. data/lib/llms/cli/test_prompt_caching.rb +275 -0
  24. data/lib/llms/cli/test_tool_use.rb +108 -0
  25. data/lib/llms/cli.rb +12 -0
  26. data/lib/llms/conversation.rb +100 -0
  27. data/lib/llms/conversation_message.rb +60 -0
  28. data/lib/llms/conversation_tool_call.rb +14 -0
  29. data/lib/llms/conversation_tool_result.rb +15 -0
  30. data/lib/llms/exceptions.rb +33 -0
  31. data/lib/llms/executors/anthropic_executor.rb +247 -0
  32. data/lib/llms/executors/base_executor.rb +144 -0
  33. data/lib/llms/executors/google_gemini_executor.rb +212 -0
  34. data/lib/llms/executors/hugging_face_executor.rb +17 -0
  35. data/lib/llms/executors/open_ai_compatible_executor.rb +209 -0
  36. data/lib/llms/executors.rb +52 -0
  37. data/lib/llms/models/model.rb +86 -0
  38. data/lib/llms/models/provider.rb +48 -0
  39. data/lib/llms/models.rb +187 -0
  40. data/lib/llms/parsers/anthropic_chat_response_stream_parser.rb +184 -0
  41. data/lib/llms/parsers/google_gemini_chat_response_stream_parser.rb +128 -0
  42. data/lib/llms/parsers/open_ai_compatible_chat_response_stream_parser.rb +170 -0
  43. data/lib/llms/parsers/partial_json_parser.rb +77 -0
  44. data/lib/llms/parsers/sse_chat_response_stream_parser.rb +72 -0
  45. data/lib/llms/public_models.json +607 -0
  46. data/lib/llms/stream/event_emitter.rb +48 -0
  47. data/lib/llms/stream/events.rb +104 -0
  48. data/lib/llms/usage/cost_calculator.rb +75 -0
  49. data/lib/llms/usage/usage_data.rb +46 -0
  50. data/lib/llms.rb +16 -0
  51. metadata +243 -0
data/lib/llms/public_models.json ADDED
@@ -0,0 +1,607 @@
1
+ {
2
+ "anthropic": {
3
+ "enabled": true,
4
+ "executor": "AnthropicExecutor",
5
+ "api_key_env_var": "ANTHROPIC_API_KEY",
6
+ "tools": true,
7
+ "vision": true,
8
+ "models": {
9
+ "claude-opus-4-20250514": {
10
+ "aliases": ["claude-opus-4-0"],
11
+ "latest": true,
12
+ "pricing": {
13
+ "input": 15.00,
14
+ "output": 75.00,
15
+ "cache_write_5min": 18.75,
16
+ "cache_write_1hr": 30.00,
17
+ "cache_read": 1.50
18
+ },
19
+ "context_window": 200000,
20
+ "max_output": 32000,
21
+ "max_output_thinking": 32000
22
+ },
23
+ "claude-sonnet-4-20250514": {
24
+ "aliases": ["claude-sonnet-4-0"],
25
+ "latest": true,
26
+ "pricing": {
27
+ "input": 3.00,
28
+ "output": 15.00,
29
+ "cache_write_5min": 3.75,
30
+ "cache_write_1hr": 6.00,
31
+ "cache_read": 0.30
32
+ },
33
+ "context_window": 200000,
34
+ "max_output": 64000,
35
+ "max_output_thinking": 64000
36
+ },
37
+ "claude-3-7-sonnet-20250219": {
38
+ "aliases": ["claude-3-7-sonnet-latest"],
39
+ "pricing": {
40
+ "input": 3.00,
41
+ "output": 15.00,
42
+ "cache_write_5min": 3.75,
43
+ "cache_write_1hr": 6.00,
44
+ "cache_read": 0.30
45
+ },
46
+ "context_window": 200000,
47
+ "max_output": 8192,
48
+ "max_output_thinking": 64000,
49
+ "_note": "Include the beta header output-128k-2025-02-19 in your API request to increase the maximum output token length to 128k tokens for Claude 3.7 Sonnet."
50
+ },
51
+ "claude-3-5-sonnet-20241022": {
52
+ "aliases": ["claude-3-5-sonnet-latest"],
53
+ "pricing": {
54
+ "input": 3.00,
55
+ "output": 15.00,
56
+ "cache_write_5min": 3.75,
57
+ "cache_write_1hr": 6.00,
58
+ "cache_read": 0.30
59
+ },
60
+ "context_window": 200000,
61
+ "max_output": 8192
62
+ },
63
+ "claude-3-5-sonnet-20240620": {
64
+ "pricing": {
65
+ "input": 3.00,
66
+ "output": 15.00,
67
+ "cache_write_5min": 3.75,
68
+ "cache_write_1hr": 6.00,
69
+ "cache_read": 0.30
70
+ },
71
+ "context_window": 200000,
72
+ "max_output": 8192
73
+ },
74
+ "claude-3-5-haiku-20241022": {
75
+ "aliases": ["claude-3-5-haiku-latest"],
76
+ "latest": true,
77
+ "pricing": {
78
+ "input": 0.8,
79
+ "output": 4.00,
80
+ "cache_write_5min": 1.00,
81
+ "cache_write_1hr": 1.60,
82
+ "cache_read": 0.08
83
+ },
84
+ "context_window": 200000,
85
+ "max_output": 8192
86
+ },
87
+ "claude-3-opus-20240229": {
88
+ "pricing": {
89
+ "input": 15.00,
90
+ "output": 75.00,
91
+ "cache_write_5min": 18.75,
92
+ "cache_write_1hr": 30.00,
93
+ "cache_read": 1.50
94
+ },
95
+ "context_window": 200000,
96
+ "max_output": 4096
97
+ },
98
+ "claude-3-haiku-20240307": {
99
+ "pricing": {
100
+ "input": 0.25,
101
+ "output": 1.25,
102
+ "cache_write_5min": 0.30,
103
+ "cache_write_1hr": 0.50,
104
+ "cache_read": 0.03
105
+ },
106
+ "context_window": 200000,
107
+ "max_output": 4096
108
+ }
109
+ }
110
+ },
111
+
112
+ "google": {
113
+ "enabled": true,
114
+ "executor": "GoogleGeminiExecutor",
115
+ "api_key_env_var": "GOOGLE_GEMINI_API_KEY",
116
+ "tools": true,
117
+ "vision": true,
118
+ "_TODO": "check all below do support vision; mark which do not support thinking",
119
+ "models": {
120
+ "gemini-2.5-pro": {
121
+ "latest": true,
122
+ "pricing": {
123
+ "input": 1.25,
124
+ "output": 10.00,
125
+ "cache_write_1mt/hr": 4.50,
126
+ "cache_read": 0.31
127
+ },
128
+ "context_window": 1048576,
129
+ "max_output": 65536
130
+ },
131
+ "gemini-2.5-flash": {
132
+ "latest": true,
133
+ "pricing": {
134
+ "input": 0.30,
135
+ "output": 1.00,
136
+ "cache_write_1mt/hr": 1.00,
137
+ "cache_read": 0.075,
138
+ "context_window": 1048576
139
+ },
140
+ "max_output": 65536
141
+ },
142
+ "gemini-2.5-flash-lite-preview-06-17": {
143
+ "latest": true,
144
+ "pricing": {
145
+ "input": 0.10,
146
+ "output": 0.40,
147
+ "cache_write_1mt/hr": 1.00,
148
+ "cache_read": 0.025
149
+ },
150
+ "context_window": 1000000,
151
+ "max_output": 64000
152
+ },
153
+ "gemini-2.0-flash": {
154
+ "pricing": {
155
+ "input": 0.10,
156
+ "output": 0.40,
157
+ "cache_write_1mt/hr": 1.00,
158
+ "cache_read": 0.025
159
+ },
160
+ "context_window": 1048576,
161
+ "max_output": 8192
162
+ },
163
+ "gemini-2.0-flash-lite": {
164
+ "pricing": {
165
+ "input": 0.075,
166
+ "output": 0.30
167
+ },
168
+ "context_window": 1048576,
169
+ "max_output": 8192
170
+ },
171
+ "gemini-1.5-pro": {
172
+ "pricing": {
173
+ "input": 1.25,
174
+ "output": 5.00,
175
+ "cache_write_1mt/hr": 4.50,
176
+ "cache_read": 0.3125
177
+ },
178
+ "context_window": 2097152,
179
+ "max_output": 8192
180
+ },
181
+ "gemini-1.5-flash": {
182
+ "pricing": {
183
+ "input": 0.075,
184
+ "output": 0.30,
185
+ "cache_write_1mt/hr": 1.00,
186
+ "cache_read": 0.0185
187
+ },
188
+ "context_window": 1048576,
189
+ "max_output": 8192
190
+ },
191
+ "gemini-1.5-flash-8b": {
192
+ "pricing": {
193
+ "input": 0.0375,
194
+ "output": 0.075,
195
+ "cache_write_1mt/hr": 0.25,
196
+ "cache_read": 0.01
197
+ },
198
+ "context_window": 1048576,
199
+ "max_output": 8192
200
+ }
201
+ }
202
+ },
203
+
204
+ "xai": {
205
+ "enabled": true,
206
+ "executor": "OpenAICompatibleExecutor",
207
+ "api_key_env_var": "XAI_API_KEY",
208
+ "base_url": "https://api.x.ai/v1",
209
+ "tools": true,
210
+ "models": {
211
+ "grok-4-0709": {
212
+ "latest": true,
213
+ "pricing": {
214
+ "input": 3.00,
215
+ "cached_input": 0.75,
216
+ "output": 15.00
217
+ },
218
+ "context_window": 256000,
219
+ "vision": true
220
+ },
221
+ "grok-3": {
222
+ "pricing": {
223
+ "input": 3.00,
224
+ "cached_input": 0.75,
225
+ "output": 15.00
226
+ },
227
+ "context_window": 131072,
228
+ "vision": false
229
+ },
230
+ "grok-3-mini": {
231
+ "latest": true,
232
+ "pricing": {
233
+ "input": 0.30,
234
+ "cached_input": 0.075,
235
+ "output": 0.50
236
+ },
237
+ "context_window": 131072,
238
+ "vision": false
239
+ },
240
+ "grok-3-fast": {
241
+ "pricing": {
242
+ "input": 5.00,
243
+ "cached_input": 1.25,
244
+ "output": 25.00
245
+ },
246
+ "context_window": 131072,
247
+ "vision": false
248
+ },
249
+ "grok-3-mini-fast": {
250
+ "latest": true,
251
+ "pricing": {
252
+ "input": 0.60,
253
+ "cached_input": 0.15,
254
+ "output": 4.00
255
+ },
256
+ "context_window": 131072,
257
+ "vision": false
258
+ },
259
+ "grok-2-1212": {
260
+ "pricing": {
261
+ "input": 2.00,
262
+ "output": 10.00
263
+ },
264
+ "context_window": 131072,
265
+ "vision": false
266
+ },
267
+ "grok-2-vision-1212": {
268
+ "pricing": {
269
+ "input": 2.00,
270
+ "output": 10.00
271
+ },
272
+ "context_window": 32768,
273
+ "vision": true
274
+ }
275
+ }
276
+ },
277
+
278
+ "cerebras": {
279
+ "enabled": true,
280
+ "executor": "OpenAICompatibleExecutor",
281
+ "api_key_env_var": "CEREBRAS_API_KEY",
282
+ "base_url": "https://api.cerebras.ai/v1",
283
+ "exclude_params": ["max_tokens"],
284
+ "models": {
285
+ "llama-4-scout-17b-16e-instruct": {
286
+ "pricing": {
287
+ "input": 0.65,
288
+ "output": 0.85
289
+ },
290
+ "context_window": 8192,
291
+ "tools": true
292
+ },
293
+ "llama3.1-8b": {
294
+ "pricing": {
295
+ "input": 0.10,
296
+ "output": 0.10
297
+ },
298
+ "context_window": 8192,
299
+ "tools": true
300
+ },
301
+ "llama3.3-70b": {
302
+ "pricing": {
303
+ "input": 0.85,
304
+ "output": 1.20
305
+ },
306
+ "context_window": 8192,
307
+ "tools": true
308
+ },
309
+ "qwen-3-32b": {
310
+ "pricing": {
311
+ "input": 0.40,
312
+ "output": 0.80
313
+ },
314
+ "context_window": 64000,
315
+ "tools": true,
316
+ "enabled": false
317
+ },
318
+ "qwen-3-235b-a22b": {
319
+ "pricing": {
320
+ "input": 0.60,
321
+ "output": 1.20
322
+ },
323
+ "context_window": 41000,
324
+ "tools": true,
325
+ "enabled": false
326
+ }
327
+ }
328
+ },
329
+
330
+ "hyperbolic": {
331
+ "enabled": true,
332
+ "executor": "OpenAICompatibleExecutor",
333
+ "api_key_env_var": "HYPERBOLIC_API_KEY",
334
+ "base_url": "https://api.hyperbolic.xyz/v1",
335
+ "tools": false,
336
+ "models": {
337
+ "moonshotai/Kimi-K2-Instruct": {
338
+ "pricing": {
339
+ "input": 2.00,
340
+ "output": 2.00
341
+ },
342
+ "context_window": 131069,
343
+ "vision": false
344
+ },
345
+ "deepseek-ai/DeepSeek-R1-0528": {
346
+ "pricing": {
347
+ "input": 3.00,
348
+ "output": 3.00
349
+ },
350
+ "context_window": 163840,
351
+ "vision": false
352
+ },
353
+ "Qwen/Qwen3-235B-A22B": {
354
+ "pricing": {
355
+ "input": 0.40,
356
+ "output": 0.40
357
+ },
358
+ "context_window": 40960
359
+ },
360
+ "deepseek-ai/DeepSeek-V3-0324": {
361
+ "pricing": {
362
+ "input": 1.25,
363
+ "output": 1.25
364
+ },
365
+ "context_window": 131069,
366
+ "vision": false
367
+ },
368
+ "Qwen/QwQ-32B": {
369
+ "pricing": {
370
+ "input": 0.40,
371
+ "output": 0.40
372
+ },
373
+ "context_window": 131069,
374
+ "vision": false
375
+ },
376
+ "meta-llama/Llama-3.3-70B-Instruct": {
377
+ "pricing": {
378
+ "input": 0.40,
379
+ "output": 0.40
380
+ },
381
+ "context_window": 131069,
382
+ "vision": false
383
+ },
384
+ "Qwen/Qwen2.5-Coder-32B-Instruct": {
385
+ "pricing": {
386
+ "input": 0.2,
387
+ "output": 0.2
388
+ },
389
+ "context_window": 131072,
390
+ "vision": false
391
+ }
392
+ }
393
+ },
394
+
395
+ "groq": {
396
+ "enabled": true,
397
+ "executor": "OpenAICompatibleExecutor",
398
+ "api_key_env_var": "GROQ_API_KEY",
399
+ "base_url": "https://api.groq.com/openai/v1",
400
+ "models": {
401
+ "llama-3.3-70b-versatile": {
402
+ "pricing": {
403
+ "input": 0.59,
404
+ "output": 0.79
405
+ },
406
+ "context_window": 131072,
407
+ "max_tokens": 32768,
408
+ "tools": true
409
+ },
410
+ "llama-3.1-8b-instant": {
411
+ "pricing": {
412
+ "input": 0.05,
413
+ "output": 0.08
414
+ },
415
+ "context_window": 131072,
416
+ "max_tokens": 131072,
417
+ "tools": true
418
+ }
419
+ }
420
+ },
421
+
422
+ "together": {
423
+ "enabled": true,
424
+ "executor": "OpenAICompatibleExecutor",
425
+ "api_key_env_var": "TOGETHER_API_KEY",
426
+ "base_url": "https://api.together.xyz/v1",
427
+ "exclude_params": ["max_completion_tokens"],
428
+ "models": {
429
+ "moonshotai/Kimi-K2-Instruct": {
430
+ "pricing": {
431
+ "input": 1.00,
432
+ "output": 3.00
433
+ },
434
+ "context_window": 128000,
435
+ "quantization": "FP8",
436
+ "tools": true
437
+ },
438
+ "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
439
+ "pricing": {
440
+ "input": 0.27,
441
+ "output": 0.85
442
+ },
443
+ "context_window": 1048576,
444
+ "quantization": "FP8",
445
+ "tools": true
446
+ },
447
+ "meta-llama/Llama-4-Scout-17B-16E-Instruct": {
448
+ "pricing": {
449
+ "input": 0.18,
450
+ "output": 0.59
451
+ },
452
+ "context_window": 1048576,
453
+ "tools": true
454
+ },
455
+ "meta-llama/Llama-3.3-70B-Instruct-Turbo": {
456
+ "pricing": {
457
+ "input": 0.88,
458
+ "output": 0.88
459
+ },
460
+ "context_window": 131072,
461
+ "quantization": "FP8",
462
+ "tools": true
463
+ },
464
+ "meta-llama/Llama-3.2-3B-Instruct-Turbo": {
465
+ "pricing": {
466
+ "input": 0.06,
467
+ "output": 0.06
468
+ },
469
+ "context_window": 131072,
470
+ "tools": true
471
+ },
472
+ "Qwen/Qwen3-235B-A22B-fp8-tput": {
473
+ "pricing": {
474
+ "input": 0.20,
475
+ "output": 0.60
476
+ },
477
+ "context_window": 40960,
478
+ "quantization": "FP8",
479
+ "tools": true
480
+ },
481
+ "deepseek-ai/DeepSeek-V3": {
482
+ "pricing": {
483
+ "input": 1.25,
484
+ "output": 1.25
485
+ },
486
+ "context_window": 163839,
487
+ "quantization": "FP8",
488
+ "tools": true
489
+ }
490
+ }
491
+ },
492
+
493
+ "fireworks": {
494
+ "enabled": true,
495
+ "executor": "OpenAICompatibleExecutor",
496
+ "api_key_env_var": "FIREWORKS_API_KEY",
497
+ "base_url": "https://api.fireworks.ai/inference/v1",
498
+ "exclude_params": ["max_tokens"],
499
+ "vision": true,
500
+ "models": {
501
+ "accounts/fireworks/models/deepseek-r1-0528": {
502
+ "pricing": {
503
+ "input": 3.00,
504
+ "output": 8.00
505
+ },
506
+ "context_window": 160000,
507
+ "tools": false,
508
+ "vision": false
509
+ },
510
+ "accounts/fireworks/models/qwen3-235b-a22b": {
511
+ "pricing": {
512
+ "input": 0.22,
513
+ "output": 0.88
514
+ },
515
+ "context_window": 128000,
516
+ "tools": true,
517
+ "vision": false
518
+ },
519
+ "accounts/fireworks/models/qwen3-30b-a3b": {
520
+ "pricing": {
521
+ "input": 0.15,
522
+ "output": 0.60
523
+ },
524
+ "context_window": 128000,
525
+ "tools": true,
526
+ "vision": false
527
+ },
528
+ "accounts/fireworks/models/llama4-maverick-instruct-basic": {
529
+ "pricing": {
530
+ "input": 0.22,
531
+ "output": 0.88
532
+ },
533
+ "context_window": 1000000,
534
+ "tools": true
535
+ },
536
+ "accounts/fireworks/models/llama4-scout-instruct-basic": {
537
+ "pricing": {
538
+ "input": 0.15,
539
+ "output": 0.60
540
+ },
541
+ "context_window": 10000000,
542
+ "tools": true
543
+ },
544
+ "accounts/fireworks/models/deepseek-v3-0324": {
545
+ "pricing": {
546
+ "input": 0.90,
547
+ "output": 0.90
548
+ },
549
+ "context_window": 160000,
550
+ "tools": true,
551
+ "vision": false
552
+ },
553
+ "accounts/fireworks/models/kimi-k2-instruct" : {
554
+ "pricing": {
555
+ "input": 1.00,
556
+ "output": 3.00
557
+ },
558
+ "context_window": 128000,
559
+ "tools": true,
560
+ "vision": false
561
+ }
562
+ }
563
+ },
564
+
565
+ "deepinfra": {
566
+ "enabled": false,
567
+ "executor": "OpenAICompatibleExecutor",
568
+ "api_key_env_var": "DEEPINFRA_API_KEY",
569
+ "base_url": "https://api.deepinfra.com/v1/openai",
570
+ "models": {
571
+ "moonshotai/Kimi-K2-Instruct": {
572
+ "pricing": {
573
+ "input": 0.55,
574
+ "output": 2.20
575
+ },
576
+ "context_window": 120000,
577
+ "quantization": "FP4",
578
+ "tools": true
579
+ }
580
+ }
581
+ },
582
+
583
+ "novita": {
584
+ "enabled": false,
585
+ "executor": "OpenAICompatibleExecutor",
586
+ "api_key_env_var": "NOVITA_API_KEY",
587
+ "base_url": "https://api.novita.ai/v3/openai",
588
+ "models": {
589
+ "moonshotai/kimi-k2-instruct": {
590
+ "pricing": {
591
+ "input": 0.57,
592
+ "output": 2.30
593
+ },
594
+ "context_window": 131072,
595
+ "tools": true
596
+ },
597
+ "deepseek/deepseek-v3-0324": {
598
+ "pricing": {
599
+ "input": 0.28,
600
+ "output": 1.14
601
+ },
602
+ "context_window": 163840,
603
+ "tools": true
604
+ }
605
+ }
606
+ }
607
+ }
data/lib/llms/stream/event_emitter.rb ADDED
@@ -0,0 +1,48 @@
1
+ module LLMs
2
+ module Stream
3
+ class EventEmitter
4
+ VALID_EVENTS = [
5
+ :message_started,
6
+ :usage_updated,
7
+ :text_delta,
8
+ :thinking_delta,
9
+ :tool_call_started,
10
+ :tool_call_arguments_json_delta,
11
+ :tool_call_arguments_updated,
12
+ :tool_call_completed,
13
+ :message_completed
14
+ ]
15
+
16
+ def initialize
17
+ @handlers = {}
18
+ end
19
+
20
+ def on(event_type, &block)
21
+ add_handler(event_type, block)
22
+ self
23
+ end
24
+
25
+ def connect(obj)
26
+ VALID_EVENTS.each do |event_type|
27
+ self.on(event_type) do |data|
28
+ obj.send(event_type, data)
29
+ end
30
+ end
31
+ self
32
+ end
33
+
34
+ def add_handler(event_type, callable)
35
+ raise ArgumentError, "Unknown event type: #{event_type}" unless VALID_EVENTS.include?(event_type)
36
+ @handlers[event_type] ||= []
37
+ @handlers[event_type] << callable
38
+ end
39
+
40
+ def emit(event_type, data)
41
+ raise ArgumentError, "Unknown event type: #{event_type}" unless VALID_EVENTS.include?(event_type)
42
+ @handlers[event_type]&.each do |handler|
43
+ handler.call(data)
44
+ end
45
+ end
46
+ end
47
+ end
48
+ end