adaptive-memory-multi-model-router 2.13.3 → 2.13.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/README.md +181 -110
  2. package/benchmark-provider-results.json +120 -41
  3. package/benchmark-results.json +46 -620
  4. package/dist/tui/index.js +0 -0
  5. package/docs/BENCHMARK.md +96 -0
  6. package/docs/benchmark-chart.png +0 -0
  7. package/package.json +18 -3
  8. package/scripts/routing-benchmark-v3.js +118 -0
package/benchmark-results.json CHANGED
@@ -1,628 +1,54 @@
1
1
  {
2
- "meta": {
3
- "date": "2026-05-22T06:44:38.534Z",
4
- "type": "mmlu_quality"
5
- },
6
- "results": {
7
- "groq-llama-3.1-8b": {
8
- "name": "Groq Llama 3.1 8B",
9
- "mmlu_accuracy": 0.8,
10
- "quality_score": 9.4,
11
- "mmlu_detail": [
12
- {
13
- "q": "What is the derivative of x²?",
14
- "response": "B",
15
- "correct": true
16
- },
17
- {
18
- "q": "Solve for x: 2x + 6 = 14",
19
- "response": "A",
20
- "correct": true
21
- },
22
- {
23
- "q": "Time complexity of binary search?",
24
- "response": "B",
25
- "correct": true
26
- },
27
- {
28
- "q": "In Python, list.append() does what?",
29
- "response": "B",
30
- "correct": true
31
- },
32
- {
33
- "q": "Output of type([])?",
34
- "response": "B",
35
- "correct": true
36
- },
37
- {
38
- "q": "GDP stands for?",
39
- "response": "A",
40
- "correct": true
41
- },
42
- {
43
- "q": "If supply increases and demand stay",
44
- "response": "B",
45
- "correct": true
46
- },
47
- {
48
- "q": "Powerhouse of the cell?",
49
- "response": "C",
50
- "correct": true
51
- },
52
- {
53
- "q": "DNA stands for?",
54
- "response": "A",
55
- "correct": true
56
- },
57
- {
58
- "q": "Primary purpose of a contract?",
59
- "response": "B)",
60
- "correct": false
61
- },
62
- {
63
- "q": "Which US amendment protects free sp",
64
- "response": "A",
65
- "correct": true
66
- },
67
- {
68
- "q": "Car accelerates from rest at 2 m/s²",
69
- "response": "A)",
70
- "correct": false
71
- },
72
- {
73
- "q": "Unit of force in SI?",
74
- "response": "C)",
75
- "correct": false
76
- },
77
- {
78
- "q": "If gas temp increases at constant v",
79
- "response": "B",
80
- "correct": true
81
- },
82
- {
83
- "q": "What is π to two decimal places?",
84
- "response": "A",
85
- "correct": true
86
- }
87
- ],
88
- "quality_detail": [
89
- {
90
- "prompt": "If John has 5 apples and gives",
91
- "response": "To find out how many apples Jo",
92
- "score": 10
93
- },
94
- {
95
- "prompt": "Write a Python function that c",
96
- "response": "**Palindrome Checker Function*",
97
- "score": 10
98
- },
99
- {
100
- "prompt": "Compare supervised vs unsuperv",
101
- "response": "Supervised learning involves t",
102
- "score": 10
103
- },
104
- {
105
- "prompt": "What is the capital of France?",
106
- "response": "The capital of France is Paris",
107
- "score": 10
108
- },
109
- {
110
- "prompt": "Train travels 300km in 4 hours",
111
- "response": "To find the average speed, we ",
112
- "score": 10
113
- },
114
- {
115
- "prompt": "All cats are animals. Some ani",
116
- "response": "To determine if some cats are ",
117
- "score": 5
118
- },
119
- {
120
- "prompt": "What is machine learning?",
121
- "response": "Machine learning is a subset o",
122
- "score": 10
123
- },
124
- {
125
- "prompt": "Write a haiku about programmin",
126
- "response": "Code flows like a stream\nLogic",
127
- "score": 10
128
- }
129
- ]
2
+ "timestamp": "2026-05-26T20:21:58.203Z",
3
+ "version": "2.13.3",
4
+ "queries": 200,
5
+ "exact_accuracy": 64.5,
6
+ "adjacent_accuracy": 99.5,
7
+ "over_routed": 14,
8
+ "under_routed": 57,
9
+ "cost_savings_vs_premium": 61.6,
10
+ "by_tier": {
11
+ "free": {
12
+ "correct": 46,
13
+ "total": 50
14
+ },
15
+ "cheap": {
16
+ "correct": 47,
17
+ "total": 60
18
+ },
19
+ "mid": {
20
+ "correct": 18,
21
+ "total": 50
130
22
  },
131
- "groq-llama-3.3-70b": {
132
- "name": "Groq Llama 3.3 70B",
133
- "mmlu_accuracy": 0.6666666666666666,
134
- "quality_score": 9.4,
135
- "mmlu_detail": [
136
- {
137
- "q": "What is the derivative of x²?",
138
- "response": "B",
139
- "correct": true
140
- },
141
- {
142
- "q": "Solve for x: 2x + 6 = 14",
143
- "response": "B)",
144
- "correct": false
145
- },
146
- {
147
- "q": "Time complexity of binary search?",
148
- "response": "B)",
149
- "correct": false
150
- },
151
- {
152
- "q": "In Python, list.append() does what?",
153
- "response": "B",
154
- "correct": true
155
- },
156
- {
157
- "q": "Output of type([])?",
158
- "response": "B)",
159
- "correct": false
160
- },
161
- {
162
- "q": "GDP stands for?",
163
- "response": "A",
164
- "correct": true
165
- },
166
- {
167
- "q": "If supply increases and demand stay",
168
- "response": "B",
169
- "correct": true
170
- },
171
- {
172
- "q": "Powerhouse of the cell?",
173
- "response": "C)",
174
- "correct": false
175
- },
176
- {
177
- "q": "DNA stands for?",
178
- "response": "A",
179
- "correct": true
180
- },
181
- {
182
- "q": "Primary purpose of a contract?",
183
- "response": "B",
184
- "correct": true
185
- },
186
- {
187
- "q": "Which US amendment protects free sp",
188
- "response": "A",
189
- "correct": true
190
- },
191
- {
192
- "q": "Car accelerates from rest at 2 m/s²",
193
- "response": "B)",
194
- "correct": false
195
- },
196
- {
197
- "q": "Unit of force in SI?",
198
- "response": "C",
199
- "correct": true
200
- },
201
- {
202
- "q": "If gas temp increases at constant v",
203
- "response": "B",
204
- "correct": true
205
- },
206
- {
207
- "q": "What is π to two decimal places?",
208
- "response": "A",
209
- "correct": true
210
- }
211
- ],
212
- "quality_detail": [
213
- {
214
- "prompt": "If John has 5 apples and gives",
215
- "response": "To find out how many apples Jo",
216
- "score": 10
217
- },
218
- {
219
- "prompt": "Write a Python function that c",
220
- "response": "**Palindrome Checker Function*",
221
- "score": 10
222
- },
223
- {
224
- "prompt": "Compare supervised vs unsuperv",
225
- "response": "Supervised learning involves t",
226
- "score": 10
227
- },
228
- {
229
- "prompt": "What is the capital of France?",
230
- "response": "The capital of France is Paris",
231
- "score": 10
232
- },
233
- {
234
- "prompt": "Train travels 300km in 4 hours",
235
- "response": "To find the average speed, you",
236
- "score": 10
237
- },
238
- {
239
- "prompt": "All cats are animals. Some ani",
240
- "response": "A classic example of a syllogi",
241
- "score": 5
242
- },
243
- {
244
- "prompt": "What is machine learning?",
245
- "response": "**Machine Learning Overview**\n",
246
- "score": 10
247
- },
248
- {
249
- "prompt": "Write a haiku about programmin",
250
- "response": "Code whispers sweet\nLogic's ge",
251
- "score": 10
252
- }
253
- ]
23
+ "premium": {
24
+ "correct": 18,
25
+ "total": 40
26
+ }
27
+ },
28
+ "confusion": {
29
+ "free": {
30
+ "free": 46,
31
+ "cheap": 4,
32
+ "mid": 0,
33
+ "premium": 0
254
34
  },
255
- "groq-allam-2-7b": {
256
- "name": "Groq Allam 2 7B",
257
- "mmlu_accuracy": 0.8666666666666667,
258
- "quality_score": 9.4,
259
- "mmlu_detail": [
260
- {
261
- "q": "What is the derivative of x²?",
262
- "response": "C) 2 ",
263
- "correct": false
264
- },
265
- {
266
- "q": "Solve for x: 2x + 6 = 14",
267
- "response": "A) 4 ",
268
- "correct": true
269
- },
270
- {
271
- "q": "Time complexity of binary search?",
272
- "response": "B) O(log n) ",
273
- "correct": true
274
- },
275
- {
276
- "q": "In Python, list.append() does what?",
277
- "response": "B) Adds to end ",
278
- "correct": true
279
- },
280
- {
281
- "q": "Output of type([])?",
282
- "response": "A) <class 'array'> ",
283
- "correct": true
284
- },
285
- {
286
- "q": "GDP stands for?",
287
- "response": "A) Gross Domestic Pr",
288
- "correct": true
289
- },
290
- {
291
- "q": "If supply increases and demand stay",
292
- "response": "B) Decreases ",
293
- "correct": true
294
- },
295
- {
296
- "q": "Powerhouse of the cell?",
297
- "response": "A) Nucleus ",
298
- "correct": false
299
- },
300
- {
301
- "q": "DNA stands for?",
302
- "response": "A) Deoxyribonucleic ",
303
- "correct": true
304
- },
305
- {
306
- "q": "Primary purpose of a contract?",
307
- "response": "B) Create legal obli",
308
- "correct": true
309
- },
310
- {
311
- "q": "Which US amendment protects free sp",
312
- "response": "A) 1st ",
313
- "correct": true
314
- },
315
- {
316
- "q": "Car accelerates from rest at 2 m/s²",
317
- "response": "B) 10 m/s ",
318
- "correct": true
319
- },
320
- {
321
- "q": "Unit of force in SI?",
322
- "response": "C) Newton ",
323
- "correct": true
324
- },
325
- {
326
- "q": "If gas temp increases at constant v",
327
- "response": "B) Increases ",
328
- "correct": true
329
- },
330
- {
331
- "q": "What is π to two decimal places?",
332
- "response": "A) 3.14 ",
333
- "correct": true
334
- }
335
- ],
336
- "quality_detail": [
337
- {
338
- "prompt": "If John has 5 apples and gives",
339
- "response": "John starts with 5 apples. If ",
340
- "score": 10
341
- },
342
- {
343
- "prompt": "Write a Python function that c",
344
- "response": "Here's a simple Python functio",
345
- "score": 10
346
- },
347
- {
348
- "prompt": "Compare supervised vs unsuperv",
349
- "response": "Supervised learning involves t",
350
- "score": 10
351
- },
352
- {
353
- "prompt": "What is the capital of France?",
354
- "response": "The capital of France is Paris",
355
- "score": 10
356
- },
357
- {
358
- "prompt": "Train travels 300km in 4 hours",
359
- "response": "To calculate the average speed",
360
- "score": 10
361
- },
362
- {
363
- "prompt": "All cats are animals. Some ani",
364
- "response": "Yes, based on the information ",
365
- "score": 5
366
- },
367
- {
368
- "prompt": "What is machine learning?",
369
- "response": "Machine learning is a branch o",
370
- "score": 10
371
- },
372
- {
373
- "prompt": "Write a haiku about programmin",
374
- "response": "Coding lines of code\nA world o",
375
- "score": 10
376
- }
377
- ]
35
+ "cheap": {
36
+ "free": 11,
37
+ "cheap": 47,
38
+ "mid": 2,
39
+ "premium": 0
378
40
  },
379
- "cerebras-llama3.1-8b": {
380
- "name": "Cerebras Llama 3.1 8B",
381
- "mmlu_accuracy": 0.3333333333333333,
382
- "quality_score": 0.6,
383
- "mmlu_detail": [
384
- {
385
- "q": "What is the derivative of x²?",
386
- "response": "B",
387
- "correct": true
388
- },
389
- {
390
- "q": "Solve for x: 2x + 6 = 14",
391
- "response": "A",
392
- "correct": true
393
- },
394
- {
395
- "q": "Time complexity of binary search?",
396
- "response": "B",
397
- "correct": true
398
- },
399
- {
400
- "q": "In Python, list.append() does what?",
401
- "response": "B",
402
- "correct": true
403
- },
404
- {
405
- "q": "Output of type([])?",
406
- "response": "B",
407
- "correct": true
408
- },
409
- {
410
- "q": "GDP stands for?",
411
- "response": "",
412
- "correct": false
413
- },
414
- {
415
- "q": "If supply increases and demand stay",
416
- "response": "",
417
- "correct": false
418
- },
419
- {
420
- "q": "Powerhouse of the cell?",
421
- "response": "",
422
- "correct": false
423
- },
424
- {
425
- "q": "DNA stands for?",
426
- "response": "",
427
- "correct": false
428
- },
429
- {
430
- "q": "Primary purpose of a contract?",
431
- "response": "",
432
- "correct": false
433
- },
434
- {
435
- "q": "Which US amendment protects free sp",
436
- "response": "",
437
- "correct": false
438
- },
439
- {
440
- "q": "Car accelerates from rest at 2 m/s²",
441
- "response": "",
442
- "correct": false
443
- },
444
- {
445
- "q": "Unit of force in SI?",
446
- "response": "",
447
- "correct": false
448
- },
449
- {
450
- "q": "If gas temp increases at constant v",
451
- "response": "",
452
- "correct": false
453
- },
454
- {
455
- "q": "What is π to two decimal places?",
456
- "response": "",
457
- "correct": false
458
- }
459
- ],
460
- "quality_detail": [
461
- {
462
- "prompt": "If John has 5 apples and gives",
463
- "response": "",
464
- "score": 0
465
- },
466
- {
467
- "prompt": "Write a Python function that c",
468
- "response": "",
469
- "score": 0
470
- },
471
- {
472
- "prompt": "Compare supervised vs unsuperv",
473
- "response": "",
474
- "score": 0
475
- },
476
- {
477
- "prompt": "What is the capital of France?",
478
- "response": "",
479
- "score": 0
480
- },
481
- {
482
- "prompt": "Train travels 300km in 4 hours",
483
- "response": "",
484
- "score": 0
485
- },
486
- {
487
- "prompt": "All cats are animals. Some ani",
488
- "response": "To determine if some cats are ",
489
- "score": 5
490
- },
491
- {
492
- "prompt": "What is machine learning?",
493
- "response": "",
494
- "score": 0
495
- },
496
- {
497
- "prompt": "Write a haiku about programmin",
498
- "response": "",
499
- "score": 0
500
- }
501
- ]
41
+ "mid": {
42
+ "free": 0,
43
+ "cheap": 24,
44
+ "mid": 18,
45
+ "premium": 8
502
46
  },
503
- "cerebras-qwen-3-235b": {
504
- "name": "Cerebras Qwen 3 235B",
505
- "mmlu_accuracy": 0.4,
506
- "quality_score": 0,
507
- "mmlu_detail": [
508
- {
509
- "q": "What is the derivative of x²?",
510
- "response": "B) 2x",
511
- "correct": true
512
- },
513
- {
514
- "q": "Solve for x: 2x + 6 = 14",
515
- "response": "A) 4\n\nSolution:\n2x +",
516
- "correct": true
517
- },
518
- {
519
- "q": "Time complexity of binary search?",
520
- "response": "B) O(log n)",
521
- "correct": true
522
- },
523
- {
524
- "q": "In Python, list.append() does what?",
525
- "response": "B) Adds to end",
526
- "correct": true
527
- },
528
- {
529
- "q": "Output of type([])?",
530
- "response": "B) <class 'list'>",
531
- "correct": true
532
- },
533
- {
534
- "q": "GDP stands for?",
535
- "response": "",
536
- "correct": false
537
- },
538
- {
539
- "q": "If supply increases and demand stay",
540
- "response": "",
541
- "correct": false
542
- },
543
- {
544
- "q": "Powerhouse of the cell?",
545
- "response": "C",
546
- "correct": true
547
- },
548
- {
549
- "q": "DNA stands for?",
550
- "response": "",
551
- "correct": false
552
- },
553
- {
554
- "q": "Primary purpose of a contract?",
555
- "response": "",
556
- "correct": false
557
- },
558
- {
559
- "q": "Which US amendment protects free sp",
560
- "response": "",
561
- "correct": false
562
- },
563
- {
564
- "q": "Car accelerates from rest at 2 m/s²",
565
- "response": "",
566
- "correct": false
567
- },
568
- {
569
- "q": "Unit of force in SI?",
570
- "response": "",
571
- "correct": false
572
- },
573
- {
574
- "q": "If gas temp increases at constant v",
575
- "response": "",
576
- "correct": false
577
- },
578
- {
579
- "q": "What is π to two decimal places?",
580
- "response": "",
581
- "correct": false
582
- }
583
- ],
584
- "quality_detail": [
585
- {
586
- "prompt": "If John has 5 apples and gives",
587
- "response": "",
588
- "score": 0
589
- },
590
- {
591
- "prompt": "Write a Python function that c",
592
- "response": "",
593
- "score": 0
594
- },
595
- {
596
- "prompt": "Compare supervised vs unsuperv",
597
- "response": "",
598
- "score": 0
599
- },
600
- {
601
- "prompt": "What is the capital of France?",
602
- "response": "",
603
- "score": 0
604
- },
605
- {
606
- "prompt": "Train travels 300km in 4 hours",
607
- "response": "",
608
- "score": 0
609
- },
610
- {
611
- "prompt": "All cats are animals. Some ani",
612
- "response": "",
613
- "score": 0
614
- },
615
- {
616
- "prompt": "What is machine learning?",
617
- "response": "",
618
- "score": 0
619
- },
620
- {
621
- "prompt": "Write a haiku about programmin",
622
- "response": "",
623
- "score": 0
624
- }
625
- ]
47
+ "premium": {
48
+ "free": 0,
49
+ "cheap": 1,
50
+ "mid": 21,
51
+ "premium": 18
626
52
  }
627
53
  }
628
54
  }
package/dist/tui/index.js CHANGED
File without changes