claude-autorouter 0.3.6 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.env.example +12 -7
  2. package/CONTRIBUTING.md +37 -0
  3. package/README.md +43 -70
  4. package/bin/autorouter.mjs +40 -56
  5. package/docs/development.md +50 -2
  6. package/docs/hardware-benchmark.md +29 -0
  7. package/docs/hardware-comparison.md +55 -0
  8. package/docs/hardware-results-16gb.json +4002 -0
  9. package/docs/hardware-results-16gb.md +26 -0
  10. package/docs/hardware-results-64gb.json +4020 -0
  11. package/docs/reference.md +83 -40
  12. package/docs/releasing.md +76 -34
  13. package/docs/router-performance.json +1697 -0
  14. package/docs/router-performance.md +50 -0
  15. package/docs/status-performance.json +363 -0
  16. package/docs/status-performance.md +44 -0
  17. package/package.json +57 -9
  18. package/src/auto-routing.mjs +214 -0
  19. package/src/bounded-json.mjs +57 -0
  20. package/src/cli-help.mjs +87 -0
  21. package/src/config-command.mjs +141 -0
  22. package/src/config.mjs +52 -27
  23. package/src/contracts.mjs +123 -0
  24. package/src/evaluation-report.mjs +114 -0
  25. package/src/local-diagnostic.mjs +191 -0
  26. package/src/model-catalog.mjs +96 -0
  27. package/src/model-request.mjs +10 -6
  28. package/src/ollama-evaluator.mjs +9 -27
  29. package/src/onboarding.mjs +82 -23
  30. package/src/request-validation.mjs +54 -0
  31. package/src/response-observer.mjs +126 -18
  32. package/src/router.mjs +174 -70
  33. package/src/savings.mjs +74 -16
  34. package/src/server.mjs +79 -12
  35. package/src/session-history.mjs +261 -0
  36. package/src/session-log.mjs +9 -58
  37. package/src/status-state.mjs +110 -62
  38. package/src/statusline.mjs +57 -27
  39. package/src/telemetry-event.mjs +196 -0
  40. package/src/token-counter.mjs +3 -1
  41. package/src/turn-state.mjs +132 -0
  42. package/src/user-config.mjs +18 -8
@@ -0,0 +1,1697 @@
1
+ {
2
+ "baseline": {
3
+ "schema_version": 1,
4
+ "type": "synthetic_router_benchmark",
5
+ "timestamp": "2026-10-05T08:14:55.682Z",
6
+ "inputs": {
7
+ "iterations": 60,
8
+ "rounds": 3,
9
+ "concurrency": 8,
10
+ "evaluator_delay_ms": 1,
11
+ "large_request_bytes": 286807,
12
+ "real_provider_calls": 0,
13
+ "gc_exposed": true
14
+ },
15
+ "environment": {
16
+ "node": "v26.10.0",
17
+ "platform": "darwin",
18
+ "os_release": "25.6.0",
19
+ "arch": "arm64",
20
+ "cpu": "Apple M4",
21
+ "logical_cpus": 10,
22
+ "total_memory_bytes": 17179869184,
23
+ "free_memory_bytes": 172720128,
24
+ "load_average": [
25
+ 2.318359375,
26
+ 2.76708984375,
27
+ 2.97265625
28
+ ],
29
+ "background_processes": "Uncontrolled developer workstation; no user process names or arguments collected."
30
+ },
31
+ "results": [
32
+ {
33
+ "scenario": "small",
34
+ "round": 1,
35
+ "requests": 60,
36
+ "setup_evaluator_calls": 0,
37
+ "evaluator_calls": 60,
38
+ "aborted_evaluations": 0,
39
+ "route_ms": {
40
+ "samples": 60,
41
+ "p50": 1.2013340000000312,
42
+ "p95": 1.3706669999999974,
43
+ "max": 43.80841699999999
44
+ },
45
+ "policy_overhead_ms": {
46
+ "samples": 60,
47
+ "p50": 0.010000000000000009,
48
+ "p95": 0.030000000000000027,
49
+ "max": 0.3999999999999986
50
+ },
51
+ "memory_bytes": {
52
+ "retained_heap_delta": 2952656,
53
+ "retained_rss_delta": 7193464,
54
+ "sampled_peak_heap_delta": 4280064,
55
+ "sampled_peak_rss_delta": 6980424
56
+ },
57
+ "event_loop_delay_ms": {
58
+ "p50": 10.133503,
59
+ "p95": 45.383679,
60
+ "max": 45.383679
61
+ }
62
+ },
63
+ {
64
+ "scenario": "large_tools",
65
+ "round": 1,
66
+ "requests": 60,
67
+ "setup_evaluator_calls": 0,
68
+ "evaluator_calls": 60,
69
+ "aborted_evaluations": 0,
70
+ "route_ms": {
71
+ "samples": 60,
72
+ "p50": 2.29362500000002,
73
+ "p95": 2.9194590000000176,
74
+ "max": 4.994334000000038
75
+ },
76
+ "policy_overhead_ms": {
77
+ "samples": 60,
78
+ "p50": 0.71,
79
+ "p95": 0.9299999999999999,
80
+ "max": 3.5200000000000005
81
+ },
82
+ "memory_bytes": {
83
+ "retained_heap_delta": 247424,
84
+ "retained_rss_delta": 6406792,
85
+ "sampled_peak_heap_delta": 4686184,
86
+ "sampled_peak_rss_delta": 6390408
87
+ },
88
+ "event_loop_delay_ms": {
89
+ "p50": 10.371071,
90
+ "p95": 10.862591,
91
+ "max": 11.288575
92
+ }
93
+ },
94
+ {
95
+ "scenario": "cache",
96
+ "round": 1,
97
+ "requests": 60,
98
+ "setup_evaluator_calls": 1,
99
+ "evaluator_calls": 0,
100
+ "aborted_evaluations": 0,
101
+ "route_ms": {
102
+ "samples": 60,
103
+ "p50": 0.005375000000015007,
104
+ "p95": 0.011459000000002106,
105
+ "max": 0.04337500000002592
106
+ },
107
+ "policy_overhead_ms": {
108
+ "samples": 60,
109
+ "p50": 0,
110
+ "p95": 0.01,
111
+ "max": 0.02
112
+ },
113
+ "memory_bytes": {
114
+ "retained_heap_delta": -32880,
115
+ "retained_rss_delta": 16384,
116
+ "sampled_peak_heap_delta": 460616,
117
+ "sampled_peak_rss_delta": 16384
118
+ },
119
+ "event_loop_delay_ms": {
120
+ "p50": 10.739711,
121
+ "p95": 10.739711,
122
+ "max": 10.739711
123
+ }
124
+ },
125
+ {
126
+ "scenario": "pinned",
127
+ "round": 1,
128
+ "requests": 60,
129
+ "setup_evaluator_calls": 1,
130
+ "evaluator_calls": 60,
131
+ "aborted_evaluations": 0,
132
+ "route_ms": {
133
+ "samples": 60,
134
+ "p50": 1.2022080000000415,
135
+ "p95": 1.7778749999999945,
136
+ "max": 2.263667000000055
137
+ },
138
+ "policy_overhead_ms": {
139
+ "samples": 60,
140
+ "p50": 0.010000000000000009,
141
+ "p95": 0.09000000000000008,
142
+ "max": 0.41999999999999993
143
+ },
144
+ "memory_bytes": {
145
+ "retained_heap_delta": 69424,
146
+ "retained_rss_delta": 245760,
147
+ "sampled_peak_heap_delta": 1873648,
148
+ "sampled_peak_rss_delta": 212992
149
+ },
150
+ "event_loop_delay_ms": {
151
+ "p50": 9.961471,
152
+ "p95": 10.674175,
153
+ "max": 10.674175
154
+ }
155
+ },
156
+ {
157
+ "scenario": "concurrent_identical",
158
+ "round": 1,
159
+ "requests": 480,
160
+ "setup_evaluator_calls": 0,
161
+ "evaluator_calls": 480,
162
+ "aborted_evaluations": 0,
163
+ "route_ms": {
164
+ "samples": 480,
165
+ "p50": 1.8419999999999845,
166
+ "p95": 2.4129999999998972,
167
+ "max": 3.305082999999968
168
+ },
169
+ "policy_overhead_ms": {
170
+ "samples": 480,
171
+ "p50": 0.09000000000000008,
172
+ "p95": 0.13000000000000034,
173
+ "max": 0.1900000000000004
174
+ },
175
+ "memory_bytes": {
176
+ "retained_heap_delta": 734344,
177
+ "retained_rss_delta": 7685152,
178
+ "sampled_peak_heap_delta": 5254592,
179
+ "sampled_peak_rss_delta": 7603160
180
+ },
181
+ "event_loop_delay_ms": {
182
+ "p50": 10.387455,
183
+ "p95": 10.887167,
184
+ "max": 11.100159
185
+ }
186
+ },
187
+ {
188
+ "scenario": "concurrent_identical_agents",
189
+ "round": 1,
190
+ "requests": 480,
191
+ "setup_evaluator_calls": 0,
192
+ "evaluator_calls": 480,
193
+ "aborted_evaluations": 0,
194
+ "route_ms": {
195
+ "samples": 480,
196
+ "p50": 1.714749999999981,
197
+ "p95": 2.219333000000006,
198
+ "max": 2.6454579999999623
199
+ },
200
+ "policy_overhead_ms": {
201
+ "samples": 480,
202
+ "p50": 0.050000000000000044,
203
+ "p95": 0.1100000000000001,
204
+ "max": 0.17000000000000015
205
+ },
206
+ "memory_bytes": {
207
+ "retained_heap_delta": 73024,
208
+ "retained_rss_delta": 671744,
209
+ "sampled_peak_heap_delta": 4173792,
210
+ "sampled_peak_rss_delta": 671744
211
+ },
212
+ "event_loop_delay_ms": {
213
+ "p50": 10.174463,
214
+ "p95": 10.551295,
215
+ "max": 10.936319
216
+ }
217
+ },
218
+ {
219
+ "scenario": "concurrent_distinct_agents",
220
+ "round": 1,
221
+ "requests": 480,
222
+ "setup_evaluator_calls": 0,
223
+ "evaluator_calls": 480,
224
+ "aborted_evaluations": 0,
225
+ "route_ms": {
226
+ "samples": 480,
227
+ "p50": 1.4834169999999176,
228
+ "p95": 2.056874999999991,
229
+ "max": 2.7068340000000717
230
+ },
231
+ "policy_overhead_ms": {
232
+ "samples": 480,
233
+ "p50": 0.030000000000000027,
234
+ "p95": 0.06000000000000005,
235
+ "max": 0.16999999999999993
236
+ },
237
+ "memory_bytes": {
238
+ "retained_heap_delta": 102008,
239
+ "retained_rss_delta": 360448,
240
+ "sampled_peak_heap_delta": 4235680,
241
+ "sampled_peak_rss_delta": 360448
242
+ },
243
+ "event_loop_delay_ms": {
244
+ "p50": 10.346495,
245
+ "p95": 10.641407,
246
+ "max": 10.690559
247
+ }
248
+ },
249
+ {
250
+ "scenario": "cancellation",
251
+ "round": 1,
252
+ "requests": 60,
253
+ "setup_evaluator_calls": 0,
254
+ "evaluator_calls": 60,
255
+ "aborted_evaluations": 60,
256
+ "route_ms": {
257
+ "samples": 60,
258
+ "p50": 0.0441249999998945,
259
+ "p95": 0.12341700000001765,
260
+ "max": 0.5816250000000309
261
+ },
262
+ "policy_overhead_ms": {
263
+ "samples": 0,
264
+ "p50": 0,
265
+ "p95": 0,
266
+ "max": 0
267
+ },
268
+ "memory_bytes": {
269
+ "retained_heap_delta": -472320,
270
+ "retained_rss_delta": 49152,
271
+ "sampled_peak_heap_delta": 1005152,
272
+ "sampled_peak_rss_delta": 49152
273
+ },
274
+ "event_loop_delay_ms": {
275
+ "p50": 9.543679,
276
+ "p95": 9.543679,
277
+ "max": 9.543679
278
+ }
279
+ },
280
+ {
281
+ "scenario": "small",
282
+ "round": 2,
283
+ "requests": 60,
284
+ "setup_evaluator_calls": 0,
285
+ "evaluator_calls": 60,
286
+ "aborted_evaluations": 0,
287
+ "route_ms": {
288
+ "samples": 60,
289
+ "p50": 1.331083000000035,
290
+ "p95": 1.6460839999999735,
291
+ "max": 4.464750000000095
292
+ },
293
+ "policy_overhead_ms": {
294
+ "samples": 60,
295
+ "p50": 0.050000000000000044,
296
+ "p95": 0.1100000000000001,
297
+ "max": 0.13999999999999968
298
+ },
299
+ "memory_bytes": {
300
+ "retained_heap_delta": 51024,
301
+ "retained_rss_delta": 0,
302
+ "sampled_peak_heap_delta": 1714184,
303
+ "sampled_peak_rss_delta": 0
304
+ },
305
+ "event_loop_delay_ms": {
306
+ "p50": 10.207231,
307
+ "p95": 10.862591,
308
+ "max": 10.862591
309
+ }
310
+ },
311
+ {
312
+ "scenario": "large_tools",
313
+ "round": 2,
314
+ "requests": 60,
315
+ "setup_evaluator_calls": 0,
316
+ "evaluator_calls": 60,
317
+ "aborted_evaluations": 0,
318
+ "route_ms": {
319
+ "samples": 60,
320
+ "p50": 3.4210830000001806,
321
+ "p95": 4.365916999999854,
322
+ "max": 4.819874999999911
323
+ },
324
+ "policy_overhead_ms": {
325
+ "samples": 60,
326
+ "p50": 1.3400000000000003,
327
+ "p95": 1.7999999999999998,
328
+ "max": 2.1499999999999995
329
+ },
330
+ "memory_bytes": {
331
+ "retained_heap_delta": -96800,
332
+ "retained_rss_delta": 5587640,
333
+ "sampled_peak_heap_delta": 8902728,
334
+ "sampled_peak_rss_delta": 5587640
335
+ },
336
+ "event_loop_delay_ms": {
337
+ "p50": 10.641407,
338
+ "p95": 11.993087,
339
+ "max": 12.075007
340
+ }
341
+ },
342
+ {
343
+ "scenario": "cache",
344
+ "round": 2,
345
+ "requests": 60,
346
+ "setup_evaluator_calls": 1,
347
+ "evaluator_calls": 0,
348
+ "aborted_evaluations": 0,
349
+ "route_ms": {
350
+ "samples": 60,
351
+ "p50": 0.025208000000020547,
352
+ "p95": 0.03533300000003692,
353
+ "max": 0.18183300000009694
354
+ },
355
+ "policy_overhead_ms": {
356
+ "samples": 60,
357
+ "p50": 0.01,
358
+ "p95": 0.019999999999999997,
359
+ "max": 0.08000000000000002
360
+ },
361
+ "memory_bytes": {
362
+ "retained_heap_delta": -2336,
363
+ "retained_rss_delta": 0,
364
+ "sampled_peak_heap_delta": 456432,
365
+ "sampled_peak_rss_delta": 0
366
+ },
367
+ "event_loop_delay_ms": {
368
+ "p50": 10.027007,
369
+ "p95": 10.027007,
370
+ "max": 10.027007
371
+ }
372
+ },
373
+ {
374
+ "scenario": "pinned",
375
+ "round": 2,
376
+ "requests": 60,
377
+ "setup_evaluator_calls": 1,
378
+ "evaluator_calls": 60,
379
+ "aborted_evaluations": 0,
380
+ "route_ms": {
381
+ "samples": 60,
382
+ "p50": 1.34699999999998,
383
+ "p95": 1.5060839999998734,
384
+ "max": 2.033583000000135
385
+ },
386
+ "policy_overhead_ms": {
387
+ "samples": 60,
388
+ "p50": 0.050000000000000044,
389
+ "p95": 0.09000000000000008,
390
+ "max": 0.1100000000000001
391
+ },
392
+ "memory_bytes": {
393
+ "retained_heap_delta": 46928,
394
+ "retained_rss_delta": 16384,
395
+ "sampled_peak_heap_delta": 1783344,
396
+ "sampled_peak_rss_delta": 16384
397
+ },
398
+ "event_loop_delay_ms": {
399
+ "p50": 10.043391,
400
+ "p95": 10.641407,
401
+ "max": 10.641407
402
+ }
403
+ },
404
+ {
405
+ "scenario": "concurrent_identical",
406
+ "round": 2,
407
+ "requests": 480,
408
+ "setup_evaluator_calls": 0,
409
+ "evaluator_calls": 480,
410
+ "aborted_evaluations": 0,
411
+ "route_ms": {
412
+ "samples": 480,
413
+ "p50": 1.8821669999999813,
414
+ "p95": 2.8152919999999995,
415
+ "max": 10.332791999999927
416
+ },
417
+ "policy_overhead_ms": {
418
+ "samples": 480,
419
+ "p50": 0.08999999999999986,
420
+ "p95": 0.16999999999999993,
421
+ "max": 5.59
422
+ },
423
+ "memory_bytes": {
424
+ "retained_heap_delta": 447184,
425
+ "retained_rss_delta": 6243144,
426
+ "sampled_peak_heap_delta": 7489936,
427
+ "sampled_peak_rss_delta": 6243144
428
+ },
429
+ "event_loop_delay_ms": {
430
+ "p50": 10.321919,
431
+ "p95": 10.928127,
432
+ "max": 15.958015
433
+ }
434
+ },
435
+ {
436
+ "scenario": "concurrent_identical_agents",
437
+ "round": 2,
438
+ "requests": 480,
439
+ "setup_evaluator_calls": 0,
440
+ "evaluator_calls": 480,
441
+ "aborted_evaluations": 0,
442
+ "route_ms": {
443
+ "samples": 480,
444
+ "p50": 1.5882910000000265,
445
+ "p95": 1.9520420000001195,
446
+ "max": 2.251166999999896
447
+ },
448
+ "policy_overhead_ms": {
449
+ "samples": 480,
450
+ "p50": 0.040000000000000036,
451
+ "p95": 0.07000000000000006,
452
+ "max": 0.2100000000000002
453
+ },
454
+ "memory_bytes": {
455
+ "retained_heap_delta": 73456,
456
+ "retained_rss_delta": 294960,
457
+ "sampled_peak_heap_delta": 7093736,
458
+ "sampled_peak_rss_delta": 245808
459
+ },
460
+ "event_loop_delay_ms": {
461
+ "p50": 10.174463,
462
+ "p95": 10.862591,
463
+ "max": 11.001855
464
+ }
465
+ },
466
+ {
467
+ "scenario": "concurrent_distinct_agents",
468
+ "round": 2,
469
+ "requests": 480,
470
+ "setup_evaluator_calls": 0,
471
+ "evaluator_calls": 480,
472
+ "aborted_evaluations": 0,
473
+ "route_ms": {
474
+ "samples": 480,
475
+ "p50": 1.5821670000000267,
476
+ "p95": 1.950749999999971,
477
+ "max": 2.6000830000000406
478
+ },
479
+ "policy_overhead_ms": {
480
+ "samples": 480,
481
+ "p50": 0.040000000000000036,
482
+ "p95": 0.07000000000000006,
483
+ "max": 0.19999999999999996
484
+ },
485
+ "memory_bytes": {
486
+ "retained_heap_delta": 84632,
487
+ "retained_rss_delta": 409648,
488
+ "sampled_peak_heap_delta": 7152560,
489
+ "sampled_peak_rss_delta": 294960
490
+ },
491
+ "event_loop_delay_ms": {
492
+ "p50": 10.076159,
493
+ "p95": 10.559487,
494
+ "max": 10.674175
495
+ }
496
+ },
497
+ {
498
+ "scenario": "cancellation",
499
+ "round": 2,
500
+ "requests": 60,
501
+ "setup_evaluator_calls": 0,
502
+ "evaluator_calls": 60,
503
+ "aborted_evaluations": 60,
504
+ "route_ms": {
505
+ "samples": 60,
506
+ "p50": 0.03304100000013932,
507
+ "p95": 0.07345800000030067,
508
+ "max": 0.3361670000003869
509
+ },
510
+ "policy_overhead_ms": {
511
+ "samples": 0,
512
+ "p50": 0,
513
+ "p95": 0,
514
+ "max": 0
515
+ },
516
+ "memory_bytes": {
517
+ "retained_heap_delta": -416368,
518
+ "retained_rss_delta": 0,
519
+ "sampled_peak_heap_delta": 912392,
520
+ "sampled_peak_rss_delta": 0
521
+ },
522
+ "event_loop_delay_ms": {
523
+ "p50": 10.960895,
524
+ "p95": 10.960895,
525
+ "max": 10.960895
526
+ }
527
+ },
528
+ {
529
+ "scenario": "small",
530
+ "round": 3,
531
+ "requests": 60,
532
+ "setup_evaluator_calls": 0,
533
+ "evaluator_calls": 60,
534
+ "aborted_evaluations": 0,
535
+ "route_ms": {
536
+ "samples": 60,
537
+ "p50": 1.3212499999999636,
538
+ "p95": 1.4821249999999964,
539
+ "max": 1.570416000000023
540
+ },
541
+ "policy_overhead_ms": {
542
+ "samples": 60,
543
+ "p50": 0.050000000000000044,
544
+ "p95": 0.08999999999999986,
545
+ "max": 0.1100000000000001
546
+ },
547
+ "memory_bytes": {
548
+ "retained_heap_delta": 48496,
549
+ "retained_rss_delta": 0,
550
+ "sampled_peak_heap_delta": 1595328,
551
+ "sampled_peak_rss_delta": 0
552
+ },
553
+ "event_loop_delay_ms": {
554
+ "p50": 9.797631,
555
+ "p95": 10.846207,
556
+ "max": 10.846207
557
+ }
558
+ },
559
+ {
560
+ "scenario": "large_tools",
561
+ "round": 3,
562
+ "requests": 60,
563
+ "setup_evaluator_calls": 0,
564
+ "evaluator_calls": 60,
565
+ "aborted_evaluations": 0,
566
+ "route_ms": {
567
+ "samples": 60,
568
+ "p50": 3.0105419999999867,
569
+ "p95": 4.4152500000000146,
570
+ "max": 4.534333000000061
571
+ },
572
+ "policy_overhead_ms": {
573
+ "samples": 60,
574
+ "p50": 1.0999999999999999,
575
+ "p95": 1.83,
576
+ "max": 2.06
577
+ },
578
+ "memory_bytes": {
579
+ "retained_heap_delta": -76264,
580
+ "retained_rss_delta": 933888,
581
+ "sampled_peak_heap_delta": 8876456,
582
+ "sampled_peak_rss_delta": 933888
583
+ },
584
+ "event_loop_delay_ms": {
585
+ "p50": 10.264575,
586
+ "p95": 12.009471,
587
+ "max": 12.353535
588
+ }
589
+ },
590
+ {
591
+ "scenario": "cache",
592
+ "round": 3,
593
+ "requests": 60,
594
+ "setup_evaluator_calls": 1,
595
+ "evaluator_calls": 0,
596
+ "aborted_evaluations": 0,
597
+ "route_ms": {
598
+ "samples": 60,
599
+ "p50": 0.017832999999882304,
600
+ "p95": 0.030040999999982887,
601
+ "max": 0.1517079999998714
602
+ },
603
+ "policy_overhead_ms": {
604
+ "samples": 60,
605
+ "p50": 0.01,
606
+ "p95": 0.02,
607
+ "max": 0.07999999999999999
608
+ },
609
+ "memory_bytes": {
610
+ "retained_heap_delta": 848,
611
+ "retained_rss_delta": 0,
612
+ "sampled_peak_heap_delta": 439432,
613
+ "sampled_peak_rss_delta": 0
614
+ },
615
+ "event_loop_delay_ms": {
616
+ "p50": 9.789439,
617
+ "p95": 9.789439,
618
+ "max": 9.789439
619
+ }
620
+ },
621
+ {
622
+ "scenario": "pinned",
623
+ "round": 3,
624
+ "requests": 60,
625
+ "setup_evaluator_calls": 1,
626
+ "evaluator_calls": 60,
627
+ "aborted_evaluations": 0,
628
+ "route_ms": {
629
+ "samples": 60,
630
+ "p50": 1.3730000000000473,
631
+ "p95": 1.6894589999997152,
632
+ "max": 1.8885000000000218
633
+ },
634
+ "policy_overhead_ms": {
635
+ "samples": 60,
636
+ "p50": 0.050000000000000044,
637
+ "p95": 0.14000000000000012,
638
+ "max": 0.2200000000000002
639
+ },
640
+ "memory_bytes": {
641
+ "retained_heap_delta": 12992,
642
+ "retained_rss_delta": 0,
643
+ "sampled_peak_heap_delta": 1828792,
644
+ "sampled_peak_rss_delta": 0
645
+ },
646
+ "event_loop_delay_ms": {
647
+ "p50": 9.814015,
648
+ "p95": 10.895359,
649
+ "max": 10.895359
650
+ }
651
+ },
652
+ {
653
+ "scenario": "concurrent_identical",
654
+ "round": 3,
655
+ "requests": 480,
656
+ "setup_evaluator_calls": 0,
657
+ "evaluator_calls": 480,
658
+ "aborted_evaluations": 0,
659
+ "route_ms": {
660
+ "samples": 480,
661
+ "p50": 1.9362920000003214,
662
+ "p95": 2.5029999999997017,
663
+ "max": 3.0052920000002814
664
+ },
665
+ "policy_overhead_ms": {
666
+ "samples": 480,
667
+ "p50": 0.09999999999999987,
668
+ "p95": 0.15000000000000013,
669
+ "max": 0.74
670
+ },
671
+ "memory_bytes": {
672
+ "retained_heap_delta": 443400,
673
+ "retained_rss_delta": 360448,
674
+ "sampled_peak_heap_delta": 7816744,
675
+ "sampled_peak_rss_delta": 344064
676
+ },
677
+ "event_loop_delay_ms": {
678
+ "p50": 10.108927,
679
+ "p95": 10.854399,
680
+ "max": 11.026431
681
+ }
682
+ },
683
+ {
684
+ "scenario": "concurrent_identical_agents",
685
+ "round": 3,
686
+ "requests": 480,
687
+ "setup_evaluator_calls": 0,
688
+ "evaluator_calls": 480,
689
+ "aborted_evaluations": 0,
690
+ "route_ms": {
691
+ "samples": 480,
692
+ "p50": 1.600666999999703,
693
+ "p95": 1.9589579999997113,
694
+ "max": 2.2774999999996908
695
+ },
696
+ "policy_overhead_ms": {
697
+ "samples": 480,
698
+ "p50": 0.040000000000000036,
699
+ "p95": 0.08000000000000007,
700
+ "max": 0.20999999999999996
701
+ },
702
+ "memory_bytes": {
703
+ "retained_heap_delta": 77912,
704
+ "retained_rss_delta": 278624,
705
+ "sampled_peak_heap_delta": 6929344,
706
+ "sampled_peak_rss_delta": 196656
707
+ },
708
+ "event_loop_delay_ms": {
709
+ "p50": 10.215423,
710
+ "p95": 10.354687,
711
+ "max": 10.412031
712
+ }
713
+ },
714
+ {
715
+ "scenario": "concurrent_distinct_agents",
716
+ "round": 3,
717
+ "requests": 480,
718
+ "setup_evaluator_calls": 0,
719
+ "evaluator_calls": 480,
720
+ "aborted_evaluations": 0,
721
+ "route_ms": {
722
+ "samples": 480,
723
+ "p50": 1.6864169999998921,
724
+ "p95": 2.2006250000004,
725
+ "max": 2.577708000000257
726
+ },
727
+ "policy_overhead_ms": {
728
+ "samples": 480,
729
+ "p50": 0.05999999999999983,
730
+ "p95": 0.10999999999999988,
731
+ "max": 0.30000000000000004
732
+ },
733
+ "memory_bytes": {
734
+ "retained_heap_delta": 73616,
735
+ "retained_rss_delta": 229376,
736
+ "sampled_peak_heap_delta": 7035312,
737
+ "sampled_peak_rss_delta": 196608
738
+ },
739
+ "event_loop_delay_ms": {
740
+ "p50": 10.371071,
741
+ "p95": 10.788863,
742
+ "max": 10.829823
743
+ }
744
+ },
745
+ {
746
+ "scenario": "cancellation",
747
+ "round": 3,
748
+ "requests": 60,
749
+ "setup_evaluator_calls": 0,
750
+ "evaluator_calls": 60,
751
+ "aborted_evaluations": 60,
752
+ "route_ms": {
753
+ "samples": 60,
754
+ "p50": 0.09712500000023283,
755
+ "p95": 0.12445900000011534,
756
+ "max": 0.5854159999998956
757
+ },
758
+ "policy_overhead_ms": {
759
+ "samples": 0,
760
+ "p50": 0,
761
+ "p95": 0,
762
+ "max": 0
763
+ },
764
+ "memory_bytes": {
765
+ "retained_heap_delta": -409000,
766
+ "retained_rss_delta": 0,
767
+ "sampled_peak_heap_delta": 889544,
768
+ "sampled_peak_rss_delta": 0
769
+ },
770
+ "event_loop_delay_ms": {
771
+ "p50": 9.461759,
772
+ "p95": 10.018815,
773
+ "max": 10.018815
774
+ }
775
+ }
776
+ ]
777
+ },
778
+ "candidate": {
779
+ "schema_version": 1,
780
+ "type": "synthetic_router_benchmark",
781
+ "timestamp": "2026-10-05T10:25:48.205Z",
782
+ "inputs": {
783
+ "iterations": 60,
784
+ "rounds": 3,
785
+ "concurrency": 8,
786
+ "evaluator_delay_ms": 1,
787
+ "large_request_bytes": 286807,
788
+ "real_provider_calls": 0,
789
+ "gc_exposed": true
790
+ },
791
+ "environment": {
792
+ "node": "v26.10.0",
793
+ "platform": "darwin",
794
+ "os_release": "25.6.0",
795
+ "arch": "arm64",
796
+ "cpu": "Apple M4",
797
+ "logical_cpus": 10,
798
+ "total_memory_bytes": 17179869184,
799
+ "free_memory_bytes": 115310592,
800
+ "load_average": [
801
+ 2.2236328125,
802
+ 3.240234375,
803
+ 3.08642578125
804
+ ],
805
+ "background_processes": "Uncontrolled developer workstation; no user process names or arguments collected."
806
+ },
807
+ "results": [
808
+ {
809
+ "scenario": "small",
810
+ "round": 1,
811
+ "requests": 60,
812
+ "setup_evaluator_calls": 0,
813
+ "evaluator_calls": 60,
814
+ "aborted_evaluations": 0,
815
+ "route_ms": {
816
+ "samples": 60,
817
+ "p50": 1.2611249999999927,
818
+ "p95": 1.3964169999999854,
819
+ "max": 39.64937500000002
820
+ },
821
+ "policy_overhead_ms": {
822
+ "samples": 60,
823
+ "p50": 0.020000000000000018,
824
+ "p95": 0.040000000000000036,
825
+ "max": 0.44999999999999574
826
+ },
827
+ "memory_bytes": {
828
+ "retained_heap_delta": 3274296,
829
+ "retained_rss_delta": 7406528,
830
+ "sampled_peak_heap_delta": 5038504,
831
+ "sampled_peak_rss_delta": 7226256
832
+ },
833
+ "event_loop_delay_ms": {
834
+ "p50": 10.125311,
835
+ "p95": 42.598399,
836
+ "max": 42.598399
837
+ }
838
+ },
839
+ {
840
+ "scenario": "large_tools",
841
+ "round": 1,
842
+ "requests": 60,
843
+ "setup_evaluator_calls": 0,
844
+ "evaluator_calls": 60,
845
+ "aborted_evaluations": 0,
846
+ "route_ms": {
847
+ "samples": 60,
848
+ "p50": 2.302166999999997,
849
+ "p95": 2.4647079999999733,
850
+ "max": 4.373499999999979
851
+ },
852
+ "policy_overhead_ms": {
853
+ "samples": 60,
854
+ "p50": 0.69,
855
+ "p95": 0.81,
856
+ "max": 2.63
857
+ },
858
+ "memory_bytes": {
859
+ "retained_heap_delta": 290632,
860
+ "retained_rss_delta": 6095520,
861
+ "sampled_peak_heap_delta": 5063944,
862
+ "sampled_peak_rss_delta": 6062752
863
+ },
864
+ "event_loop_delay_ms": {
865
+ "p50": 10.199039,
866
+ "p95": 10.592255,
867
+ "max": 10.764287
868
+ }
869
+ },
870
+ {
871
+ "scenario": "cache",
872
+ "round": 1,
873
+ "requests": 60,
874
+ "setup_evaluator_calls": 1,
875
+ "evaluator_calls": 0,
876
+ "aborted_evaluations": 0,
877
+ "route_ms": {
878
+ "samples": 60,
879
+ "p50": 0.007624999999961801,
880
+ "p95": 0.014207999999996446,
881
+ "max": 0.09195800000003373
882
+ },
883
+ "policy_overhead_ms": {
884
+ "samples": 60,
885
+ "p50": 0.01,
886
+ "p95": 0.01,
887
+ "max": 0.039999999999999994
888
+ },
889
+ "memory_bytes": {
890
+ "retained_heap_delta": -34240,
891
+ "retained_rss_delta": 16384,
892
+ "sampled_peak_heap_delta": 498888,
893
+ "sampled_peak_rss_delta": 0
894
+ },
895
+ "event_loop_delay_ms": {
896
+ "p50": 9.986047,
897
+ "p95": 9.986047,
898
+ "max": 9.986047
899
+ }
900
+ },
901
+ {
902
+ "scenario": "pinned",
903
+ "round": 1,
904
+ "requests": 60,
905
+ "setup_evaluator_calls": 1,
906
+ "evaluator_calls": 60,
907
+ "aborted_evaluations": 0,
908
+ "route_ms": {
909
+ "samples": 60,
910
+ "p50": 1.2605419999999867,
911
+ "p95": 1.317083000000025,
912
+ "max": 1.5854590000000144
913
+ },
914
+ "policy_overhead_ms": {
915
+ "samples": 60,
916
+ "p50": 0.020000000000000018,
917
+ "p95": 0.030000000000000027,
918
+ "max": 0.14000000000000012
919
+ },
920
+ "memory_bytes": {
921
+ "retained_heap_delta": 144480,
922
+ "retained_rss_delta": 753712,
923
+ "sampled_peak_heap_delta": 2411504,
924
+ "sampled_peak_rss_delta": 753712
925
+ },
926
+ "event_loop_delay_ms": {
927
+ "p50": 10.100735,
928
+ "p95": 10.231807,
929
+ "max": 10.231807
930
+ }
931
+ },
932
+ {
933
+ "scenario": "concurrent_identical",
934
+ "round": 1,
935
+ "requests": 480,
936
+ "setup_evaluator_calls": 0,
937
+ "evaluator_calls": 60,
938
+ "aborted_evaluations": 0,
939
+ "route_ms": {
940
+ "samples": 480,
941
+ "p50": 1.9149169999999458,
942
+ "p95": 2.2353749999999764,
943
+ "max": 2.3296249999999645
944
+ },
945
+ "policy_overhead_ms": {
946
+ "samples": 480,
947
+ "p50": 0.08000000000000007,
948
+ "p95": 0.10000000000000009,
949
+ "max": 0.1599999999999997
950
+ },
951
+ "memory_bytes": {
952
+ "retained_heap_delta": 441288,
953
+ "retained_rss_delta": 5292680,
954
+ "sampled_peak_heap_delta": 4129120,
955
+ "sampled_peak_rss_delta": 5259912
956
+ },
957
+ "event_loop_delay_ms": {
958
+ "p50": 9.854975,
959
+ "p95": 10.436607,
960
+ "max": 10.625023
961
+ }
962
+ },
963
+ {
964
+ "scenario": "concurrent_identical_agents",
965
+ "round": 1,
966
+ "requests": 480,
967
+ "setup_evaluator_calls": 0,
968
+ "evaluator_calls": 60,
969
+ "aborted_evaluations": 0,
970
+ "route_ms": {
971
+ "samples": 480,
972
+ "p50": 1.5662080000000742,
973
+ "p95": 1.703375000000051,
974
+ "max": 1.9496669999999767
975
+ },
976
+ "policy_overhead_ms": {
977
+ "samples": 480,
978
+ "p50": 0.040000000000000036,
979
+ "p95": 0.050000000000000044,
980
+ "max": 0.07999999999999996
981
+ },
982
+ "memory_bytes": {
983
+ "retained_heap_delta": 147784,
984
+ "retained_rss_delta": 475232,
985
+ "sampled_peak_heap_delta": 4121112,
986
+ "sampled_peak_rss_delta": 475232
987
+ },
988
+ "event_loop_delay_ms": {
989
+ "p50": 9.846783,
990
+ "p95": 10.903551,
991
+ "max": 10.903551
992
+ }
993
+ },
994
+ {
995
+ "scenario": "concurrent_distinct_agents",
996
+ "round": 1,
997
+ "requests": 480,
998
+ "setup_evaluator_calls": 0,
999
+ "evaluator_calls": 480,
1000
+ "aborted_evaluations": 0,
1001
+ "route_ms": {
1002
+ "samples": 480,
1003
+ "p50": 1.5232919999999694,
1004
+ "p95": 1.9074590000000171,
1005
+ "max": 2.1829999999999927
1006
+ },
1007
+ "policy_overhead_ms": {
1008
+ "samples": 480,
1009
+ "p50": 0.030000000000000027,
1010
+ "p95": 0.04999999999999982,
1011
+ "max": 0.08000000000000007
1012
+ },
1013
+ "memory_bytes": {
1014
+ "retained_heap_delta": 705400,
1015
+ "retained_rss_delta": 1704032,
1016
+ "sampled_peak_heap_delta": 4745576,
1017
+ "sampled_peak_rss_delta": 1589344
1018
+ },
1019
+ "event_loop_delay_ms": {
1020
+ "p50": 10.190847,
1021
+ "p95": 10.534911,
1022
+ "max": 10.780671
1023
+ }
1024
+ },
1025
+ {
1026
+ "scenario": "cancellation",
1027
+ "round": 1,
1028
+ "requests": 60,
1029
+ "setup_evaluator_calls": 0,
1030
+ "evaluator_calls": 0,
1031
+ "aborted_evaluations": 0,
1032
+ "route_ms": {
1033
+ "samples": 60,
1034
+ "p50": 0.02779200000009041,
1035
+ "p95": 0.05587499999978718,
1036
+ "max": 0.4133750000000873
1037
+ },
1038
+ "policy_overhead_ms": {
1039
+ "samples": 0,
1040
+ "p50": 0,
1041
+ "p95": 0,
1042
+ "max": 0
1043
+ },
1044
+ "memory_bytes": {
1045
+ "retained_heap_delta": -488952,
1046
+ "retained_rss_delta": 49152,
1047
+ "sampled_peak_heap_delta": 750816,
1048
+ "sampled_peak_rss_delta": 49152
1049
+ },
1050
+ "event_loop_delay_ms": {
1051
+ "p50": 10.502143,
1052
+ "p95": 10.502143,
1053
+ "max": 10.502143
1054
+ }
1055
+ },
1056
+ {
1057
+ "scenario": "small",
1058
+ "round": 2,
1059
+ "requests": 60,
1060
+ "setup_evaluator_calls": 0,
1061
+ "evaluator_calls": 60,
1062
+ "aborted_evaluations": 0,
1063
+ "route_ms": {
1064
+ "samples": 60,
1065
+ "p50": 1.2395420000000286,
1066
+ "p95": 1.2839579999999842,
1067
+ "max": 1.52537499999994
1068
+ },
1069
+ "policy_overhead_ms": {
1070
+ "samples": 60,
1071
+ "p50": 0.020000000000000018,
1072
+ "p95": 0.030000000000000027,
1073
+ "max": 0.030000000000000027
1074
+ },
1075
+ "memory_bytes": {
1076
+ "retained_heap_delta": 136336,
1077
+ "retained_rss_delta": 0,
1078
+ "sampled_peak_heap_delta": 2168568,
1079
+ "sampled_peak_rss_delta": 0
1080
+ },
1081
+ "event_loop_delay_ms": {
1082
+ "p50": 9.936895,
1083
+ "p95": 10.952703,
1084
+ "max": 10.952703
1085
+ }
1086
+ },
1087
+ {
1088
+ "scenario": "large_tools",
1089
+ "round": 2,
1090
+ "requests": 60,
1091
+ "setup_evaluator_calls": 0,
1092
+ "evaluator_calls": 60,
1093
+ "aborted_evaluations": 0,
1094
+ "route_ms": {
1095
+ "samples": 60,
1096
+ "p50": 2.769042000000127,
1097
+ "p95": 3.023249999999962,
1098
+ "max": 3.326082999999926
1099
+ },
1100
+ "policy_overhead_ms": {
1101
+ "samples": 60,
1102
+ "p50": 0.9899999999999998,
1103
+ "p95": 1.15,
1104
+ "max": 1.1900000000000002
1105
+ },
1106
+ "memory_bytes": {
1107
+ "retained_heap_delta": -68032,
1108
+ "retained_rss_delta": 5259960,
1109
+ "sampled_peak_heap_delta": 9242064,
1110
+ "sampled_peak_rss_delta": 5259960
1111
+ },
1112
+ "event_loop_delay_ms": {
1113
+ "p50": 9.986047,
1114
+ "p95": 10.969087,
1115
+ "max": 11.124735
1116
+ }
1117
+ },
1118
+ {
1119
+ "scenario": "cache",
1120
+ "round": 2,
1121
+ "requests": 60,
1122
+ "setup_evaluator_calls": 1,
1123
+ "evaluator_calls": 0,
1124
+ "aborted_evaluations": 0,
1125
+ "route_ms": {
1126
+ "samples": 60,
1127
+ "p50": 0.009708000000046013,
1128
+ "p95": 0.013500000000021828,
1129
+ "max": 0.09229200000004312
1130
+ },
1131
+ "policy_overhead_ms": {
1132
+ "samples": 60,
1133
+ "p50": 0.01,
1134
+ "p95": 0.01,
1135
+ "max": 0.039999999999999994
1136
+ },
1137
+ "memory_bytes": {
1138
+ "retained_heap_delta": -3736,
1139
+ "retained_rss_delta": 0,
1140
+ "sampled_peak_heap_delta": 493400,
1141
+ "sampled_peak_rss_delta": 0
1142
+ },
1143
+ "event_loop_delay_ms": {
1144
+ "p50": 10.133503,
1145
+ "p95": 10.133503,
1146
+ "max": 10.133503
1147
+ }
1148
+ },
1149
+ {
1150
+ "scenario": "pinned",
1151
+ "round": 2,
1152
+ "requests": 60,
1153
+ "setup_evaluator_calls": 1,
1154
+ "evaluator_calls": 60,
1155
+ "aborted_evaluations": 0,
1156
+ "route_ms": {
1157
+ "samples": 60,
1158
+ "p50": 1.2509589999999662,
1159
+ "p95": 1.30074999999988,
1160
+ "max": 1.484124999999949
1161
+ },
1162
+ "policy_overhead_ms": {
1163
+ "samples": 60,
1164
+ "p50": 0.020000000000000018,
1165
+ "p95": 0.030000000000000027,
1166
+ "max": 0.040000000000000036
1167
+ },
1168
+ "memory_bytes": {
1169
+ "retained_heap_delta": 108544,
1170
+ "retained_rss_delta": 0,
1171
+ "sampled_peak_heap_delta": 2306736,
1172
+ "sampled_peak_rss_delta": 0
1173
+ },
1174
+ "event_loop_delay_ms": {
1175
+ "p50": 9.986047,
1176
+ "p95": 10.919935,
1177
+ "max": 10.919935
1178
+ }
1179
+ },
1180
+ {
1181
+ "scenario": "concurrent_identical",
1182
+ "round": 2,
1183
+ "requests": 480,
1184
+ "setup_evaluator_calls": 0,
1185
+ "evaluator_calls": 60,
1186
+ "aborted_evaluations": 0,
1187
+ "route_ms": {
1188
+ "samples": 480,
1189
+ "p50": 1.708667000000105,
1190
+ "p95": 2.39654100000007,
1191
+ "max": 2.734083000000055
1192
+ },
1193
+ "policy_overhead_ms": {
1194
+ "samples": 480,
1195
+ "p50": 0.050000000000000044,
1196
+ "p95": 0.10999999999999988,
1197
+ "max": 0.2799999999999998
1198
+ },
1199
+ "memory_bytes": {
1200
+ "retained_heap_delta": 400712,
1201
+ "retained_rss_delta": 6816728,
1202
+ "sampled_peak_heap_delta": 7236016,
1203
+ "sampled_peak_rss_delta": 6816728
1204
+ },
1205
+ "event_loop_delay_ms": {
1206
+ "p50": 10.010623,
1207
+ "p95": 10.805247,
1208
+ "max": 11.493375
1209
+ }
1210
+ },
1211
+ {
1212
+ "scenario": "concurrent_identical_agents",
1213
+ "round": 2,
1214
+ "requests": 480,
1215
+ "setup_evaluator_calls": 0,
1216
+ "evaluator_calls": 60,
1217
+ "aborted_evaluations": 0,
1218
+ "route_ms": {
1219
+ "samples": 480,
1220
+ "p50": 1.3475419999999758,
1221
+ "p95": 1.4352499999999964,
1222
+ "max": 1.5907079999999496
1223
+ },
1224
+ "policy_overhead_ms": {
1225
+ "samples": 480,
1226
+ "p50": 0.010000000000000009,
1227
+ "p95": 0.030000000000000027,
1228
+ "max": 0.050000000000000044
1229
+ },
1230
+ "memory_bytes": {
1231
+ "retained_heap_delta": 137848,
1232
+ "retained_rss_delta": 573536,
1233
+ "sampled_peak_heap_delta": 7134464,
1234
+ "sampled_peak_rss_delta": 426080
1235
+ },
1236
+ "event_loop_delay_ms": {
1237
+ "p50": 10.092543,
1238
+ "p95": 10.764287,
1239
+ "max": 10.764287
1240
+ }
1241
+ },
1242
+ {
1243
+ "scenario": "concurrent_distinct_agents",
1244
+ "round": 2,
1245
+ "requests": 480,
1246
+ "setup_evaluator_calls": 0,
1247
+ "evaluator_calls": 480,
1248
+ "aborted_evaluations": 0,
1249
+ "route_ms": {
1250
+ "samples": 480,
1251
+ "p50": 1.4201249999998709,
1252
+ "p95": 1.6052500000000691,
1253
+ "max": 2.001708000000008
1254
+ },
1255
+ "policy_overhead_ms": {
1256
+ "samples": 480,
1257
+ "p50": 0.020000000000000018,
1258
+ "p95": 0.039999999999999813,
1259
+ "max": 0.08999999999999986
1260
+ },
1261
+ "memory_bytes": {
1262
+ "retained_heap_delta": 429856,
1263
+ "retained_rss_delta": 1229064,
1264
+ "sampled_peak_heap_delta": 7657848,
1265
+ "sampled_peak_rss_delta": 1065224
1266
+ },
1267
+ "event_loop_delay_ms": {
1268
+ "p50": 9.969663,
1269
+ "p95": 10.862591,
1270
+ "max": 10.862591
1271
+ }
1272
+ },
1273
+ {
1274
+ "scenario": "cancellation",
1275
+ "round": 2,
1276
+ "requests": 60,
1277
+ "setup_evaluator_calls": 0,
1278
+ "evaluator_calls": 0,
1279
+ "aborted_evaluations": 0,
1280
+ "route_ms": {
1281
+ "samples": 60,
1282
+ "p50": 0.015124999999898137,
1283
+ "p95": 0.022125000000187356,
1284
+ "max": 0.1478749999998854
1285
+ },
1286
+ "policy_overhead_ms": {
1287
+ "samples": 0,
1288
+ "p50": 0,
1289
+ "p95": 0,
1290
+ "max": 0
1291
+ },
1292
+ "memory_bytes": {
1293
+ "retained_heap_delta": -448672,
1294
+ "retained_rss_delta": 0,
1295
+ "sampled_peak_heap_delta": 691880,
1296
+ "sampled_peak_rss_delta": 0
1297
+ },
1298
+ "event_loop_delay_ms": {
1299
+ "p50": 10.526719,
1300
+ "p95": 10.526719,
1301
+ "max": 10.526719
1302
+ }
1303
+ },
1304
+ {
1305
+ "scenario": "small",
1306
+ "round": 3,
1307
+ "requests": 60,
1308
+ "setup_evaluator_calls": 0,
1309
+ "evaluator_calls": 60,
1310
+ "aborted_evaluations": 0,
1311
+ "route_ms": {
1312
+ "samples": 60,
1313
+ "p50": 1.2419580000000678,
1314
+ "p95": 1.2870840000000499,
1315
+ "max": 1.5305409999998574
1316
+ },
1317
+ "policy_overhead_ms": {
1318
+ "samples": 60,
1319
+ "p50": 0.020000000000000018,
1320
+ "p95": 0.030000000000000027,
1321
+ "max": 0.040000000000000036
1322
+ },
1323
+ "memory_bytes": {
1324
+ "retained_heap_delta": 107008,
1325
+ "retained_rss_delta": 32768,
1326
+ "sampled_peak_heap_delta": 2128576,
1327
+ "sampled_peak_rss_delta": 0
1328
+ },
1329
+ "event_loop_delay_ms": {
1330
+ "p50": 9.895935,
1331
+ "p95": 10.100735,
1332
+ "max": 10.100735
1333
+ }
1334
+ },
1335
+ {
1336
+ "scenario": "large_tools",
1337
+ "round": 3,
1338
+ "requests": 60,
1339
+ "setup_evaluator_calls": 0,
1340
+ "evaluator_calls": 60,
1341
+ "aborted_evaluations": 0,
1342
+ "route_ms": {
1343
+ "samples": 60,
1344
+ "p50": 2.80987499999992,
1345
+ "p95": 3.0369579999999132,
1346
+ "max": 3.5410420000000613
1347
+ },
1348
+ "policy_overhead_ms": {
1349
+ "samples": 60,
1350
+ "p50": 1,
1351
+ "p95": 1.1600000000000001,
1352
+ "max": 1.35
1353
+ },
1354
+ "memory_bytes": {
1355
+ "retained_heap_delta": -107648,
1356
+ "retained_rss_delta": 32768,
1357
+ "sampled_peak_heap_delta": 9258240,
1358
+ "sampled_peak_rss_delta": 32768
1359
+ },
1360
+ "event_loop_delay_ms": {
1361
+ "p50": 10.567679,
1362
+ "p95": 11.403263,
1363
+ "max": 11.558911
1364
+ }
1365
+ },
1366
+ {
1367
+ "scenario": "cache",
1368
+ "round": 3,
1369
+ "requests": 60,
1370
+ "setup_evaluator_calls": 1,
1371
+ "evaluator_calls": 0,
1372
+ "aborted_evaluations": 0,
1373
+ "route_ms": {
1374
+ "samples": 60,
1375
+ "p50": 0.00787499999978536,
1376
+ "p95": 0.014375000000200089,
1377
+ "max": 0.08358300000008967
1378
+ },
1379
+ "policy_overhead_ms": {
1380
+ "samples": 60,
1381
+ "p50": 0.01,
1382
+ "p95": 0.01,
1383
+ "max": 0.04
1384
+ },
1385
+ "memory_bytes": {
1386
+ "retained_heap_delta": -23160,
1387
+ "retained_rss_delta": 0,
1388
+ "sampled_peak_heap_delta": 494360,
1389
+ "sampled_peak_rss_delta": 0
1390
+ },
1391
+ "event_loop_delay_ms": {
1392
+ "p50": 9.895935,
1393
+ "p95": 9.895935,
1394
+ "max": 9.895935
1395
+ }
1396
+ },
1397
+ {
1398
+ "scenario": "pinned",
1399
+ "round": 3,
1400
+ "requests": 60,
1401
+ "setup_evaluator_calls": 1,
1402
+ "evaluator_calls": 60,
1403
+ "aborted_evaluations": 0,
1404
+ "route_ms": {
1405
+ "samples": 60,
1406
+ "p50": 1.177042000000256,
1407
+ "p95": 1.3462500000000546,
1408
+ "max": 1.4081670000000486
1409
+ },
1410
+ "policy_overhead_ms": {
1411
+ "samples": 60,
1412
+ "p50": 0.010000000000000009,
1413
+ "p95": 0.030000000000000027,
1414
+ "max": 0.05999999999999983
1415
+ },
1416
+ "memory_bytes": {
1417
+ "retained_heap_delta": 76816,
1418
+ "retained_rss_delta": 49152,
1419
+ "sampled_peak_heap_delta": 2316072,
1420
+ "sampled_peak_rss_delta": 16384
1421
+ },
1422
+ "event_loop_delay_ms": {
1423
+ "p50": 9.961471,
1424
+ "p95": 10.846207,
1425
+ "max": 10.846207
1426
+ }
1427
+ },
1428
+ {
1429
+ "scenario": "concurrent_identical",
1430
+ "round": 3,
1431
+ "requests": 480,
1432
+ "setup_evaluator_calls": 0,
1433
+ "evaluator_calls": 60,
1434
+ "aborted_evaluations": 0,
1435
+ "route_ms": {
1436
+ "samples": 480,
1437
+ "p50": 1.7448749999998654,
1438
+ "p95": 2.1965409999997974,
1439
+ "max": 3.3958330000000387
1440
+ },
1441
+ "policy_overhead_ms": {
1442
+ "samples": 480,
1443
+ "p50": 0.06000000000000005,
1444
+ "p95": 0.08000000000000007,
1445
+ "max": 0.24
1446
+ },
1447
+ "memory_bytes": {
1448
+ "retained_heap_delta": 275000,
1449
+ "retained_rss_delta": 131072,
1450
+ "sampled_peak_heap_delta": 7608200,
1451
+ "sampled_peak_rss_delta": 65536
1452
+ },
1453
+ "event_loop_delay_ms": {
1454
+ "p50": 10.018815,
1455
+ "p95": 10.641407,
1456
+ "max": 10.928127
1457
+ }
1458
+ },
1459
+ {
1460
+ "scenario": "concurrent_identical_agents",
1461
+ "round": 3,
1462
+ "requests": 480,
1463
+ "setup_evaluator_calls": 0,
1464
+ "evaluator_calls": 60,
1465
+ "aborted_evaluations": 0,
1466
+ "route_ms": {
1467
+ "samples": 480,
1468
+ "p50": 1.4071250000001783,
1469
+ "p95": 1.814665999999761,
1470
+ "max": 1.9708749999999782
1471
+ },
1472
+ "policy_overhead_ms": {
1473
+ "samples": 480,
1474
+ "p50": 0.020000000000000018,
1475
+ "p95": 0.050000000000000044,
1476
+ "max": 0.13000000000000012
1477
+ },
1478
+ "memory_bytes": {
1479
+ "retained_heap_delta": -93056,
1480
+ "retained_rss_delta": 49152,
1481
+ "sampled_peak_heap_delta": 7095968,
1482
+ "sampled_peak_rss_delta": 49152
1483
+ },
1484
+ "event_loop_delay_ms": {
1485
+ "p50": 10.289151,
1486
+ "p95": 10.797055,
1487
+ "max": 10.797055
1488
+ }
1489
+ },
1490
+ {
1491
+ "scenario": "concurrent_distinct_agents",
1492
+ "round": 3,
1493
+ "requests": 480,
1494
+ "setup_evaluator_calls": 0,
1495
+ "evaluator_calls": 480,
1496
+ "aborted_evaluations": 0,
1497
+ "route_ms": {
1498
+ "samples": 480,
1499
+ "p50": 1.5049579999999878,
1500
+ "p95": 1.727624999999989,
1501
+ "max": 2.165917000000263
1502
+ },
1503
+ "policy_overhead_ms": {
1504
+ "samples": 480,
1505
+ "p50": 0.030000000000000027,
1506
+ "p95": 0.040000000000000036,
1507
+ "max": 0.06999999999999984
1508
+ },
1509
+ "memory_bytes": {
1510
+ "retained_heap_delta": 393104,
1511
+ "retained_rss_delta": 425984,
1512
+ "sampled_peak_heap_delta": 7599696,
1513
+ "sampled_peak_rss_delta": 278528
1514
+ },
1515
+ "event_loop_delay_ms": {
1516
+ "p50": 10.264575,
1517
+ "p95": 10.608639,
1518
+ "max": 10.649599
1519
+ }
1520
+ },
1521
+ {
1522
+ "scenario": "cancellation",
1523
+ "round": 3,
1524
+ "requests": 60,
1525
+ "setup_evaluator_calls": 0,
1526
+ "evaluator_calls": 0,
1527
+ "aborted_evaluations": 0,
1528
+ "route_ms": {
1529
+ "samples": 60,
1530
+ "p50": 0.017875000000003638,
1531
+ "p95": 0.02070800000001327,
1532
+ "max": 0.10670899999968242
1533
+ },
1534
+ "policy_overhead_ms": {
1535
+ "samples": 0,
1536
+ "p50": 0,
1537
+ "p95": 0,
1538
+ "max": 0
1539
+ },
1540
+ "memory_bytes": {
1541
+ "retained_heap_delta": -453616,
1542
+ "retained_rss_delta": 0,
1543
+ "sampled_peak_heap_delta": 667120,
1544
+ "sampled_peak_rss_delta": 0
1545
+ },
1546
+ "event_loop_delay_ms": {
1547
+ "p50": 9.256959,
1548
+ "p95": 9.256959,
1549
+ "max": 9.256959
1550
+ }
1551
+ }
1552
+ ]
1553
+ },
1554
+ "comparison": {
1555
+ "passed": true,
1556
+ "comparable": {
1557
+ "passed": true,
1558
+ "expected": "Same workload parameters, Node version and hardware; background load remains reported separately."
1559
+ },
1560
+ "evaluator_calls": {
1561
+ "passed": true,
1562
+ "expected": "One evaluation per group of eight identical concurrent requests, including different agents."
1563
+ },
1564
+ "latency": [
1565
+ {
1566
+ "scenario": "small",
1567
+ "baseline_max_p95_ms": 1.646,
1568
+ "candidate_max_p95_ms": 1.396,
1569
+ "limit_ms": 3.292,
1570
+ "passed": true,
1571
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1572
+ },
1573
+ {
1574
+ "scenario": "large_tools",
1575
+ "baseline_max_p95_ms": 4.415,
1576
+ "candidate_max_p95_ms": 3.037,
1577
+ "limit_ms": 8.831,
1578
+ "passed": true,
1579
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1580
+ },
1581
+ {
1582
+ "scenario": "cache",
1583
+ "baseline_max_p95_ms": 0.035,
1584
+ "candidate_max_p95_ms": 0.014,
1585
+ "limit_ms": 0.071,
1586
+ "passed": true,
1587
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1588
+ },
1589
+ {
1590
+ "scenario": "pinned",
1591
+ "baseline_max_p95_ms": 1.778,
1592
+ "candidate_max_p95_ms": 1.346,
1593
+ "limit_ms": 3.556,
1594
+ "passed": true,
1595
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1596
+ },
1597
+ {
1598
+ "scenario": "concurrent_identical",
1599
+ "baseline_max_p95_ms": 2.815,
1600
+ "candidate_max_p95_ms": 2.397,
1601
+ "limit_ms": 5.631,
1602
+ "passed": true,
1603
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1604
+ },
1605
+ {
1606
+ "scenario": "concurrent_identical_agents",
1607
+ "baseline_max_p95_ms": 2.219,
1608
+ "candidate_max_p95_ms": 1.815,
1609
+ "limit_ms": 4.439,
1610
+ "passed": true,
1611
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1612
+ },
1613
+ {
1614
+ "scenario": "concurrent_distinct_agents",
1615
+ "baseline_max_p95_ms": 2.201,
1616
+ "candidate_max_p95_ms": 1.907,
1617
+ "limit_ms": 4.401,
1618
+ "passed": true,
1619
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1620
+ },
1621
+ {
1622
+ "scenario": "cancellation",
1623
+ "baseline_max_p95_ms": 0.124,
1624
+ "candidate_max_p95_ms": 0.056,
1625
+ "limit_ms": 0.249,
1626
+ "passed": true,
1627
+ "rationale": "Local regression guard: twice the largest p95 across three pre-change rounds; not a universal latency target."
1628
+ }
1629
+ ],
1630
+ "resources": [
1631
+ {
1632
+ "scenario": "small",
1633
+ "sampled_peak_heap_limit_bytes": 8560128,
1634
+ "candidate_sampled_peak_heap_bytes": 5038504,
1635
+ "event_loop_p95_limit_ms": 90.767,
1636
+ "candidate_event_loop_p95_ms": 42.598,
1637
+ "passed": true
1638
+ },
1639
+ {
1640
+ "scenario": "large_tools",
1641
+ "sampled_peak_heap_limit_bytes": 17805456,
1642
+ "candidate_sampled_peak_heap_bytes": 9258240,
1643
+ "event_loop_p95_limit_ms": 24.019,
1644
+ "candidate_event_loop_p95_ms": 11.403,
1645
+ "passed": true
1646
+ },
1647
+ {
1648
+ "scenario": "cache",
1649
+ "sampled_peak_heap_limit_bytes": 921232,
1650
+ "candidate_sampled_peak_heap_bytes": 498888,
1651
+ "event_loop_p95_limit_ms": 21.479,
1652
+ "candidate_event_loop_p95_ms": 10.134,
1653
+ "passed": true
1654
+ },
1655
+ {
1656
+ "scenario": "pinned",
1657
+ "sampled_peak_heap_limit_bytes": 3747296,
1658
+ "candidate_sampled_peak_heap_bytes": 2411504,
1659
+ "event_loop_p95_limit_ms": 21.791,
1660
+ "candidate_event_loop_p95_ms": 10.92,
1661
+ "passed": true
1662
+ },
1663
+ {
1664
+ "scenario": "concurrent_identical",
1665
+ "sampled_peak_heap_limit_bytes": 15633488,
1666
+ "candidate_sampled_peak_heap_bytes": 7608200,
1667
+ "event_loop_p95_limit_ms": 21.856,
1668
+ "candidate_event_loop_p95_ms": 10.805,
1669
+ "passed": true
1670
+ },
1671
+ {
1672
+ "scenario": "concurrent_identical_agents",
1673
+ "sampled_peak_heap_limit_bytes": 14187472,
1674
+ "candidate_sampled_peak_heap_bytes": 7134464,
1675
+ "event_loop_p95_limit_ms": 21.725,
1676
+ "candidate_event_loop_p95_ms": 10.904,
1677
+ "passed": true
1678
+ },
1679
+ {
1680
+ "scenario": "concurrent_distinct_agents",
1681
+ "sampled_peak_heap_limit_bytes": 14305120,
1682
+ "candidate_sampled_peak_heap_bytes": 7657848,
1683
+ "event_loop_p95_limit_ms": 21.578,
1684
+ "candidate_event_loop_p95_ms": 10.863,
1685
+ "passed": true
1686
+ },
1687
+ {
1688
+ "scenario": "cancellation",
1689
+ "sampled_peak_heap_limit_bytes": 2010304,
1690
+ "candidate_sampled_peak_heap_bytes": 750816,
1691
+ "event_loop_p95_limit_ms": 21.922,
1692
+ "candidate_event_loop_p95_ms": 10.527,
1693
+ "passed": true
1694
+ }
1695
+ ]
1696
+ }
1697
+ }