@cosyte/bench 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,781 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/cosyte/config/blob/main/packages/bench/schema/results-v1.schema.json",
4
+ "title": "@cosyte/bench results, version 1",
5
+ "type": "object",
6
+ "required": ["schemaVersion", "kind", "tool", "createdAt", "suite", "environment", "contract"],
7
+ "properties": {
8
+ "schemaVersion": {
9
+ "const": 1
10
+ },
11
+ "kind": {
12
+ "enum": ["reading", "comparison", "calibration"]
13
+ },
14
+ "tool": {
15
+ "type": "object",
16
+ "required": ["name", "version"],
17
+ "properties": {
18
+ "name": {
19
+ "const": "@cosyte/bench"
20
+ },
21
+ "version": {
22
+ "type": "string"
23
+ }
24
+ }
25
+ },
26
+ "createdAt": {
27
+ "type": "string"
28
+ },
29
+ "suite": {
30
+ "type": "object",
31
+ "required": ["name", "file"],
32
+ "properties": {
33
+ "name": {
34
+ "type": "string"
35
+ },
36
+ "file": {
37
+ "type": "string"
38
+ }
39
+ }
40
+ },
41
+ "environment": {
42
+ "$ref": "#/$defs/environment"
43
+ },
44
+ "contract": {
45
+ "type": "object",
46
+ "required": ["bench", "warmup"],
47
+ "properties": {
48
+ "bench": {
49
+ "type": "object"
50
+ },
51
+ "warmup": {
52
+ "type": "object"
53
+ }
54
+ }
55
+ }
56
+ },
57
+ "oneOf": [
58
+ {
59
+ "required": ["subject", "reading"],
60
+ "properties": {
61
+ "kind": {
62
+ "const": "reading"
63
+ },
64
+ "subject": {
65
+ "$ref": "#/$defs/build"
66
+ },
67
+ "reading": {
68
+ "$ref": "#/$defs/reading"
69
+ }
70
+ }
71
+ },
72
+ {
73
+ "required": ["base", "head", "comparison"],
74
+ "properties": {
75
+ "kind": {
76
+ "const": "comparison"
77
+ },
78
+ "base": {
79
+ "$ref": "#/$defs/build"
80
+ },
81
+ "head": {
82
+ "$ref": "#/$defs/build"
83
+ },
84
+ "comparison": {
85
+ "$ref": "#/$defs/comparison"
86
+ }
87
+ }
88
+ },
89
+ {
90
+ "required": ["base", "head", "calibration"],
91
+ "properties": {
92
+ "kind": {
93
+ "const": "calibration"
94
+ },
95
+ "base": {
96
+ "$ref": "#/$defs/build"
97
+ },
98
+ "head": {
99
+ "$ref": "#/$defs/build"
100
+ },
101
+ "calibration": {
102
+ "$ref": "#/$defs/calibration"
103
+ }
104
+ }
105
+ }
106
+ ],
107
+ "$defs": {
108
+ "build": {
109
+ "type": "object",
110
+ "required": ["target", "commit"],
111
+ "properties": {
112
+ "target": {
113
+ "type": "string"
114
+ },
115
+ "commit": {
116
+ "type": ["string", "null"]
117
+ }
118
+ }
119
+ },
120
+ "environment": {
121
+ "type": "object",
122
+ "required": [
123
+ "node",
124
+ "v8",
125
+ "platform",
126
+ "arch",
127
+ "osRelease",
128
+ "cpuModel",
129
+ "logicalCpus",
130
+ "totalMemoryBytes",
131
+ "cgroupCpuMax",
132
+ "runner"
133
+ ],
134
+ "properties": {
135
+ "node": {
136
+ "type": "string"
137
+ },
138
+ "v8": {
139
+ "type": "string"
140
+ },
141
+ "platform": {
142
+ "type": "string"
143
+ },
144
+ "arch": {
145
+ "type": "string"
146
+ },
147
+ "osRelease": {
148
+ "type": "string"
149
+ },
150
+ "cpuModel": {
151
+ "type": ["string", "null"]
152
+ },
153
+ "logicalCpus": {
154
+ "type": "integer",
155
+ "minimum": 0
156
+ },
157
+ "totalMemoryBytes": {
158
+ "type": "integer",
159
+ "minimum": 0
160
+ },
161
+ "cgroupCpuMax": {
162
+ "type": ["string", "null"]
163
+ },
164
+ "loadAverage": {
165
+ "type": "array",
166
+ "items": {
167
+ "type": "number"
168
+ }
169
+ },
170
+ "runner": {
171
+ "type": "object",
172
+ "required": ["ci", "provider", "class"],
173
+ "properties": {
174
+ "ci": {
175
+ "type": "boolean"
176
+ },
177
+ "provider": {
178
+ "type": ["string", "null"]
179
+ },
180
+ "class": {
181
+ "type": "string"
182
+ }
183
+ }
184
+ }
185
+ }
186
+ },
187
+ "interval": {
188
+ "oneOf": [
189
+ {
190
+ "type": "null"
191
+ },
192
+ {
193
+ "type": "object",
194
+ "required": ["low", "high", "rank", "tail"],
195
+ "properties": {
196
+ "low": {
197
+ "type": "number"
198
+ },
199
+ "high": {
200
+ "type": "number"
201
+ },
202
+ "rank": {
203
+ "type": "integer",
204
+ "minimum": 0
205
+ },
206
+ "tail": {
207
+ "type": "number"
208
+ }
209
+ }
210
+ }
211
+ ]
212
+ },
213
+ "caseComparison": {
214
+ "type": "object",
215
+ "required": [
216
+ "benchmark",
217
+ "size",
218
+ "baseMsPerOp",
219
+ "headMsPerOp",
220
+ "ratios",
221
+ "medianRatio",
222
+ "interval",
223
+ "verdict",
224
+ "skipReason"
225
+ ],
226
+ "properties": {
227
+ "benchmark": {
228
+ "type": "string"
229
+ },
230
+ "size": {
231
+ "type": "string"
232
+ },
233
+ "baseMsPerOp": {
234
+ "type": "array",
235
+ "items": {
236
+ "type": "number"
237
+ }
238
+ },
239
+ "headMsPerOp": {
240
+ "type": "array",
241
+ "items": {
242
+ "type": "number"
243
+ }
244
+ },
245
+ "ratios": {
246
+ "type": "array",
247
+ "items": {
248
+ "type": "number"
249
+ }
250
+ },
251
+ "medianRatio": {
252
+ "type": ["number", "null"]
253
+ },
254
+ "basePlans": {
255
+ "type": "array",
256
+ "items": {
257
+ "enum": ["standard", "large"]
258
+ }
259
+ },
260
+ "headPlans": {
261
+ "type": "array",
262
+ "items": {
263
+ "enum": ["standard", "large"]
264
+ }
265
+ },
266
+ "interval": {
267
+ "$ref": "#/$defs/interval"
268
+ },
269
+ "verdict": {
270
+ "enum": ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
271
+ },
272
+ "skipReason": {
273
+ "type": ["string", "null"]
274
+ }
275
+ }
276
+ },
277
+ "comparison": {
278
+ "type": "object",
279
+ "required": ["rounds", "threshold", "alpha", "inject", "cases"],
280
+ "properties": {
281
+ "rounds": {
282
+ "type": "integer",
283
+ "minimum": 0
284
+ },
285
+ "threshold": {
286
+ "type": "number"
287
+ },
288
+ "alpha": {
289
+ "type": "number"
290
+ },
291
+ "inject": {
292
+ "type": "number"
293
+ },
294
+ "cases": {
295
+ "type": "array",
296
+ "items": {
297
+ "$ref": "#/$defs/caseComparison"
298
+ }
299
+ }
300
+ }
301
+ },
302
+ "rate": {
303
+ "type": "object",
304
+ "required": ["events", "trials", "rate", "upperBound"],
305
+ "properties": {
306
+ "events": {
307
+ "type": "integer",
308
+ "minimum": 0
309
+ },
310
+ "trials": {
311
+ "type": "integer",
312
+ "minimum": 0
313
+ },
314
+ "rate": {
315
+ "type": "number"
316
+ },
317
+ "upperBound": {
318
+ "type": "number"
319
+ }
320
+ }
321
+ },
322
+ "calibration": {
323
+ "type": "object",
324
+ "required": [
325
+ "inject",
326
+ "comparisons",
327
+ "runs",
328
+ "perCase",
329
+ "perComparison",
330
+ "verdicts",
331
+ "requiredRule"
332
+ ],
333
+ "properties": {
334
+ "inject": {
335
+ "type": "number"
336
+ },
337
+ "comparisons": {
338
+ "type": "integer",
339
+ "minimum": 0
340
+ },
341
+ "runs": {
342
+ "type": "array",
343
+ "items": {
344
+ "type": "array",
345
+ "items": {
346
+ "$ref": "#/$defs/caseComparison"
347
+ }
348
+ }
349
+ },
350
+ "perCase": {
351
+ "$ref": "#/$defs/rate"
352
+ },
353
+ "perComparison": {
354
+ "$ref": "#/$defs/rate"
355
+ },
356
+ "skippedComparisons": {
357
+ "$ref": "#/$defs/rate"
358
+ },
359
+ "verdicts": {
360
+ "type": "object"
361
+ },
362
+ "observed": {
363
+ "type": "array",
364
+ "items": {
365
+ "type": "object",
366
+ "required": ["benchmark", "size", "rounds", "medianRatio"],
367
+ "properties": {
368
+ "benchmark": {
369
+ "type": "string"
370
+ },
371
+ "size": {
372
+ "type": "string"
373
+ },
374
+ "rounds": {
375
+ "type": "integer",
376
+ "minimum": 0
377
+ },
378
+ "medianRatio": {
379
+ "type": ["number", "null"]
380
+ }
381
+ }
382
+ }
383
+ },
384
+ "requiredRule": {
385
+ "type": "object",
386
+ "required": ["applies", "met", "reason"],
387
+ "properties": {
388
+ "applies": {
389
+ "type": "boolean"
390
+ },
391
+ "met": {
392
+ "type": "boolean"
393
+ },
394
+ "reason": {
395
+ "type": "string"
396
+ }
397
+ }
398
+ }
399
+ }
400
+ },
401
+ "reading": {
402
+ "type": "object",
403
+ "required": ["processes", "cases", "importCost"],
404
+ "properties": {
405
+ "processes": {
406
+ "type": "integer",
407
+ "minimum": 0
408
+ },
409
+ "cases": {
410
+ "type": "array",
411
+ "items": {
412
+ "$ref": "#/$defs/caseReading"
413
+ }
414
+ },
415
+ "importCost": {
416
+ "oneOf": [
417
+ {
418
+ "type": "null"
419
+ },
420
+ {
421
+ "type": "object",
422
+ "required": [
423
+ "runs",
424
+ "coldImportMs",
425
+ "medianMs",
426
+ "p95Ms",
427
+ "emptyImportMs",
428
+ "emptyMedianMs",
429
+ "files",
430
+ "totalBytes",
431
+ "totalGzipBytes"
432
+ ],
433
+ "properties": {
434
+ "runs": {
435
+ "type": "integer",
436
+ "minimum": 0
437
+ },
438
+ "coldImportMs": {
439
+ "type": "array",
440
+ "items": {
441
+ "type": "number"
442
+ }
443
+ },
444
+ "medianMs": {
445
+ "type": "number"
446
+ },
447
+ "p95Ms": {
448
+ "type": "number"
449
+ },
450
+ "emptyImportMs": {
451
+ "type": "array",
452
+ "items": {
453
+ "type": "number"
454
+ }
455
+ },
456
+ "emptyMedianMs": {
457
+ "type": "number"
458
+ },
459
+ "files": {
460
+ "type": "array",
461
+ "items": {
462
+ "type": "object",
463
+ "required": ["path", "bytes", "gzipBytes"],
464
+ "properties": {
465
+ "path": {
466
+ "type": "string"
467
+ },
468
+ "bytes": {
469
+ "type": "integer",
470
+ "minimum": 0
471
+ },
472
+ "gzipBytes": {
473
+ "type": "integer",
474
+ "minimum": 0
475
+ }
476
+ }
477
+ }
478
+ },
479
+ "totalBytes": {
480
+ "type": "integer",
481
+ "minimum": 0
482
+ },
483
+ "totalGzipBytes": {
484
+ "type": "integer",
485
+ "minimum": 0
486
+ }
487
+ }
488
+ }
489
+ ]
490
+ }
491
+ }
492
+ },
493
+ "caseReading": {
494
+ "type": "object",
495
+ "required": [
496
+ "benchmark",
497
+ "size",
498
+ "inputsPerPass",
499
+ "bytesPerPass",
500
+ "throughput",
501
+ "latency",
502
+ "allocations",
503
+ "peak",
504
+ "longRun"
505
+ ],
506
+ "properties": {
507
+ "benchmark": {
508
+ "type": "string"
509
+ },
510
+ "size": {
511
+ "type": "string"
512
+ },
513
+ "inputsPerPass": {
514
+ "type": "integer",
515
+ "minimum": 0
516
+ },
517
+ "bytesPerPass": {
518
+ "type": ["number", "null"]
519
+ },
520
+ "throughput": {
521
+ "oneOf": [
522
+ {
523
+ "type": "object",
524
+ "required": [
525
+ "status",
526
+ "processes",
527
+ "medianMsPerOp",
528
+ "opsPerSecond",
529
+ "bytesPerSecond"
530
+ ],
531
+ "properties": {
532
+ "status": {
533
+ "const": "measured"
534
+ },
535
+ "processes": {
536
+ "type": "array",
537
+ "items": {
538
+ "type": "object",
539
+ "required": ["samplesMsPerOp", "medianMsPerOp"],
540
+ "properties": {
541
+ "samplesMsPerOp": {
542
+ "type": "array",
543
+ "items": {
544
+ "type": "number"
545
+ }
546
+ },
547
+ "medianMsPerOp": {
548
+ "type": "number"
549
+ },
550
+ "plan": {
551
+ "enum": ["standard", "large"]
552
+ }
553
+ }
554
+ }
555
+ },
556
+ "medianMsPerOp": {
557
+ "type": "number"
558
+ },
559
+ "opsPerSecond": {
560
+ "type": "number"
561
+ },
562
+ "bytesPerSecond": {
563
+ "type": ["number", "null"]
564
+ }
565
+ }
566
+ },
567
+ {
568
+ "type": "object",
569
+ "required": ["status", "reason"],
570
+ "properties": {
571
+ "status": {
572
+ "const": "skipped"
573
+ },
574
+ "reason": {
575
+ "type": "string"
576
+ }
577
+ }
578
+ }
579
+ ]
580
+ },
581
+ "latency": {
582
+ "oneOf": [
583
+ {
584
+ "type": "null"
585
+ },
586
+ {
587
+ "type": "object",
588
+ "required": ["status", "calls", "timerOverheadNs", "p50Ns"],
589
+ "properties": {
590
+ "status": {
591
+ "enum": ["measured", "skipped"]
592
+ },
593
+ "calls": {
594
+ "type": "integer",
595
+ "minimum": 0
596
+ },
597
+ "timerOverheadNs": {
598
+ "type": "number"
599
+ },
600
+ "p50Ns": {
601
+ "type": ["number", "null"]
602
+ },
603
+ "p95Ns": {
604
+ "type": ["number", "null"]
605
+ },
606
+ "p99Ns": {
607
+ "type": ["number", "null"]
608
+ },
609
+ "maxNs": {
610
+ "type": "number"
611
+ }
612
+ }
613
+ }
614
+ ]
615
+ },
616
+ "allocations": {
617
+ "oneOf": [
618
+ {
619
+ "type": "null"
620
+ },
621
+ {
622
+ "oneOf": [
623
+ {
624
+ "type": "object",
625
+ "required": ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
626
+ "properties": {
627
+ "status": {
628
+ "const": "measured"
629
+ },
630
+ "method": {
631
+ "enum": ["no-gc-window", "gc-accounted"]
632
+ },
633
+ "collections": {
634
+ "type": "integer",
635
+ "minimum": 0
636
+ },
637
+ "windowOps": {
638
+ "type": "integer",
639
+ "minimum": 0
640
+ },
641
+ "heapBytesPerOp": {
642
+ "type": "number"
643
+ },
644
+ "externalBytesPerOp": {
645
+ "type": "number"
646
+ }
647
+ }
648
+ },
649
+ {
650
+ "type": "object",
651
+ "required": ["status", "reason"],
652
+ "properties": {
653
+ "status": {
654
+ "const": "skipped"
655
+ },
656
+ "reason": {
657
+ "type": "string"
658
+ }
659
+ }
660
+ }
661
+ ]
662
+ }
663
+ ]
664
+ },
665
+ "peak": {
666
+ "oneOf": [
667
+ {
668
+ "type": "null"
669
+ },
670
+ {
671
+ "oneOf": [
672
+ {
673
+ "type": "object",
674
+ "required": ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
675
+ "properties": {
676
+ "status": {
677
+ "const": "measured"
678
+ },
679
+ "calls": {
680
+ "type": "integer",
681
+ "minimum": 0
682
+ },
683
+ "peakHeapBytes": {
684
+ "type": "number"
685
+ },
686
+ "medianPeakHeapBytes": {
687
+ "type": "number"
688
+ },
689
+ "peakExternalBytes": {
690
+ "type": "number"
691
+ },
692
+ "medianPeakExternalBytes": {
693
+ "type": "number"
694
+ },
695
+ "peakTotalBytes": {
696
+ "type": "number"
697
+ },
698
+ "medianPeakTotalBytes": {
699
+ "type": "number"
700
+ }
701
+ }
702
+ },
703
+ {
704
+ "type": "object",
705
+ "required": ["status", "reason"],
706
+ "properties": {
707
+ "status": {
708
+ "const": "skipped"
709
+ },
710
+ "reason": {
711
+ "type": "string"
712
+ }
713
+ }
714
+ }
715
+ ]
716
+ }
717
+ ]
718
+ },
719
+ "longRun": {
720
+ "oneOf": [
721
+ {
722
+ "type": "null"
723
+ },
724
+ {
725
+ "oneOf": [
726
+ {
727
+ "type": "object",
728
+ "required": [
729
+ "status",
730
+ "verdict",
731
+ "ops",
732
+ "opsPerRound",
733
+ "settledHeap",
734
+ "slopeBytesPerOp"
735
+ ],
736
+ "properties": {
737
+ "status": {
738
+ "const": "measured"
739
+ },
740
+ "verdict": {
741
+ "enum": ["flat", "growing"]
742
+ },
743
+ "ops": {
744
+ "type": "integer",
745
+ "minimum": 0
746
+ },
747
+ "opsPerRound": {
748
+ "type": "integer",
749
+ "minimum": 0
750
+ },
751
+ "settledHeap": {
752
+ "type": "array",
753
+ "items": {
754
+ "type": "number"
755
+ }
756
+ },
757
+ "slopeBytesPerOp": {
758
+ "type": "number"
759
+ }
760
+ }
761
+ },
762
+ {
763
+ "type": "object",
764
+ "required": ["status", "reason"],
765
+ "properties": {
766
+ "status": {
767
+ "const": "skipped"
768
+ },
769
+ "reason": {
770
+ "type": "string"
771
+ }
772
+ }
773
+ }
774
+ ]
775
+ }
776
+ ]
777
+ }
778
+ }
779
+ }
780
+ }
781
+ }