@cosyte/bench 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,747 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/cosyte/config/blob/main/packages/bench/schema/results-v1.schema.json",
4
+ "title": "@cosyte/bench results, version 1",
5
+ "type": "object",
6
+ "required": ["schemaVersion", "kind", "tool", "createdAt", "suite", "environment", "contract"],
7
+ "properties": {
8
+ "schemaVersion": {
9
+ "const": 1
10
+ },
11
+ "kind": {
12
+ "enum": ["reading", "comparison", "calibration"]
13
+ },
14
+ "tool": {
15
+ "type": "object",
16
+ "required": ["name", "version"],
17
+ "properties": {
18
+ "name": {
19
+ "const": "@cosyte/bench"
20
+ },
21
+ "version": {
22
+ "type": "string"
23
+ }
24
+ }
25
+ },
26
+ "createdAt": {
27
+ "type": "string"
28
+ },
29
+ "suite": {
30
+ "type": "object",
31
+ "required": ["name", "file"],
32
+ "properties": {
33
+ "name": {
34
+ "type": "string"
35
+ },
36
+ "file": {
37
+ "type": "string"
38
+ }
39
+ }
40
+ },
41
+ "environment": {
42
+ "$ref": "#/$defs/environment"
43
+ },
44
+ "contract": {
45
+ "type": "object",
46
+ "required": ["bench", "warmup"],
47
+ "properties": {
48
+ "bench": {
49
+ "type": "object"
50
+ },
51
+ "warmup": {
52
+ "type": "object"
53
+ }
54
+ }
55
+ }
56
+ },
57
+ "oneOf": [
58
+ {
59
+ "required": ["subject", "reading"],
60
+ "properties": {
61
+ "kind": {
62
+ "const": "reading"
63
+ },
64
+ "subject": {
65
+ "$ref": "#/$defs/build"
66
+ },
67
+ "reading": {
68
+ "$ref": "#/$defs/reading"
69
+ }
70
+ }
71
+ },
72
+ {
73
+ "required": ["base", "head", "comparison"],
74
+ "properties": {
75
+ "kind": {
76
+ "const": "comparison"
77
+ },
78
+ "base": {
79
+ "$ref": "#/$defs/build"
80
+ },
81
+ "head": {
82
+ "$ref": "#/$defs/build"
83
+ },
84
+ "comparison": {
85
+ "$ref": "#/$defs/comparison"
86
+ }
87
+ }
88
+ },
89
+ {
90
+ "required": ["base", "head", "calibration"],
91
+ "properties": {
92
+ "kind": {
93
+ "const": "calibration"
94
+ },
95
+ "base": {
96
+ "$ref": "#/$defs/build"
97
+ },
98
+ "head": {
99
+ "$ref": "#/$defs/build"
100
+ },
101
+ "calibration": {
102
+ "$ref": "#/$defs/calibration"
103
+ }
104
+ }
105
+ }
106
+ ],
107
+ "$defs": {
108
+ "build": {
109
+ "type": "object",
110
+ "required": ["target", "commit"],
111
+ "properties": {
112
+ "target": {
113
+ "type": "string"
114
+ },
115
+ "commit": {
116
+ "type": ["string", "null"]
117
+ }
118
+ }
119
+ },
120
+ "environment": {
121
+ "type": "object",
122
+ "required": [
123
+ "node",
124
+ "v8",
125
+ "platform",
126
+ "arch",
127
+ "osRelease",
128
+ "cpuModel",
129
+ "logicalCpus",
130
+ "totalMemoryBytes",
131
+ "cgroupCpuMax",
132
+ "runner"
133
+ ],
134
+ "properties": {
135
+ "node": {
136
+ "type": "string"
137
+ },
138
+ "v8": {
139
+ "type": "string"
140
+ },
141
+ "platform": {
142
+ "type": "string"
143
+ },
144
+ "arch": {
145
+ "type": "string"
146
+ },
147
+ "osRelease": {
148
+ "type": "string"
149
+ },
150
+ "cpuModel": {
151
+ "type": ["string", "null"]
152
+ },
153
+ "logicalCpus": {
154
+ "type": "integer",
155
+ "minimum": 0
156
+ },
157
+ "totalMemoryBytes": {
158
+ "type": "integer",
159
+ "minimum": 0
160
+ },
161
+ "cgroupCpuMax": {
162
+ "type": ["string", "null"]
163
+ },
164
+ "loadAverage": {
165
+ "type": "array",
166
+ "items": {
167
+ "type": "number"
168
+ }
169
+ },
170
+ "runner": {
171
+ "type": "object",
172
+ "required": ["ci", "provider", "class"],
173
+ "properties": {
174
+ "ci": {
175
+ "type": "boolean"
176
+ },
177
+ "provider": {
178
+ "type": ["string", "null"]
179
+ },
180
+ "class": {
181
+ "type": "string"
182
+ }
183
+ }
184
+ }
185
+ }
186
+ },
187
+ "interval": {
188
+ "oneOf": [
189
+ {
190
+ "type": "null"
191
+ },
192
+ {
193
+ "type": "object",
194
+ "required": ["low", "high", "rank", "tail"],
195
+ "properties": {
196
+ "low": {
197
+ "type": "number"
198
+ },
199
+ "high": {
200
+ "type": "number"
201
+ },
202
+ "rank": {
203
+ "type": "integer",
204
+ "minimum": 0
205
+ },
206
+ "tail": {
207
+ "type": "number"
208
+ }
209
+ }
210
+ }
211
+ ]
212
+ },
213
+ "caseComparison": {
214
+ "type": "object",
215
+ "required": [
216
+ "benchmark",
217
+ "size",
218
+ "baseMsPerOp",
219
+ "headMsPerOp",
220
+ "ratios",
221
+ "medianRatio",
222
+ "interval",
223
+ "verdict",
224
+ "skipReason"
225
+ ],
226
+ "properties": {
227
+ "benchmark": {
228
+ "type": "string"
229
+ },
230
+ "size": {
231
+ "type": "string"
232
+ },
233
+ "baseMsPerOp": {
234
+ "type": "array",
235
+ "items": {
236
+ "type": "number"
237
+ }
238
+ },
239
+ "headMsPerOp": {
240
+ "type": "array",
241
+ "items": {
242
+ "type": "number"
243
+ }
244
+ },
245
+ "ratios": {
246
+ "type": "array",
247
+ "items": {
248
+ "type": "number"
249
+ }
250
+ },
251
+ "medianRatio": {
252
+ "type": ["number", "null"]
253
+ },
254
+ "interval": {
255
+ "$ref": "#/$defs/interval"
256
+ },
257
+ "verdict": {
258
+ "enum": ["regression", "improvement", "no-regression", "inconclusive", "skipped"]
259
+ },
260
+ "skipReason": {
261
+ "type": ["string", "null"]
262
+ }
263
+ }
264
+ },
265
+ "comparison": {
266
+ "type": "object",
267
+ "required": ["rounds", "threshold", "alpha", "inject", "cases"],
268
+ "properties": {
269
+ "rounds": {
270
+ "type": "integer",
271
+ "minimum": 0
272
+ },
273
+ "threshold": {
274
+ "type": "number"
275
+ },
276
+ "alpha": {
277
+ "type": "number"
278
+ },
279
+ "inject": {
280
+ "type": "number"
281
+ },
282
+ "cases": {
283
+ "type": "array",
284
+ "items": {
285
+ "$ref": "#/$defs/caseComparison"
286
+ }
287
+ }
288
+ }
289
+ },
290
+ "rate": {
291
+ "type": "object",
292
+ "required": ["events", "trials", "rate", "upperBound"],
293
+ "properties": {
294
+ "events": {
295
+ "type": "integer",
296
+ "minimum": 0
297
+ },
298
+ "trials": {
299
+ "type": "integer",
300
+ "minimum": 0
301
+ },
302
+ "rate": {
303
+ "type": "number"
304
+ },
305
+ "upperBound": {
306
+ "type": "number"
307
+ }
308
+ }
309
+ },
310
+ "calibration": {
311
+ "type": "object",
312
+ "required": [
313
+ "inject",
314
+ "comparisons",
315
+ "runs",
316
+ "perCase",
317
+ "perComparison",
318
+ "verdicts",
319
+ "requiredRule"
320
+ ],
321
+ "properties": {
322
+ "inject": {
323
+ "type": "number"
324
+ },
325
+ "comparisons": {
326
+ "type": "integer",
327
+ "minimum": 0
328
+ },
329
+ "runs": {
330
+ "type": "array",
331
+ "items": {
332
+ "type": "array",
333
+ "items": {
334
+ "$ref": "#/$defs/caseComparison"
335
+ }
336
+ }
337
+ },
338
+ "perCase": {
339
+ "$ref": "#/$defs/rate"
340
+ },
341
+ "perComparison": {
342
+ "$ref": "#/$defs/rate"
343
+ },
344
+ "skippedComparisons": {
345
+ "$ref": "#/$defs/rate"
346
+ },
347
+ "verdicts": {
348
+ "type": "object"
349
+ },
350
+ "observed": {
351
+ "type": "array",
352
+ "items": {
353
+ "type": "object",
354
+ "required": ["benchmark", "size", "rounds", "medianRatio"],
355
+ "properties": {
356
+ "benchmark": {
357
+ "type": "string"
358
+ },
359
+ "size": {
360
+ "type": "string"
361
+ },
362
+ "rounds": {
363
+ "type": "integer",
364
+ "minimum": 0
365
+ },
366
+ "medianRatio": {
367
+ "type": ["number", "null"]
368
+ }
369
+ }
370
+ }
371
+ },
372
+ "requiredRule": {
373
+ "type": "object",
374
+ "required": ["applies", "met", "reason"],
375
+ "properties": {
376
+ "applies": {
377
+ "type": "boolean"
378
+ },
379
+ "met": {
380
+ "type": "boolean"
381
+ },
382
+ "reason": {
383
+ "type": "string"
384
+ }
385
+ }
386
+ }
387
+ }
388
+ },
389
+ "reading": {
390
+ "type": "object",
391
+ "required": ["processes", "cases", "importCost"],
392
+ "properties": {
393
+ "processes": {
394
+ "type": "integer",
395
+ "minimum": 0
396
+ },
397
+ "cases": {
398
+ "type": "array",
399
+ "items": {
400
+ "$ref": "#/$defs/caseReading"
401
+ }
402
+ },
403
+ "importCost": {
404
+ "oneOf": [
405
+ {
406
+ "type": "null"
407
+ },
408
+ {
409
+ "type": "object",
410
+ "required": [
411
+ "runs",
412
+ "coldImportMs",
413
+ "medianMs",
414
+ "p95Ms",
415
+ "emptyImportMs",
416
+ "emptyMedianMs",
417
+ "files",
418
+ "totalBytes",
419
+ "totalGzipBytes"
420
+ ],
421
+ "properties": {
422
+ "runs": {
423
+ "type": "integer",
424
+ "minimum": 0
425
+ },
426
+ "coldImportMs": {
427
+ "type": "array",
428
+ "items": {
429
+ "type": "number"
430
+ }
431
+ },
432
+ "medianMs": {
433
+ "type": "number"
434
+ },
435
+ "p95Ms": {
436
+ "type": "number"
437
+ },
438
+ "emptyImportMs": {
439
+ "type": "array",
440
+ "items": {
441
+ "type": "number"
442
+ }
443
+ },
444
+ "emptyMedianMs": {
445
+ "type": "number"
446
+ },
447
+ "files": {
448
+ "type": "array",
449
+ "items": {
450
+ "type": "object",
451
+ "required": ["path", "bytes", "gzipBytes"],
452
+ "properties": {
453
+ "path": {
454
+ "type": "string"
455
+ },
456
+ "bytes": {
457
+ "type": "integer",
458
+ "minimum": 0
459
+ },
460
+ "gzipBytes": {
461
+ "type": "integer",
462
+ "minimum": 0
463
+ }
464
+ }
465
+ }
466
+ },
467
+ "totalBytes": {
468
+ "type": "integer",
469
+ "minimum": 0
470
+ },
471
+ "totalGzipBytes": {
472
+ "type": "integer",
473
+ "minimum": 0
474
+ }
475
+ }
476
+ }
477
+ ]
478
+ }
479
+ }
480
+ },
481
+ "caseReading": {
482
+ "type": "object",
483
+ "required": [
484
+ "benchmark",
485
+ "size",
486
+ "inputsPerPass",
487
+ "bytesPerPass",
488
+ "throughput",
489
+ "latency",
490
+ "allocations",
491
+ "peak",
492
+ "longRun"
493
+ ],
494
+ "properties": {
495
+ "benchmark": {
496
+ "type": "string"
497
+ },
498
+ "size": {
499
+ "type": "string"
500
+ },
501
+ "inputsPerPass": {
502
+ "type": "integer",
503
+ "minimum": 0
504
+ },
505
+ "bytesPerPass": {
506
+ "type": ["number", "null"]
507
+ },
508
+ "throughput": {
509
+ "oneOf": [
510
+ {
511
+ "type": "object",
512
+ "required": [
513
+ "status",
514
+ "processes",
515
+ "medianMsPerOp",
516
+ "opsPerSecond",
517
+ "bytesPerSecond"
518
+ ],
519
+ "properties": {
520
+ "status": {
521
+ "const": "measured"
522
+ },
523
+ "processes": {
524
+ "type": "array",
525
+ "items": {
526
+ "type": "object",
527
+ "required": ["samplesMsPerOp", "medianMsPerOp"],
528
+ "properties": {
529
+ "samplesMsPerOp": {
530
+ "type": "array",
531
+ "items": {
532
+ "type": "number"
533
+ }
534
+ },
535
+ "medianMsPerOp": {
536
+ "type": "number"
537
+ }
538
+ }
539
+ }
540
+ },
541
+ "medianMsPerOp": {
542
+ "type": "number"
543
+ },
544
+ "opsPerSecond": {
545
+ "type": "number"
546
+ },
547
+ "bytesPerSecond": {
548
+ "type": ["number", "null"]
549
+ }
550
+ }
551
+ },
552
+ {
553
+ "type": "object",
554
+ "required": ["status", "reason"],
555
+ "properties": {
556
+ "status": {
557
+ "const": "skipped"
558
+ },
559
+ "reason": {
560
+ "type": "string"
561
+ }
562
+ }
563
+ }
564
+ ]
565
+ },
566
+ "latency": {
567
+ "oneOf": [
568
+ {
569
+ "type": "null"
570
+ },
571
+ {
572
+ "type": "object",
573
+ "required": ["status", "calls", "timerOverheadNs", "p50Ns"],
574
+ "properties": {
575
+ "status": {
576
+ "enum": ["measured", "skipped"]
577
+ },
578
+ "calls": {
579
+ "type": "integer",
580
+ "minimum": 0
581
+ },
582
+ "timerOverheadNs": {
583
+ "type": "number"
584
+ },
585
+ "p50Ns": {
586
+ "type": ["number", "null"]
587
+ },
588
+ "p95Ns": {
589
+ "type": ["number", "null"]
590
+ },
591
+ "p99Ns": {
592
+ "type": ["number", "null"]
593
+ },
594
+ "maxNs": {
595
+ "type": "number"
596
+ }
597
+ }
598
+ }
599
+ ]
600
+ },
601
+ "allocations": {
602
+ "oneOf": [
603
+ {
604
+ "type": "null"
605
+ },
606
+ {
607
+ "oneOf": [
608
+ {
609
+ "type": "object",
610
+ "required": ["status", "windowOps", "heapBytesPerOp", "externalBytesPerOp"],
611
+ "properties": {
612
+ "status": {
613
+ "const": "measured"
614
+ },
615
+ "windowOps": {
616
+ "type": "integer",
617
+ "minimum": 0
618
+ },
619
+ "heapBytesPerOp": {
620
+ "type": "number"
621
+ },
622
+ "externalBytesPerOp": {
623
+ "type": "number"
624
+ }
625
+ }
626
+ },
627
+ {
628
+ "type": "object",
629
+ "required": ["status", "reason"],
630
+ "properties": {
631
+ "status": {
632
+ "const": "skipped"
633
+ },
634
+ "reason": {
635
+ "type": "string"
636
+ }
637
+ }
638
+ }
639
+ ]
640
+ }
641
+ ]
642
+ },
643
+ "peak": {
644
+ "oneOf": [
645
+ {
646
+ "type": "null"
647
+ },
648
+ {
649
+ "oneOf": [
650
+ {
651
+ "type": "object",
652
+ "required": ["status", "calls", "peakHeapBytes", "medianPeakHeapBytes"],
653
+ "properties": {
654
+ "status": {
655
+ "const": "measured"
656
+ },
657
+ "calls": {
658
+ "type": "integer",
659
+ "minimum": 0
660
+ },
661
+ "peakHeapBytes": {
662
+ "type": "number"
663
+ },
664
+ "medianPeakHeapBytes": {
665
+ "type": "number"
666
+ }
667
+ }
668
+ },
669
+ {
670
+ "type": "object",
671
+ "required": ["status", "reason"],
672
+ "properties": {
673
+ "status": {
674
+ "const": "skipped"
675
+ },
676
+ "reason": {
677
+ "type": "string"
678
+ }
679
+ }
680
+ }
681
+ ]
682
+ }
683
+ ]
684
+ },
685
+ "longRun": {
686
+ "oneOf": [
687
+ {
688
+ "type": "null"
689
+ },
690
+ {
691
+ "oneOf": [
692
+ {
693
+ "type": "object",
694
+ "required": [
695
+ "status",
696
+ "verdict",
697
+ "ops",
698
+ "opsPerRound",
699
+ "settledHeap",
700
+ "slopeBytesPerOp"
701
+ ],
702
+ "properties": {
703
+ "status": {
704
+ "const": "measured"
705
+ },
706
+ "verdict": {
707
+ "enum": ["flat", "growing"]
708
+ },
709
+ "ops": {
710
+ "type": "integer",
711
+ "minimum": 0
712
+ },
713
+ "opsPerRound": {
714
+ "type": "integer",
715
+ "minimum": 0
716
+ },
717
+ "settledHeap": {
718
+ "type": "array",
719
+ "items": {
720
+ "type": "number"
721
+ }
722
+ },
723
+ "slopeBytesPerOp": {
724
+ "type": "number"
725
+ }
726
+ }
727
+ },
728
+ {
729
+ "type": "object",
730
+ "required": ["status", "reason"],
731
+ "properties": {
732
+ "status": {
733
+ "const": "skipped"
734
+ },
735
+ "reason": {
736
+ "type": "string"
737
+ }
738
+ }
739
+ }
740
+ ]
741
+ }
742
+ ]
743
+ }
744
+ }
745
+ }
746
+ }
747
+ }