duel 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +135 -1
- data/README.md +5 -5
- data/docs/ci.md +145 -42
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +123 -35
- data/docs/json.md +18 -12
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +25 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/memory.rb +54 -0
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +383 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +85 -42
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/memory.rbs +30 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +129 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +3 -1
data/sig/duel/runner.rbs
CHANGED
|
@@ -20,6 +20,18 @@ module Duel
|
|
|
20
20
|
# @api private
|
|
21
21
|
TIME_LIMIT: Float
|
|
22
22
|
|
|
23
|
+
# The seconds a worker may take to load, or to answer, by default.
|
|
24
|
+
# @api private
|
|
25
|
+
TIMEOUT: Float
|
|
26
|
+
|
|
27
|
+
# The CPUs for each lane of workers, by default.
|
|
28
|
+
# @api private
|
|
29
|
+
CPUS_PER_LANE: Integer
|
|
30
|
+
|
|
31
|
+
# Worker processes of each version, by default.
|
|
32
|
+
# @api private
|
|
33
|
+
PROCESSES: Integer
|
|
34
|
+
|
|
23
35
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
24
36
|
# @api private
|
|
25
37
|
class Settings < Struct[String | Regexp | Integer | Float | bool | Config::ruby | Config::comment | Config::fail_on | nil]
|
|
@@ -50,17 +62,24 @@ module Duel
|
|
|
50
62
|
# The branch a pull request targets, on GitHub Actions
|
|
51
63
|
# (GITHUB_BASE_REF), on origin: see Baseline#main_branch.
|
|
52
64
|
attr_accessor target: String?
|
|
65
|
+
# On a schedule, the last commit since the last release that merged
|
|
66
|
+
# a pull request with Config#accept_label (DUEL_ACCEPTED): see
|
|
67
|
+
# Baseline#last_release.
|
|
68
|
+
attr_accessor accepted: String?
|
|
69
|
+
# Of jobs and processes, those Duel may lower to fit the memory
|
|
70
|
+
# available (see Runner#fit).
|
|
71
|
+
attr_accessor fit: Array[:jobs | :processes]
|
|
53
72
|
|
|
54
73
|
def self.new: (?String? base, ?String? base_dir, ?String? candidate_dir, ?Regexp filter, ?Integer jobs,
|
|
55
74
|
?Integer processes, ?Float limit, ?Float confidence, ?Float? warm, ?bool force, ?String? config_path,
|
|
56
75
|
?Float timeout, ?Config::ruby ruby, ?Config::comment? comment,
|
|
57
76
|
?Config::fail_on? fail_on, ?String? pushed,
|
|
58
|
-
?String? target, ?Float time_limit) -> instance
|
|
77
|
+
?String? target, ?Float time_limit, ?String? accepted, ?Array[:jobs | :processes] fit) -> instance
|
|
59
78
|
| (?base: String?, ?base_dir: String?, ?candidate_dir: String?, ?filter: Regexp, ?jobs: Integer,
|
|
60
79
|
?processes: Integer, ?limit: Float, ?confidence: Float, ?warm: Float?, ?force: bool,
|
|
61
80
|
?config_path: String?, ?timeout: Float, ?ruby: Config::ruby, ?comment: Config::comment?,
|
|
62
81
|
?fail_on: Config::fail_on?, ?pushed: String?,
|
|
63
|
-
?target: String?, ?time_limit: Float) -> instance
|
|
82
|
+
?target: String?, ?time_limit: Float, ?accepted: String?, ?fit: Array[:jobs | :processes]) -> instance
|
|
64
83
|
|
|
65
84
|
def self.defaults: () -> Settings
|
|
66
85
|
end
|
|
@@ -73,12 +92,14 @@ module Duel
|
|
|
73
92
|
attr_accessor root: String
|
|
74
93
|
attr_accessor ruby: String
|
|
75
94
|
attr_accessor env: Hash[String, String?]
|
|
95
|
+
# Its Ruby's engine (RUBY_ENGINE).
|
|
96
|
+
attr_accessor engine: String
|
|
76
97
|
|
|
77
98
|
# What each version's workers are named for: "base" and "head", as
|
|
78
99
|
# long as each other.
|
|
79
100
|
LABELS: Hash[String, String]
|
|
80
101
|
|
|
81
|
-
def self.new: (String name, String root, String ruby, Hash[String, String?] env) -> instance
|
|
102
|
+
def self.new: (String name, String root, String ruby, Hash[String, String?] env, String engine) -> instance
|
|
82
103
|
|
|
83
104
|
# Each version with nothing prepared: its tree, on the runner's Ruby.
|
|
84
105
|
def self.unprepared: (Config config, Settings settings, String base_root) -> Hash[String, Side]
|
|
@@ -146,7 +167,7 @@ module Duel
|
|
|
146
167
|
def self.load_path: (String ruby, ?specs: Hash[String, Gem::BasicSpecification]) -> Array[String]
|
|
147
168
|
|
|
148
169
|
def initialize: (String label, String root, String candidate_root, String dir, config_path: String?,
|
|
149
|
-
env: Hash[String, String?], timeout: Float, ruby: String) -> void
|
|
170
|
+
env: Hash[String, String?], timeout: Float, ruby: String, engine: String) -> void
|
|
150
171
|
|
|
151
172
|
# Sends a request, and returns the reply, or raises Failed with its
|
|
152
173
|
# error.
|
|
@@ -190,7 +211,14 @@ module Duel
|
|
|
190
211
|
HEAP_CAP: Integer
|
|
191
212
|
STEP: Integer
|
|
192
213
|
MAX_PAIRS: Integer
|
|
214
|
+
# The round of a sample from every worker from which a benchmark
|
|
215
|
+
# judged by process is decided.
|
|
216
|
+
MIN_ROUNDS: Integer
|
|
217
|
+
PROCESS_PAIRS: Integer
|
|
193
218
|
COUNTS: Integer
|
|
219
|
+
JRUBY_COUNTS: Integer
|
|
220
|
+
# The rounds of a sample from every worker a count by process takes.
|
|
221
|
+
COUNT_ROUNDS: Integer
|
|
194
222
|
MEMORY_SAMPLES: Integer
|
|
195
223
|
PROFILE_SAMPLES: Integer
|
|
196
224
|
# A process benchmark's pairs before any are measured, and its runs a
|
|
@@ -202,6 +230,11 @@ module Duel
|
|
|
202
230
|
# The longest TMPDIR a lane's directory goes in.
|
|
203
231
|
SHORT: Integer
|
|
204
232
|
|
|
233
|
+
self.@ids: Integer
|
|
234
|
+
self.@ids_lock: Thread::Mutex
|
|
235
|
+
|
|
236
|
+
# Its number (see .next_id).
|
|
237
|
+
@id: Integer
|
|
205
238
|
@dir: String
|
|
206
239
|
@resources: Array[Resource]
|
|
207
240
|
@workers: Array[WorkerProcess]
|
|
@@ -221,6 +254,13 @@ module Duel
|
|
|
221
254
|
|
|
222
255
|
def initialize: (Config config, Settings settings, Hash[String, Side] sides, ?processes: Integer) -> void
|
|
223
256
|
|
|
257
|
+
# A number of its own for each lane, from 1.
|
|
258
|
+
def self.next_id: () -> Integer
|
|
259
|
+
|
|
260
|
+
# Whether +processes+ workers of each version are enough for a
|
|
261
|
+
# +confidence+ interval over them.
|
|
262
|
+
def self.by_process?: (Integer processes, Float confidence) -> bool
|
|
263
|
+
|
|
224
264
|
# A directory for a lane's sockets.
|
|
225
265
|
def self.tmpdir: () -> String
|
|
226
266
|
|
|
@@ -281,8 +321,39 @@ module Duel
|
|
|
281
321
|
|
|
282
322
|
def warm: (String name, Integer n) -> void
|
|
283
323
|
|
|
324
|
+
# The seconds of samples whose median, no faster by STEADY_GAIN than
|
|
325
|
+
# the last's, end a worker's warm-up, without --warm.
|
|
326
|
+
STEADY: Float
|
|
327
|
+
STEADY_GAIN: Float
|
|
328
|
+
STEADY_WINDOWS: Integer
|
|
329
|
+
|
|
330
|
+
# Samples a worker until it's steady, or it's +warm_until+.
|
|
331
|
+
def steady: (WorkerProcess worker, String name, Integer n, Float warm_until) -> void
|
|
332
|
+
|
|
333
|
+
# STEADY seconds of a worker's samples' CPU time, and when they ended.
|
|
334
|
+
def window: (WorkerProcess worker, String name, Integer n, Float warm_until) -> [Array[Metric::value], Float]
|
|
335
|
+
|
|
284
336
|
def judge: (String name, Integer n, Array[[WorkerProcess::reply, WorkerProcess::reply]] floors, Result result) -> void
|
|
285
337
|
|
|
338
|
+
# Whether a benchmark's timings are judged by process.
|
|
339
|
+
def by_process?: (String name) -> bool
|
|
340
|
+
|
|
341
|
+
# Judges each of +judged+ over workers, in rounds of a sample from
|
|
342
|
+
# every worker.
|
|
343
|
+
def judge_processes: (String name, Integer n, Array[Metric::spec] judged, Result result) -> void
|
|
344
|
+
|
|
345
|
+
# The most rounds of +k+ workers' pairs.
|
|
346
|
+
def rounds: (Integer k) -> Integer
|
|
347
|
+
|
|
348
|
+
# Each worker's median, by version.
|
|
349
|
+
def medians: (Hash[String, Array[Array[Metric::value]]] values) -> Hash[String, Array[Metric::value]]
|
|
350
|
+
|
|
351
|
+
# The baseline's medians, at least 1, and the candidate's.
|
|
352
|
+
def clamped: (Hash[String, Array[Metric::value]] each) -> [Array[Metric::value], Array[Metric::value]]
|
|
353
|
+
|
|
354
|
+
# What each worker's medians say against the spec's limit.
|
|
355
|
+
def process_verdict: (Hash[String, Array[Metric::value]] each, Metric::spec spec) -> Stats::verdict
|
|
356
|
+
|
|
286
357
|
# Whether both versions' CPU time is below its floor.
|
|
287
358
|
def below_cpu_floor?: (Array[[Worker::sampled, Worker::sampled]] floors) -> bool
|
|
288
359
|
|
|
@@ -309,8 +380,22 @@ module Duel
|
|
|
309
380
|
# The +index+th pair: [baseline's reply, candidate's].
|
|
310
381
|
def pair: (String name, Integer n, Integer index, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
311
382
|
|
|
383
|
+
# A sample from a baseline worker and from a candidate worker: [baseline's reply, candidate's].
|
|
384
|
+
def sampled: (String name, Integer n, WorkerProcess base, WorkerProcess candidate, base_first: bool, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
385
|
+
|
|
312
386
|
def count: (String name, Integer n, Result result) -> void
|
|
313
387
|
|
|
388
|
+
# The samples a count takes.
|
|
389
|
+
def counts: () -> Integer
|
|
390
|
+
|
|
391
|
+
# Each counted measure, by each worker's fewest, and how much more
|
|
392
|
+
# the candidate's are at least.
|
|
393
|
+
def count_by_process: (String name, Integer n, Result result) -> void
|
|
394
|
+
|
|
395
|
+
# Keeps each count of a worker's sample where it's that worker's
|
|
396
|
+
# fewest so far.
|
|
397
|
+
def fewest!: (Hash[String, Hash[String, Hash[Integer, Metric::value]]] fewest, String side, Integer worker, WorkerProcess::reply sample) -> void
|
|
398
|
+
|
|
314
399
|
def peak_memory: (String name, Integer n, Result result) -> void
|
|
315
400
|
|
|
316
401
|
def fresh: [T] (String side) { (WorkerProcess worker) -> T } -> T
|
|
@@ -324,10 +409,14 @@ module Duel
|
|
|
324
409
|
# A ratio metric's verdict, and each version's (the median of its
|
|
325
410
|
# samples).
|
|
326
411
|
type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
|
|
327
|
-
"verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
412
|
+
?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
413
|
+
|
|
414
|
+
# Each worker's median of a measure judged by process, by version,
|
|
415
|
+
# and its lane's number.
|
|
416
|
+
type processes = { "lane" => Integer, "baseline" => Array[Metric::value], "candidate" => Array[Metric::value] }
|
|
328
417
|
|
|
329
418
|
# A counted metric's fewest, on each version, in +n+ iterations.
|
|
330
|
-
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, "n" => Integer,
|
|
419
|
+
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
|
|
331
420
|
"tolerance" => Metric::value, "allowance" => Metric::value, "unit" => String?, "plural" => String? }
|
|
332
421
|
|
|
333
422
|
# Where a regression's time went (see Profile#to_h), or why there's no
|
|
@@ -357,6 +446,7 @@ module Duel
|
|
|
357
446
|
@details: Hash[String, Hash[String, String]]
|
|
358
447
|
@confirmations: Array[Result]
|
|
359
448
|
@undecided: Array[Result]
|
|
449
|
+
@processes: Hash[String, processes]
|
|
360
450
|
@n: Integer?
|
|
361
451
|
@seconds: Float?
|
|
362
452
|
@skipped: String?
|
|
@@ -384,6 +474,10 @@ module Duel
|
|
|
384
474
|
# @api private
|
|
385
475
|
attr_reader undecided: Array[Result]
|
|
386
476
|
|
|
477
|
+
# Each worker's median of each measure judged by process, by metric.
|
|
478
|
+
# @api private
|
|
479
|
+
attr_reader processes: Hash[String, processes]
|
|
480
|
+
|
|
387
481
|
# Its iterations a sample.
|
|
388
482
|
attr_reader n: Integer?
|
|
389
483
|
|
|
@@ -475,6 +569,12 @@ module Duel
|
|
|
475
569
|
# Measures every benchmark, confirms what looks worse, and reports.
|
|
476
570
|
# @api private
|
|
477
571
|
class Runner
|
|
572
|
+
# A count to four figures, or a whole number from NUMBER.
|
|
573
|
+
def self.number: (Metric::value value) -> String
|
|
574
|
+
|
|
575
|
+
# The least count #number shows as a whole number.
|
|
576
|
+
NUMBER: Float
|
|
577
|
+
|
|
478
578
|
CONFIRM: Integer
|
|
479
579
|
|
|
480
580
|
# +number+ of +noun+, in its plural for any number but one.
|
|
@@ -544,10 +644,33 @@ module Duel
|
|
|
544
644
|
# Each version, prepared once for the run.
|
|
545
645
|
def prepare: () -> Hash[String, Side]
|
|
546
646
|
|
|
647
|
+
# What a benchmark's interval is over: the workers, or the pairs.
|
|
648
|
+
def interval_over: () -> String
|
|
649
|
+
|
|
650
|
+
# The most workers of each version #interval_over looks at.
|
|
651
|
+
PROCESSES_CHECKED: Integer
|
|
652
|
+
|
|
653
|
+
# The memory #fit leaves for everything but the workers.
|
|
654
|
+
RESERVE: Integer
|
|
655
|
+
|
|
656
|
+
# How much more than starting them a pair of workers takes, by #fit's
|
|
657
|
+
# reckoning.
|
|
658
|
+
HEADROOM: Float
|
|
659
|
+
|
|
660
|
+
# Lowers the jobs, then the processes, that Settings#fit lets it, to
|
|
661
|
+
# fit the memory available.
|
|
662
|
+
def fit: () -> void
|
|
663
|
+
|
|
664
|
+
# +bytes+ in gigabytes, to a tenth.
|
|
665
|
+
def gigabytes: (Integer bytes) -> String
|
|
666
|
+
|
|
547
667
|
def with_lanes: [T] () { (Array[Lane] lanes) -> T } -> T
|
|
548
668
|
|
|
549
669
|
def judge_across: (Array[Result] results, Array[String] names) -> Hash[String, Across::verdict]
|
|
550
670
|
|
|
671
|
+
# Across the board, of +results+.
|
|
672
|
+
def across_of: (Array[Result] results) -> Across
|
|
673
|
+
|
|
551
674
|
def open_lane: () -> Lane
|
|
552
675
|
|
|
553
676
|
def measure_all: (Array[Lane] lanes, Array[String] names) -> Array[Result]
|
data/sig/duel/stats.rbs
CHANGED
|
@@ -13,6 +13,31 @@ module Duel
|
|
|
13
13
|
# +samples+, or nil.
|
|
14
14
|
def self.median_interval: (Array[Float] samples, Float confidence) -> [Float, Float]?
|
|
15
15
|
|
|
16
|
+
# The rank r whose r-th smallest and largest cross ratio bound a
|
|
17
|
+
# +confidence+ interval for the ratio between two versions, from +m+
|
|
18
|
+
# and +n+ samples, or nil when they're too few.
|
|
19
|
+
def self.shift_rank: (Integer m, Integer n, Float confidence) -> Integer?
|
|
20
|
+
|
|
21
|
+
# P(U <= u), from 0 to m * n / 2, for the Mann-Whitney U statistic.
|
|
22
|
+
def self.u_lower_tail: (Integer m, Integer n) -> Array[Float]
|
|
23
|
+
|
|
24
|
+
# P(U = u) for u from 0 to +cap+, for samples of +i+ and +j+.
|
|
25
|
+
def self.u_probabilities: (Integer i, Integer j, Integer cap, Hash[[Integer, Integer], Array[Float]] found) -> Array[Float]
|
|
26
|
+
|
|
27
|
+
# [low, high] bounds of the +confidence+ interval for the ratio of
|
|
28
|
+
# +candidates+ to +bases+, or nil.
|
|
29
|
+
def self.shift_interval: (Array[Metric::value] bases, Array[Metric::value] candidates, Float confidence) -> [Float, Float]?
|
|
30
|
+
|
|
31
|
+
# The median of the cross ratios.
|
|
32
|
+
def self.shift: (Array[Metric::value] bases, Array[Metric::value] candidates) -> Float
|
|
33
|
+
|
|
34
|
+
# [low, high] bounds of the +confidence+ interval for how much more
|
|
35
|
+
# +candidates+ are than +bases+, or nil.
|
|
36
|
+
def self.difference_interval: (Array[Metric::value] bases, Array[Metric::value] candidates, Float confidence) -> [Metric::value, Metric::value]?
|
|
37
|
+
|
|
38
|
+
# Each of +candidates+ over each of +bases+, sorted.
|
|
39
|
+
def self.cross_ratios: (Array[Metric::value] bases, Array[Metric::value] candidates) -> Array[Float]
|
|
40
|
+
|
|
16
41
|
def self.median: (Array[Float] samples) -> Float
|
|
17
42
|
| (Array[Metric::value] samples) -> Metric::value
|
|
18
43
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: duel
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Erik Berlin
|
|
@@ -57,6 +57,7 @@ files:
|
|
|
57
57
|
- lib/duel/init.rb
|
|
58
58
|
- lib/duel/json_report.rb
|
|
59
59
|
- lib/duel/lock.rb
|
|
60
|
+
- lib/duel/memory.rb
|
|
60
61
|
- lib/duel/message.rb
|
|
61
62
|
- lib/duel/metric.rb
|
|
62
63
|
- lib/duel/native.rb
|
|
@@ -85,6 +86,7 @@ files:
|
|
|
85
86
|
- sig/duel/init.rbs
|
|
86
87
|
- sig/duel/json_report.rbs
|
|
87
88
|
- sig/duel/lock.rbs
|
|
89
|
+
- sig/duel/memory.rbs
|
|
88
90
|
- sig/duel/message.rbs
|
|
89
91
|
- sig/duel/metric.rbs
|
|
90
92
|
- sig/duel/native.rbs
|