duel 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/README.md +5 -5
- data/docs/ci.md +120 -25
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +109 -35
- data/docs/json.md +16 -10
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +18 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +337 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +71 -33
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +112 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +1 -1
data/sig/duel/runner.rbs
CHANGED
|
@@ -20,6 +20,18 @@ module Duel
|
|
|
20
20
|
# @api private
|
|
21
21
|
TIME_LIMIT: Float
|
|
22
22
|
|
|
23
|
+
# The seconds a worker may take to load, or to answer, by default.
|
|
24
|
+
# @api private
|
|
25
|
+
TIMEOUT: Float
|
|
26
|
+
|
|
27
|
+
# The CPUs for each lane of workers, by default.
|
|
28
|
+
# @api private
|
|
29
|
+
CPUS_PER_LANE: Integer
|
|
30
|
+
|
|
31
|
+
# Worker processes of each version, by default.
|
|
32
|
+
# @api private
|
|
33
|
+
PROCESSES: Integer
|
|
34
|
+
|
|
23
35
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
24
36
|
# @api private
|
|
25
37
|
class Settings < Struct[String | Regexp | Integer | Float | bool | Config::ruby | Config::comment | Config::fail_on | nil]
|
|
@@ -50,17 +62,21 @@ module Duel
|
|
|
50
62
|
# The branch a pull request targets, on GitHub Actions
|
|
51
63
|
# (GITHUB_BASE_REF), on origin: see Baseline#main_branch.
|
|
52
64
|
attr_accessor target: String?
|
|
65
|
+
# On a schedule, the last commit since the last release that merged
|
|
66
|
+
# a pull request with Config#accept_label (DUEL_ACCEPTED): see
|
|
67
|
+
# Baseline#last_release.
|
|
68
|
+
attr_accessor accepted: String?
|
|
53
69
|
|
|
54
70
|
def self.new: (?String? base, ?String? base_dir, ?String? candidate_dir, ?Regexp filter, ?Integer jobs,
|
|
55
71
|
?Integer processes, ?Float limit, ?Float confidence, ?Float? warm, ?bool force, ?String? config_path,
|
|
56
72
|
?Float timeout, ?Config::ruby ruby, ?Config::comment? comment,
|
|
57
73
|
?Config::fail_on? fail_on, ?String? pushed,
|
|
58
|
-
?String? target, ?Float time_limit) -> instance
|
|
74
|
+
?String? target, ?Float time_limit, ?String? accepted) -> instance
|
|
59
75
|
| (?base: String?, ?base_dir: String?, ?candidate_dir: String?, ?filter: Regexp, ?jobs: Integer,
|
|
60
76
|
?processes: Integer, ?limit: Float, ?confidence: Float, ?warm: Float?, ?force: bool,
|
|
61
77
|
?config_path: String?, ?timeout: Float, ?ruby: Config::ruby, ?comment: Config::comment?,
|
|
62
78
|
?fail_on: Config::fail_on?, ?pushed: String?,
|
|
63
|
-
?target: String?, ?time_limit: Float) -> instance
|
|
79
|
+
?target: String?, ?time_limit: Float, ?accepted: String?) -> instance
|
|
64
80
|
|
|
65
81
|
def self.defaults: () -> Settings
|
|
66
82
|
end
|
|
@@ -73,12 +89,14 @@ module Duel
|
|
|
73
89
|
attr_accessor root: String
|
|
74
90
|
attr_accessor ruby: String
|
|
75
91
|
attr_accessor env: Hash[String, String?]
|
|
92
|
+
# Its Ruby's engine (RUBY_ENGINE).
|
|
93
|
+
attr_accessor engine: String
|
|
76
94
|
|
|
77
95
|
# What each version's workers are named for: "base" and "head", as
|
|
78
96
|
# long as each other.
|
|
79
97
|
LABELS: Hash[String, String]
|
|
80
98
|
|
|
81
|
-
def self.new: (String name, String root, String ruby, Hash[String, String?] env) -> instance
|
|
99
|
+
def self.new: (String name, String root, String ruby, Hash[String, String?] env, String engine) -> instance
|
|
82
100
|
|
|
83
101
|
# Each version with nothing prepared: its tree, on the runner's Ruby.
|
|
84
102
|
def self.unprepared: (Config config, Settings settings, String base_root) -> Hash[String, Side]
|
|
@@ -146,7 +164,7 @@ module Duel
|
|
|
146
164
|
def self.load_path: (String ruby, ?specs: Hash[String, Gem::BasicSpecification]) -> Array[String]
|
|
147
165
|
|
|
148
166
|
def initialize: (String label, String root, String candidate_root, String dir, config_path: String?,
|
|
149
|
-
env: Hash[String, String?], timeout: Float, ruby: String) -> void
|
|
167
|
+
env: Hash[String, String?], timeout: Float, ruby: String, engine: String) -> void
|
|
150
168
|
|
|
151
169
|
# Sends a request, and returns the reply, or raises Failed with its
|
|
152
170
|
# error.
|
|
@@ -190,7 +208,14 @@ module Duel
|
|
|
190
208
|
HEAP_CAP: Integer
|
|
191
209
|
STEP: Integer
|
|
192
210
|
MAX_PAIRS: Integer
|
|
211
|
+
# The round of a sample from every worker from which a benchmark
|
|
212
|
+
# judged by process is decided.
|
|
213
|
+
MIN_ROUNDS: Integer
|
|
214
|
+
PROCESS_PAIRS: Integer
|
|
193
215
|
COUNTS: Integer
|
|
216
|
+
JRUBY_COUNTS: Integer
|
|
217
|
+
# The rounds of a sample from every worker a count by process takes.
|
|
218
|
+
COUNT_ROUNDS: Integer
|
|
194
219
|
MEMORY_SAMPLES: Integer
|
|
195
220
|
PROFILE_SAMPLES: Integer
|
|
196
221
|
# A process benchmark's pairs before any are measured, and its runs a
|
|
@@ -202,6 +227,11 @@ module Duel
|
|
|
202
227
|
# The longest TMPDIR a lane's directory goes in.
|
|
203
228
|
SHORT: Integer
|
|
204
229
|
|
|
230
|
+
self.@ids: Integer
|
|
231
|
+
self.@ids_lock: Thread::Mutex
|
|
232
|
+
|
|
233
|
+
# Its number (see .next_id).
|
|
234
|
+
@id: Integer
|
|
205
235
|
@dir: String
|
|
206
236
|
@resources: Array[Resource]
|
|
207
237
|
@workers: Array[WorkerProcess]
|
|
@@ -221,6 +251,13 @@ module Duel
|
|
|
221
251
|
|
|
222
252
|
def initialize: (Config config, Settings settings, Hash[String, Side] sides, ?processes: Integer) -> void
|
|
223
253
|
|
|
254
|
+
# A number of its own for each lane, from 1.
|
|
255
|
+
def self.next_id: () -> Integer
|
|
256
|
+
|
|
257
|
+
# Whether +processes+ workers of each version are enough for a
|
|
258
|
+
# +confidence+ interval over them.
|
|
259
|
+
def self.by_process?: (Integer processes, Float confidence) -> bool
|
|
260
|
+
|
|
224
261
|
# A directory for a lane's sockets.
|
|
225
262
|
def self.tmpdir: () -> String
|
|
226
263
|
|
|
@@ -281,8 +318,39 @@ module Duel
|
|
|
281
318
|
|
|
282
319
|
def warm: (String name, Integer n) -> void
|
|
283
320
|
|
|
321
|
+
# The seconds of samples whose median, no faster by STEADY_GAIN than
|
|
322
|
+
# the last's, end a worker's warm-up, without --warm.
|
|
323
|
+
STEADY: Float
|
|
324
|
+
STEADY_GAIN: Float
|
|
325
|
+
STEADY_WINDOWS: Integer
|
|
326
|
+
|
|
327
|
+
# Samples a worker until it's steady, or it's +warm_until+.
|
|
328
|
+
def steady: (WorkerProcess worker, String name, Integer n, Float warm_until) -> void
|
|
329
|
+
|
|
330
|
+
# STEADY seconds of a worker's samples' CPU time, and when they ended.
|
|
331
|
+
def window: (WorkerProcess worker, String name, Integer n, Float warm_until) -> [Array[Metric::value], Float]
|
|
332
|
+
|
|
284
333
|
def judge: (String name, Integer n, Array[[WorkerProcess::reply, WorkerProcess::reply]] floors, Result result) -> void
|
|
285
334
|
|
|
335
|
+
# Whether a benchmark's timings are judged by process.
|
|
336
|
+
def by_process?: (String name) -> bool
|
|
337
|
+
|
|
338
|
+
# Judges each of +judged+ over workers, in rounds of a sample from
|
|
339
|
+
# every worker.
|
|
340
|
+
def judge_processes: (String name, Integer n, Array[Metric::spec] judged, Result result) -> void
|
|
341
|
+
|
|
342
|
+
# The most rounds of +k+ workers' pairs.
|
|
343
|
+
def rounds: (Integer k) -> Integer
|
|
344
|
+
|
|
345
|
+
# Each worker's median, by version.
|
|
346
|
+
def medians: (Hash[String, Array[Array[Metric::value]]] values) -> Hash[String, Array[Metric::value]]
|
|
347
|
+
|
|
348
|
+
# The baseline's medians, at least 1, and the candidate's.
|
|
349
|
+
def clamped: (Hash[String, Array[Metric::value]] each) -> [Array[Metric::value], Array[Metric::value]]
|
|
350
|
+
|
|
351
|
+
# What each worker's medians say against the spec's limit.
|
|
352
|
+
def process_verdict: (Hash[String, Array[Metric::value]] each, Metric::spec spec) -> Stats::verdict
|
|
353
|
+
|
|
286
354
|
# Whether both versions' CPU time is below its floor.
|
|
287
355
|
def below_cpu_floor?: (Array[[Worker::sampled, Worker::sampled]] floors) -> bool
|
|
288
356
|
|
|
@@ -309,8 +377,22 @@ module Duel
|
|
|
309
377
|
# The +index+th pair: [baseline's reply, candidate's].
|
|
310
378
|
def pair: (String name, Integer n, Integer index, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
311
379
|
|
|
380
|
+
# A sample from a baseline worker and from a candidate worker: [baseline's reply, candidate's].
|
|
381
|
+
def sampled: (String name, Integer n, WorkerProcess base, WorkerProcess candidate, base_first: bool, ?runs: Integer, ?count: bool) -> [WorkerProcess::reply, WorkerProcess::reply]
|
|
382
|
+
|
|
312
383
|
def count: (String name, Integer n, Result result) -> void
|
|
313
384
|
|
|
385
|
+
# The samples a count takes.
|
|
386
|
+
def counts: () -> Integer
|
|
387
|
+
|
|
388
|
+
# Each counted measure, by each worker's fewest, and how much more
|
|
389
|
+
# the candidate's are at least.
|
|
390
|
+
def count_by_process: (String name, Integer n, Result result) -> void
|
|
391
|
+
|
|
392
|
+
# Keeps each count of a worker's sample where it's that worker's
|
|
393
|
+
# fewest so far.
|
|
394
|
+
def fewest!: (Hash[String, Hash[String, Hash[Integer, Metric::value]]] fewest, String side, Integer worker, WorkerProcess::reply sample) -> void
|
|
395
|
+
|
|
314
396
|
def peak_memory: (String name, Integer n, Result result) -> void
|
|
315
397
|
|
|
316
398
|
def fresh: [T] (String side) { (WorkerProcess worker) -> T } -> T
|
|
@@ -324,10 +406,14 @@ module Duel
|
|
|
324
406
|
# A ratio metric's verdict, and each version's (the median of its
|
|
325
407
|
# samples).
|
|
326
408
|
type timed = { "median" => Float, "limit" => Float, "across" => bool, "unit" => String?, "plural" => String?, "interval" => Array[Float]?, "pairs" => Integer,
|
|
327
|
-
"verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
409
|
+
?"processes" => Integer, "verdict" => String, "baseline" => Metric::value, "candidate" => Metric::value }
|
|
410
|
+
|
|
411
|
+
# Each worker's median of a measure judged by process, by version,
|
|
412
|
+
# and its lane's number.
|
|
413
|
+
type processes = { "lane" => Integer, "baseline" => Array[Metric::value], "candidate" => Array[Metric::value] }
|
|
328
414
|
|
|
329
415
|
# A counted metric's fewest, on each version, in +n+ iterations.
|
|
330
|
-
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, "n" => Integer,
|
|
416
|
+
type counted = { "baseline" => Metric::value, "candidate" => Metric::value, ?"more" => Metric::value?, "n" => Integer,
|
|
331
417
|
"tolerance" => Metric::value, "allowance" => Metric::value, "unit" => String?, "plural" => String? }
|
|
332
418
|
|
|
333
419
|
# Where a regression's time went (see Profile#to_h), or why there's no
|
|
@@ -357,6 +443,7 @@ module Duel
|
|
|
357
443
|
@details: Hash[String, Hash[String, String]]
|
|
358
444
|
@confirmations: Array[Result]
|
|
359
445
|
@undecided: Array[Result]
|
|
446
|
+
@processes: Hash[String, processes]
|
|
360
447
|
@n: Integer?
|
|
361
448
|
@seconds: Float?
|
|
362
449
|
@skipped: String?
|
|
@@ -384,6 +471,10 @@ module Duel
|
|
|
384
471
|
# @api private
|
|
385
472
|
attr_reader undecided: Array[Result]
|
|
386
473
|
|
|
474
|
+
# Each worker's median of each measure judged by process, by metric.
|
|
475
|
+
# @api private
|
|
476
|
+
attr_reader processes: Hash[String, processes]
|
|
477
|
+
|
|
387
478
|
# Its iterations a sample.
|
|
388
479
|
attr_reader n: Integer?
|
|
389
480
|
|
|
@@ -475,6 +566,12 @@ module Duel
|
|
|
475
566
|
# Measures every benchmark, confirms what looks worse, and reports.
|
|
476
567
|
# @api private
|
|
477
568
|
class Runner
|
|
569
|
+
# A count to four figures, or a whole number from NUMBER.
|
|
570
|
+
def self.number: (Metric::value value) -> String
|
|
571
|
+
|
|
572
|
+
# The least count #number shows as a whole number.
|
|
573
|
+
NUMBER: Float
|
|
574
|
+
|
|
478
575
|
CONFIRM: Integer
|
|
479
576
|
|
|
480
577
|
# +number+ of +noun+, in its plural for any number but one.
|
|
@@ -544,10 +641,19 @@ module Duel
|
|
|
544
641
|
# Each version, prepared once for the run.
|
|
545
642
|
def prepare: () -> Hash[String, Side]
|
|
546
643
|
|
|
644
|
+
# What a benchmark's interval is over: the workers, or the pairs.
|
|
645
|
+
def interval_over: () -> String
|
|
646
|
+
|
|
647
|
+
# The most workers of each version #interval_over looks at.
|
|
648
|
+
PROCESSES_CHECKED: Integer
|
|
649
|
+
|
|
547
650
|
def with_lanes: [T] () { (Array[Lane] lanes) -> T } -> T
|
|
548
651
|
|
|
549
652
|
def judge_across: (Array[Result] results, Array[String] names) -> Hash[String, Across::verdict]
|
|
550
653
|
|
|
654
|
+
# Across the board, of +results+.
|
|
655
|
+
def across_of: (Array[Result] results) -> Across
|
|
656
|
+
|
|
551
657
|
def open_lane: () -> Lane
|
|
552
658
|
|
|
553
659
|
def measure_all: (Array[Lane] lanes, Array[String] names) -> Array[Result]
|
data/sig/duel/stats.rbs
CHANGED
|
@@ -13,6 +13,31 @@ module Duel
|
|
|
13
13
|
# +samples+, or nil.
|
|
14
14
|
def self.median_interval: (Array[Float] samples, Float confidence) -> [Float, Float]?
|
|
15
15
|
|
|
16
|
+
# The rank r whose r-th smallest and largest cross ratio bound a
|
|
17
|
+
# +confidence+ interval for the ratio between two versions, from +m+
|
|
18
|
+
# and +n+ samples, or nil when they're too few.
|
|
19
|
+
def self.shift_rank: (Integer m, Integer n, Float confidence) -> Integer?
|
|
20
|
+
|
|
21
|
+
# P(U <= u), from 0 to m * n / 2, for the Mann-Whitney U statistic.
|
|
22
|
+
def self.u_lower_tail: (Integer m, Integer n) -> Array[Float]
|
|
23
|
+
|
|
24
|
+
# P(U = u) for u from 0 to +cap+, for samples of +i+ and +j+.
|
|
25
|
+
def self.u_probabilities: (Integer i, Integer j, Integer cap, Hash[[Integer, Integer], Array[Float]] found) -> Array[Float]
|
|
26
|
+
|
|
27
|
+
# [low, high] bounds of the +confidence+ interval for the ratio of
|
|
28
|
+
# +candidates+ to +bases+, or nil.
|
|
29
|
+
def self.shift_interval: (Array[Metric::value] bases, Array[Metric::value] candidates, Float confidence) -> [Float, Float]?
|
|
30
|
+
|
|
31
|
+
# The median of the cross ratios.
|
|
32
|
+
def self.shift: (Array[Metric::value] bases, Array[Metric::value] candidates) -> Float
|
|
33
|
+
|
|
34
|
+
# [low, high] bounds of the +confidence+ interval for how much more
|
|
35
|
+
# +candidates+ are than +bases+, or nil.
|
|
36
|
+
def self.difference_interval: (Array[Metric::value] bases, Array[Metric::value] candidates, Float confidence) -> [Metric::value, Metric::value]?
|
|
37
|
+
|
|
38
|
+
# Each of +candidates+ over each of +bases+, sorted.
|
|
39
|
+
def self.cross_ratios: (Array[Metric::value] bases, Array[Metric::value] candidates) -> Array[Float]
|
|
40
|
+
|
|
16
41
|
def self.median: (Array[Float] samples) -> Float
|
|
17
42
|
| (Array[Metric::value] samples) -> Metric::value
|
|
18
43
|
|