duel 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +135 -1
- data/README.md +5 -5
- data/docs/ci.md +145 -42
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +123 -35
- data/docs/json.md +18 -12
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +25 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/memory.rb +54 -0
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +383 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +85 -42
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/memory.rbs +30 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +129 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +3 -1
data/lib/duel/runner.rb
CHANGED
|
@@ -12,32 +12,62 @@ require_relative "message"
|
|
|
12
12
|
require_relative "resource"
|
|
13
13
|
require_relative "rubies"
|
|
14
14
|
require_relative "process_sample"
|
|
15
|
+
require_relative "memory"
|
|
15
16
|
|
|
16
17
|
module Duel
|
|
17
|
-
#
|
|
18
|
-
# says:
|
|
18
|
+
# The most seconds each benchmark runs before it's measured, unless
|
|
19
|
+
# --warm says: until each worker is steady (see Lane#steady), as long
|
|
20
|
+
# as that takes. JRuby's and TruffleRuby's JITs keep compiling for
|
|
21
|
+
# seconds, and some code is still several times faster at 5 than it
|
|
22
|
+
# was, so they get up to a minute.
|
|
19
23
|
# @api private
|
|
20
|
-
JIT_WARM = {"jruby" =>
|
|
24
|
+
JIT_WARM = {"jruby" => 60.0, "truffleruby" => 60.0}.fetch(RUBY_ENGINE, 0.0)
|
|
21
25
|
|
|
22
|
-
#
|
|
23
|
-
# YJIT on (an app may turn it on as it boots, as Rails
|
|
24
|
-
# --warm says. A small library's methods are compiled
|
|
25
|
-
# warm-up samples, but a big app's only after 120 calls
|
|
26
|
-
# a slow benchmark's warm-up samples make.
|
|
26
|
+
# The most seconds each benchmark runs before it's measured where a
|
|
27
|
+
# worker has YJIT on (an app may turn it on as it boots, as Rails
|
|
28
|
+
# does), unless --warm says. A small library's methods are compiled
|
|
29
|
+
# within the warm-up samples, but a big app's only after 120 calls
|
|
30
|
+
# each, more than a slow benchmark's warm-up samples make.
|
|
27
31
|
# @api private
|
|
28
|
-
YJIT_WARM =
|
|
32
|
+
YJIT_WARM = 10.0
|
|
33
|
+
|
|
34
|
+
# Worker processes of each version, by default: enough for an interval
|
|
35
|
+
# over processes (see Lane#judge_processes), which takes 5 at 99%, and
|
|
36
|
+
# more, as it narrows with each. JRuby's and TruffleRuby's are JVMs,
|
|
37
|
+
# slow to start and big (some 500 MB), so they get fewer.
|
|
38
|
+
# @api private
|
|
39
|
+
PROCESSES = {"jruby" => 8, "truffleruby" => 8}.fetch(RUBY_ENGINE, 12)
|
|
40
|
+
|
|
41
|
+
# The CPUs for each lane of workers, by default (see CLI#parse): half
|
|
42
|
+
# the CPUs make lanes, so that each lane's pair of workers has a CPU of
|
|
43
|
+
# its own. JRuby's and TruffleRuby's JIT compilers keep other CPUs busy,
|
|
44
|
+
# and each of their workers is a big JVM, so they get a lane for every
|
|
45
|
+
# four: more lanes are no faster there, and take more memory.
|
|
46
|
+
# @api private
|
|
47
|
+
CPUS_PER_LANE = {"jruby" => 4, "truffleruby" => 4}.fetch(RUBY_ENGINE, 2)
|
|
48
|
+
|
|
49
|
+
# The seconds a worker may take to load, or to answer a request, by
|
|
50
|
+
# default (see Settings#timeout). JRuby's and TruffleRuby's JVMs take a
|
|
51
|
+
# while to start, and with -Xbatch (see Rubies::JRUBY_WORKER), they
|
|
52
|
+
# compile as they load too: 16 at once can take over a minute.
|
|
53
|
+
# @api private
|
|
54
|
+
TIMEOUT = {"jruby" => 300.0, "truffleruby" => 300.0}.fetch(RUBY_ENGINE, 60.0)
|
|
29
55
|
|
|
30
56
|
# The seconds a run measures for, at most, by default (see
|
|
31
|
-
# Settings#time_limit and Runner#out_of_time?)
|
|
32
|
-
|
|
57
|
+
# Settings#time_limit and Runner#out_of_time?): two hours, as on JRuby,
|
|
58
|
+
# a benchmark can take half a minute to warm up.
|
|
59
|
+
TIME_LIMIT = 7200.0
|
|
33
60
|
|
|
34
61
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
35
62
|
# @api private
|
|
63
|
+
# Of +jobs+ and +processes+, +fit+ names those Duel may lower to fit
|
|
64
|
+
# the memory available (see Runner#fit): the defaults, but not what
|
|
65
|
+
# an option set.
|
|
36
66
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
37
|
-
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
|
|
67
|
+
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit, :accepted, :fit) do
|
|
38
68
|
# @type self: singleton(Settings)
|
|
39
|
-
def self.defaults = new(filter: //, jobs: 1, processes:
|
|
40
|
-
timeout:
|
|
69
|
+
def self.defaults = new(filter: //, jobs: 1, processes: PROCESSES, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
70
|
+
timeout: TIMEOUT, time_limit: TIME_LIMIT, fit: [])
|
|
41
71
|
end
|
|
42
72
|
|
|
43
73
|
# One version as a run has it (see Runner#prepare), +name+d "baseline"
|
|
@@ -45,13 +75,13 @@ module Duel
|
|
|
45
75
|
# (Config#ruby), and what they get in their environment (that Ruby's,
|
|
46
76
|
# and what its Config#prepare(:version) blocks gave it).
|
|
47
77
|
# @api private
|
|
48
|
-
Side = Struct.new(:name, :root, :ruby, :env) do
|
|
78
|
+
Side = Struct.new(:name, :root, :ruby, :env, :engine) do
|
|
49
79
|
# @type self: singleton(Side)
|
|
50
80
|
|
|
51
81
|
# Each version with nothing prepared: its tree, on the runner's Ruby.
|
|
52
82
|
def self.unprepared(config, settings, base_root)
|
|
53
|
-
{"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}),
|
|
54
|
-
"candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {})}
|
|
83
|
+
{"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}, RUBY_ENGINE),
|
|
84
|
+
"candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {}, RUBY_ENGINE)}
|
|
55
85
|
end
|
|
56
86
|
end
|
|
57
87
|
|
|
@@ -104,9 +134,12 @@ module Duel
|
|
|
104
134
|
end
|
|
105
135
|
|
|
106
136
|
# +timeout+: the seconds it may take to load, or to answer a request,
|
|
107
|
-
# before it's taken for hung (and killed). It runs on +ruby
|
|
108
|
-
# +env
|
|
109
|
-
|
|
137
|
+
# before it's taken for hung (and killed). It runs on +ruby+ (whose
|
|
138
|
+
# engine is +engine+), with +env+, and on JRuby, the JVM's options for
|
|
139
|
+
# a worker (see Rubies.worker_environment), not given to the processes
|
|
140
|
+
# it starts for a process benchmark (see #run): their boot is measured
|
|
141
|
+
# as it is.
|
|
142
|
+
def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:)
|
|
110
143
|
@label = label
|
|
111
144
|
@timeout = timeout
|
|
112
145
|
@root = root
|
|
@@ -118,7 +151,7 @@ module Duel
|
|
|
118
151
|
# (A nil config_path is left out. And the worker gets no other
|
|
119
152
|
# worker's connection: TruffleRuby's sockets aren't close-on-exec,
|
|
120
153
|
# and one left open in another worker would keep it from ending.)
|
|
121
|
-
@pid = Process.spawn(env, ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
|
|
154
|
+
@pid = Process.spawn(Rubies.worker_environment(env, engine), ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
|
|
122
155
|
"-e", "Duel::Worker.main(*ARGV)", root, candidate_root, control, *[config_path].compact, in: File::NULL, close_others: true)
|
|
123
156
|
@socket = accept(server, control)
|
|
124
157
|
hello = read
|
|
@@ -210,12 +243,13 @@ module Duel
|
|
|
210
243
|
# a worker that died or hung.
|
|
211
244
|
class Failed < Error; end
|
|
212
245
|
|
|
213
|
-
# Workers of both versions, taking turns:
|
|
214
|
-
#
|
|
215
|
-
# the very same code a few percent faster or slower for that
|
|
216
|
-
# whole life
|
|
217
|
-
#
|
|
218
|
-
# its own resources
|
|
246
|
+
# Workers of both versions, taking turns: Settings#processes of each
|
|
247
|
+
# (PROCESSES, by default). Where things landed in one process's memory
|
|
248
|
+
# can make the very same code a few percent faster or slower for that
|
|
249
|
+
# process's whole life, so the interval is over the workers, where
|
|
250
|
+
# they're enough for one (see #judge_processes), and not over the
|
|
251
|
+
# pairs, which share them. Each worker has its own resources
|
|
252
|
+
# (Config#prepare(:worker)).
|
|
219
253
|
# @api private
|
|
220
254
|
class Lane
|
|
221
255
|
# CPU time (or a plugin's ratio metric's equivalent) a run aims for:
|
|
@@ -238,8 +272,20 @@ module Duel
|
|
|
238
272
|
# process benchmark's: see #judge).
|
|
239
273
|
STEP = 10
|
|
240
274
|
MAX_PAIRS = 60
|
|
241
|
-
#
|
|
275
|
+
# A benchmark judged by process (see #judge_processes) is decided from
|
|
276
|
+
# its MIN_ROUNDS-th round of a sample from every worker, a sample
|
|
277
|
+
# each takes, and up to PROCESS_PAIRS pairs.
|
|
278
|
+
MIN_ROUNDS = 4
|
|
279
|
+
PROCESS_PAIRS = 120
|
|
280
|
+
# Counts are each version's fewest in COUNTS samples, or where a
|
|
281
|
+
# version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
|
|
282
|
+
# takes one of two values, some 31 bytes an iteration apart, sample by
|
|
283
|
+
# sample, and the fewest of more are the lower one.
|
|
242
284
|
COUNTS = 3
|
|
285
|
+
JRUBY_COUNTS = 10
|
|
286
|
+
# A count by process (see #count_by_process) is each worker's fewest
|
|
287
|
+
# in COUNT_ROUNDS rounds of a sample from every worker.
|
|
288
|
+
COUNT_ROUNDS = 3
|
|
243
289
|
# Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
|
|
244
290
|
MEMORY_SAMPLES = 3
|
|
245
291
|
# A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
|
|
@@ -252,6 +298,7 @@ module Duel
|
|
|
252
298
|
# server takes seconds), and stop at once. +sides+ are the versions as
|
|
253
299
|
# the run prepared them (Runner#prepare), by name.
|
|
254
300
|
def initialize(config, settings, sides, processes: settings.processes)
|
|
301
|
+
@id = Lane.next_id
|
|
255
302
|
@dir = Lane.tmpdir
|
|
256
303
|
@resources = []
|
|
257
304
|
@workers = []
|
|
@@ -271,6 +318,18 @@ module Duel
|
|
|
271
318
|
raise
|
|
272
319
|
end
|
|
273
320
|
|
|
321
|
+
@ids = 0
|
|
322
|
+
@ids_lock = Mutex.new
|
|
323
|
+
|
|
324
|
+
# A number of its own for each lane, from 1: what tells one lane's
|
|
325
|
+
# workers from another's (see Result#processes).
|
|
326
|
+
# mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
|
|
327
|
+
def self.next_id = @ids_lock.synchronize { @ids += 1 }
|
|
328
|
+
|
|
329
|
+
# Whether +processes+ workers of each version are enough for a
|
|
330
|
+
# +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
|
|
331
|
+
def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
|
|
332
|
+
|
|
274
333
|
# A directory for a lane's sockets, short as a Unix socket's path must
|
|
275
334
|
# be (see .short_tmp).
|
|
276
335
|
def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
|
|
@@ -355,7 +414,7 @@ module Duel
|
|
|
355
414
|
env = Resource.environment(label, env, block.call(resource))
|
|
356
415
|
end
|
|
357
416
|
worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
|
|
358
|
-
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby)
|
|
417
|
+
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
|
|
359
418
|
keep(@workers, worker)
|
|
360
419
|
worker
|
|
361
420
|
end
|
|
@@ -506,30 +565,89 @@ module Duel
|
|
|
506
565
|
PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
|
|
507
566
|
end
|
|
508
567
|
|
|
509
|
-
# Every worker runs the benchmark a few times, and
|
|
510
|
-
#
|
|
511
|
-
#
|
|
568
|
+
# Every worker runs the benchmark a few times, and then, all at once
|
|
569
|
+
# (none of it is measured, so they needn't take turns), for the
|
|
570
|
+
# seconds --warm says (Settings#warm), or else until it's steady (see
|
|
571
|
+
# #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
|
|
512
572
|
def warm(name, n)
|
|
513
573
|
warm_until = monotonic + @warm
|
|
514
574
|
in_parallel(@bases + @candidates) do |worker|
|
|
515
575
|
2.times { worker.sample(name, n) }
|
|
516
|
-
|
|
576
|
+
if @settings.warm
|
|
577
|
+
worker.sample(name, n) until monotonic > warm_until
|
|
578
|
+
else
|
|
579
|
+
steady(worker, name, n, warm_until)
|
|
580
|
+
end
|
|
581
|
+
end
|
|
582
|
+
end
|
|
583
|
+
|
|
584
|
+
# A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
|
|
585
|
+
# seconds in a row each have a median of its samples no faster, by
|
|
586
|
+
# STEADY_GAIN, than the last window that was (so code that creeps
|
|
587
|
+
# faster by less a window isn't steady): a JIT has compiled what it
|
|
588
|
+
# will. (Not once its fastest sample stops getting
|
|
589
|
+
# faster: the more samples, the faster the fastest of them, by
|
|
590
|
+
# chance. And not once one window is no faster: a pause, a collection
|
|
591
|
+
# say, can slow one.) A small library's code is steady in a few
|
|
592
|
+
# seconds, where a big app's JIT keeps compiling, for up to the
|
|
593
|
+
# warm-up's seconds.
|
|
594
|
+
STEADY = 0.5
|
|
595
|
+
STEADY_GAIN = 0.01
|
|
596
|
+
STEADY_WINDOWS = 3
|
|
597
|
+
|
|
598
|
+
# Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
|
|
599
|
+
# until it's steady, or it's +warm_until+.
|
|
600
|
+
def steady(worker, name, n, warm_until)
|
|
601
|
+
window, now = window(worker, name, n, warm_until)
|
|
602
|
+
return if now > warm_until
|
|
603
|
+
|
|
604
|
+
fastest = Stats.median(window)
|
|
605
|
+
calm = 0
|
|
606
|
+
loop do
|
|
607
|
+
window, now = window(worker, name, n, warm_until)
|
|
608
|
+
return if now > warm_until
|
|
609
|
+
|
|
610
|
+
median = Stats.median(window)
|
|
611
|
+
if median > fastest * (1 - STEADY_GAIN)
|
|
612
|
+
calm += 1
|
|
613
|
+
return if calm == STEADY_WINDOWS
|
|
614
|
+
else
|
|
615
|
+
calm = 0
|
|
616
|
+
fastest = median
|
|
617
|
+
end
|
|
618
|
+
end
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
# STEADY seconds of +worker+'s samples' CPU time (a sample at least),
|
|
622
|
+
# or until +warm_until+, and when they ended.
|
|
623
|
+
def window(worker, name, n, warm_until)
|
|
624
|
+
window_until = monotonic + STEADY
|
|
625
|
+
samples = [] #: Array[Metric::value]
|
|
626
|
+
loop do
|
|
627
|
+
samples << worker.sample(name, n).fetch("timed").fetch("cpu")
|
|
628
|
+
now = monotonic
|
|
629
|
+
return [samples, now] if now > window_until || now > warm_until
|
|
517
630
|
end
|
|
518
631
|
end
|
|
519
632
|
|
|
520
633
|
# Takes pairs until every ratio metric worth judging (measured on
|
|
521
634
|
# both versions in the warm-up, and above its floors on either: one
|
|
522
635
|
# that rises from next to nothing is judged too) is decided, or
|
|
523
|
-
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs
|
|
524
|
-
#
|
|
525
|
-
#
|
|
526
|
-
#
|
|
527
|
-
#
|
|
636
|
+
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
|
|
637
|
+
# and each interval over the pairs. That's for a process benchmark, or
|
|
638
|
+
# too few workers for an interval over them: otherwise, see
|
|
639
|
+
# #judge_processes. A process benchmark's pairs take a process each,
|
|
640
|
+
# so it's decided not slower (Result::CLEARED) after any pair too.
|
|
641
|
+
# That can only stop it before it would have been found slower, never
|
|
642
|
+
# make it slower: it's found slower only at the end of a round, as any
|
|
643
|
+
# benchmark is.
|
|
528
644
|
def judge(name, n, floors, result)
|
|
529
645
|
judged = metrics.select do |spec|
|
|
530
646
|
floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
|
|
531
647
|
floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
|
|
532
648
|
end
|
|
649
|
+
return judge_processes(name, n, judged, result) if by_process?(name)
|
|
650
|
+
|
|
533
651
|
ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
534
652
|
# @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
|
|
535
653
|
raw = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
@@ -561,6 +679,78 @@ module Duel
|
|
|
561
679
|
end
|
|
562
680
|
end
|
|
563
681
|
|
|
682
|
+
# Whether a benchmark's timings are judged by process (see
|
|
683
|
+
# #judge_processes): unless it's a process benchmark (each of its
|
|
684
|
+
# samples is a fresh process already, so its pairs are independent),
|
|
685
|
+
# or there are too few workers of each version for an interval over
|
|
686
|
+
# them (Lane.by_process?).
|
|
687
|
+
def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
|
|
688
|
+
|
|
689
|
+
# Takes rounds of pairs, each a sample from every worker of each
|
|
690
|
+
# version (the baseline's in order, paired with the candidate's in
|
|
691
|
+
# turn: each round pairs each with another), until every ratio metric
|
|
692
|
+
# judged (see #judge) is decided, from the second round on, or
|
|
693
|
+
# #rounds run out. Each worker's median is a sample, and the interval
|
|
694
|
+
# is over workers (Stats.shift_interval): where a process's luck
|
|
695
|
+
# makes it faster or slower than another for its whole life (where
|
|
696
|
+
# things landed in its memory, or what a JIT compiled), the interval
|
|
697
|
+
# is as wide as that makes it. An interval over pairs would be as
|
|
698
|
+
# narrow as if every pair had processes of its own. Each worker's
|
|
699
|
+
# medians are kept, by metric, for across the board (Result#processes).
|
|
700
|
+
def judge_processes(name, n, judged, result)
|
|
701
|
+
k = @bases.size
|
|
702
|
+
# @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
|
|
703
|
+
values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
|
|
704
|
+
pending = judged.dup
|
|
705
|
+
pairs = 0
|
|
706
|
+
rounds(k).times do |round|
|
|
707
|
+
break if pending.empty?
|
|
708
|
+
|
|
709
|
+
k.times do |i|
|
|
710
|
+
j = (i + round) % k
|
|
711
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
|
|
712
|
+
result.note(base, candidate)
|
|
713
|
+
values.each do |metric, sides|
|
|
714
|
+
sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
|
|
715
|
+
sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
|
|
716
|
+
end
|
|
717
|
+
pairs += 1
|
|
718
|
+
end
|
|
719
|
+
next if round + 1 < MIN_ROUNDS
|
|
720
|
+
|
|
721
|
+
pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
|
|
722
|
+
end
|
|
723
|
+
judged.each do |spec|
|
|
724
|
+
metric = spec.fetch("name")
|
|
725
|
+
each = medians(values.fetch(metric))
|
|
726
|
+
bases, candidates = clamped(each)
|
|
727
|
+
result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
728
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
729
|
+
"interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
|
|
730
|
+
"pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
|
|
731
|
+
"baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
|
|
732
|
+
result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
|
|
733
|
+
end
|
|
734
|
+
end
|
|
735
|
+
|
|
736
|
+
# The most rounds of +k+ workers' pairs: as many as make
|
|
737
|
+
# PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
|
|
738
|
+
def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
|
|
739
|
+
|
|
740
|
+
# Each worker's median, by version, of +values+ (each worker's
|
|
741
|
+
# measures, by version).
|
|
742
|
+
def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
|
|
743
|
+
|
|
744
|
+
# The baseline's medians (of +each+) at least 1, as the ratio's
|
|
745
|
+
# denominator (so a ratio to nothing is one to 1, as a pair's is),
|
|
746
|
+
# and the candidate's.
|
|
747
|
+
def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
|
|
748
|
+
|
|
749
|
+
def process_verdict(each, spec)
|
|
750
|
+
bases, candidates = clamped(each)
|
|
751
|
+
Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
|
|
752
|
+
end
|
|
753
|
+
|
|
564
754
|
# Whether both versions' CPU time in +floors+ is below its floor: then
|
|
565
755
|
# it wouldn't be judged at all.
|
|
566
756
|
def below_cpu_floor?(floors)
|
|
@@ -632,9 +822,14 @@ module Duel
|
|
|
632
822
|
# out, and weighs on both alike.
|
|
633
823
|
def pair(name, n, index, runs: RUNS, count: false)
|
|
634
824
|
k = @bases.size
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
825
|
+
sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
|
|
826
|
+
end
|
|
827
|
+
|
|
828
|
+
# A sample (of +n+ iterations) from +base+, a baseline worker, and
|
|
829
|
+
# from +candidate+, a candidate worker, the baseline's first if
|
|
830
|
+
# +base_first+ (see #pair).
|
|
831
|
+
def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
|
|
832
|
+
order = base_first ? [base, candidate] : [candidate, base]
|
|
638
833
|
process = benchmarks.fetch(name).fetch("process")
|
|
639
834
|
replies = if process
|
|
640
835
|
# (Each run of one version's next to one of the other's.)
|
|
@@ -657,7 +852,9 @@ module Duel
|
|
|
657
852
|
# Each counted measure's fewest, on each version, over COUNTS pairs of
|
|
658
853
|
# samples, and what its metric said of those samples.
|
|
659
854
|
def count(name, n, result)
|
|
660
|
-
|
|
855
|
+
return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
|
|
856
|
+
|
|
857
|
+
samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
|
|
661
858
|
metrics.each do |spec|
|
|
662
859
|
metric = spec.fetch("name")
|
|
663
860
|
base, candidate = samples.transpose.map do |side|
|
|
@@ -672,6 +869,61 @@ module Duel
|
|
|
672
869
|
end
|
|
673
870
|
end
|
|
674
871
|
|
|
872
|
+
# Each counted measure, with workers enough for an interval over them
|
|
873
|
+
# (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
|
|
874
|
+
# of a sample from every worker, and how much more the candidate's
|
|
875
|
+
# are than the baseline's, at least (the low end of an interval for
|
|
876
|
+
# it, Stats.difference_interval), which is what's judged (see
|
|
877
|
+
# Result#worse). Where every worker's is the same, as CRuby's counts
|
|
878
|
+
# of objects are, that's exactly how many more. Where they differ, as
|
|
879
|
+
# JRuby's bytes do where one JVM's JIT does away with an allocation
|
|
880
|
+
# and another's doesn't, it's as little as the workers can tell.
|
|
881
|
+
# Each version's own is the median of its workers'.
|
|
882
|
+
def count_by_process(name, n, result)
|
|
883
|
+
k = @bases.size
|
|
884
|
+
# @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
|
|
885
|
+
fewest = Hash.new do |h, metric|
|
|
886
|
+
none = {} #: Hash[Integer, Metric::value]
|
|
887
|
+
h[metric] = {"baseline" => none, "candidate" => none.dup}
|
|
888
|
+
end
|
|
889
|
+
COUNT_ROUNDS.times do |round|
|
|
890
|
+
k.times do |i|
|
|
891
|
+
j = (i + round) % k
|
|
892
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
|
|
893
|
+
fewest!(fewest, "baseline", i, base)
|
|
894
|
+
fewest!(fewest, "candidate", j, candidate)
|
|
895
|
+
result.note(base, candidate)
|
|
896
|
+
end
|
|
897
|
+
end
|
|
898
|
+
metrics.each do |spec|
|
|
899
|
+
metric = spec.fetch("name")
|
|
900
|
+
next unless fewest.key?(metric)
|
|
901
|
+
|
|
902
|
+
found = fewest.fetch(metric)
|
|
903
|
+
bases = found.fetch("baseline").values
|
|
904
|
+
candidates = found.fetch("candidate").values
|
|
905
|
+
next if bases.empty? || candidates.empty?
|
|
906
|
+
|
|
907
|
+
result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
|
|
908
|
+
"more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
|
|
909
|
+
"n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
|
|
910
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
911
|
+
end
|
|
912
|
+
end
|
|
913
|
+
|
|
914
|
+
# Keeps each count of +sample+ (from +side+'s +worker+th worker) in
|
|
915
|
+
# +fewest+, where it's that worker's fewest so far.
|
|
916
|
+
def fewest!(fewest, side, worker, sample)
|
|
917
|
+
sample.fetch("counted").each do |metric, value|
|
|
918
|
+
least = fewest[metric].fetch(side)
|
|
919
|
+
least[worker] = [least[worker], value].compact.min
|
|
920
|
+
end
|
|
921
|
+
end
|
|
922
|
+
|
|
923
|
+
# The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
|
|
924
|
+
# or else COUNTS.
|
|
925
|
+
def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
|
|
926
|
+
|
|
675
927
|
# Each version's least peak memory over MEMORY_SAMPLES fresh workers
|
|
676
928
|
# (a long-lived one has room left over from earlier work), turn about.
|
|
677
929
|
# Fresh means no trial run either: the allocator keeps what that freed,
|
|
@@ -743,6 +995,13 @@ module Duel
|
|
|
743
995
|
# @api private
|
|
744
996
|
attr_reader :profile
|
|
745
997
|
|
|
998
|
+
# Each worker's median of each measure judged by process (see
|
|
999
|
+
# Lane#judge_processes): {metric => {"lane" => its lane's number,
|
|
1000
|
+
# "baseline" => [each baseline worker's], "candidate" => [...]}},
|
|
1001
|
+
# for across the board.
|
|
1002
|
+
# @api private
|
|
1003
|
+
attr_reader :processes
|
|
1004
|
+
|
|
746
1005
|
# The measurements again that were too close to call (see #confirm).
|
|
747
1006
|
# @api private
|
|
748
1007
|
attr_reader :undecided
|
|
@@ -758,6 +1017,7 @@ module Duel
|
|
|
758
1017
|
@details = {}
|
|
759
1018
|
@confirmations = []
|
|
760
1019
|
@undecided = []
|
|
1020
|
+
@processes = {}
|
|
761
1021
|
end
|
|
762
1022
|
|
|
763
1023
|
# Keeps what metrics said of a pair of samples (Metric#details): each
|
|
@@ -773,11 +1033,12 @@ module Duel
|
|
|
773
1033
|
|
|
774
1034
|
# The metrics on which the candidate did worse: a timed one slower, or
|
|
775
1035
|
# a counted one up by more than its tolerance per iteration and its
|
|
776
|
-
# allowance per sample
|
|
1036
|
+
# allowance per sample (counted by process, up by that much at least:
|
|
1037
|
+
# see Lane#count_by_process).
|
|
777
1038
|
# @api private
|
|
778
1039
|
def worse
|
|
779
1040
|
more = @counted.select do |_, c|
|
|
780
|
-
!Metric.within?(c.fetch("candidate") - c.fetch("baseline"), c.fetch("n"), tolerance: c.fetch("tolerance"),
|
|
1041
|
+
!Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
|
|
781
1042
|
allowance: c.fetch("allowance"))
|
|
782
1043
|
end
|
|
783
1044
|
slower + more.keys + (more_memory? ? ["peak_memory"] : [])
|
|
@@ -871,6 +1132,14 @@ module Duel
|
|
|
871
1132
|
# benchmark", "2 benchmarks").
|
|
872
1133
|
def self.count(number, noun, plural = "#{noun}s") = "#{number} #{(number == 1) ? noun : plural}"
|
|
873
1134
|
|
|
1135
|
+
# A count, an iteration's (a Float, often): to four figures, or one of
|
|
1136
|
+
# NUMBER or more (where four figures would be 1e+04) as a whole number
|
|
1137
|
+
# (25736 bytes, not 2.574e+04).
|
|
1138
|
+
def self.number(value) = (value.abs >= NUMBER) ? value.round.to_s : format("%.4g", value)
|
|
1139
|
+
|
|
1140
|
+
# The least count #number shows as a whole number.
|
|
1141
|
+
NUMBER = 9_999.5
|
|
1142
|
+
|
|
874
1143
|
# Across the board (see Across): {metric => verdict}, "confirmed" in
|
|
875
1144
|
# those slower in a second measurement too. Set by #run.
|
|
876
1145
|
attr_reader :across
|
|
@@ -957,12 +1226,13 @@ module Duel
|
|
|
957
1226
|
# What #run does, with its versions prepared in its directory.
|
|
958
1227
|
def compare_all(started)
|
|
959
1228
|
@sides = prepare
|
|
1229
|
+
fit
|
|
960
1230
|
names, results = with_lanes do |lanes|
|
|
961
1231
|
found = lanes.first.benchmarks.keys.grep(@settings.filter)
|
|
962
1232
|
@benchmarks = lanes.first.benchmarks
|
|
963
1233
|
@out.puts "Comparing #{Runner.count(found.size, "benchmark")}, #{@settings.jobs} at a time: baseline #{@base_root} " \
|
|
964
1234
|
"against candidate #{File.expand_path(@settings.candidate_dir || @config.root)} (slower: a " \
|
|
965
|
-
"#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit})"
|
|
1235
|
+
"#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit}, #{interval_over})"
|
|
966
1236
|
rubies = @rubies = lanes.first.rubies
|
|
967
1237
|
rubies.each { |side, ruby| @out.puts " #{side} on #{ruby}" } if rubies.values.uniq.size > 1
|
|
968
1238
|
@out.puts " with plugins #{plugins.join(", ")}" unless plugins.empty?
|
|
@@ -975,6 +1245,23 @@ module Duel
|
|
|
975
1245
|
results
|
|
976
1246
|
end
|
|
977
1247
|
|
|
1248
|
+
# What a benchmark's interval is over (see Lane#judge_processes): the
|
|
1249
|
+
# workers, or the pairs, where they're too few (and says how many it
|
|
1250
|
+
# takes, of the first PROCESSES_CHECKED).
|
|
1251
|
+
# mutant:disable -- fewer than 2 of each never take an interval over them, and the last checked is only a bound
|
|
1252
|
+
def interval_over
|
|
1253
|
+
processes = @settings.processes
|
|
1254
|
+
workers = Runner.count(processes, "worker process", "worker processes")
|
|
1255
|
+
return "over #{workers} of each version" if Lane.by_process?(processes, @settings.confidence)
|
|
1256
|
+
|
|
1257
|
+
least = (1..PROCESSES_CHECKED).find { |k| Lane.by_process?(k, @settings.confidence) }
|
|
1258
|
+
"over pairs: #{workers} of each version are too few for an interval over them" + (least ? ", which takes #{least}" : "")
|
|
1259
|
+
end
|
|
1260
|
+
|
|
1261
|
+
# The most workers of each version #interval_over looks at, for how
|
|
1262
|
+
# many it takes (more would take long to check, and longer to run).
|
|
1263
|
+
PROCESSES_CHECKED = 20
|
|
1264
|
+
|
|
978
1265
|
# Each version, prepared once for the run, before any of its workers
|
|
979
1266
|
# start: its Ruby found (Config#ruby, or --ruby), and its
|
|
980
1267
|
# Config#prepare(:version) blocks run, each given a Resource (named for
|
|
@@ -984,6 +1271,7 @@ module Duel
|
|
|
984
1271
|
Side.unprepared(@config, @settings, @base_root).transform_values do |side|
|
|
985
1272
|
side.ruby = Rubies.find(choice, side.root, side.name)
|
|
986
1273
|
side.env = Rubies.environment(side.ruby)
|
|
1274
|
+
side.engine = Rubies.engine(side.ruby)
|
|
987
1275
|
@config.prepares.fetch(:version).each_with_index do |block, i|
|
|
988
1276
|
resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
|
|
989
1277
|
cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
|
|
@@ -994,6 +1282,47 @@ module Duel
|
|
|
994
1282
|
end
|
|
995
1283
|
end
|
|
996
1284
|
|
|
1285
|
+
# The memory Runner#fit leaves for everything but the workers and
|
|
1286
|
+
# their resources: the runner, the system, and other processes.
|
|
1287
|
+
RESERVE = 1 << 30
|
|
1288
|
+
|
|
1289
|
+
# How much more than what it took to start a pair of workers takes
|
|
1290
|
+
# while it measures (its heap grows, and so do its resources'), by
|
|
1291
|
+
# Runner#fit's reckoning.
|
|
1292
|
+
HEADROOM = 1.25
|
|
1293
|
+
|
|
1294
|
+
# Lowers Settings#jobs, and then Settings#processes, as far as
|
|
1295
|
+
# Settings#fit lets it, where their workers (and each one's
|
|
1296
|
+
# resources, a database server, say) wouldn't fit in the memory
|
|
1297
|
+
# available: more than that and the machine swaps, or a CI runner
|
|
1298
|
+
# stops answering and its job is lost. What one worker of each
|
|
1299
|
+
# version takes is found by starting them (a lane of them), and every
|
|
1300
|
+
# pair is taken to need that, HEADROOM over.
|
|
1301
|
+
def fit
|
|
1302
|
+
return if @settings.fit.empty?
|
|
1303
|
+
|
|
1304
|
+
available = Memory.available or return
|
|
1305
|
+
taken = Lane.open(@config, @settings, @sides, processes: 1) { available - (Memory.available || available) }
|
|
1306
|
+
return unless taken.positive?
|
|
1307
|
+
|
|
1308
|
+
pair = (taken * HEADROOM).ceil
|
|
1309
|
+
pairs = [(available - RESERVE) / pair, 1].max
|
|
1310
|
+
processes = @settings.fit.include?(:processes) ? [@settings.processes, pairs].min : @settings.processes
|
|
1311
|
+
# (More jobs than the settings' are never taken: only fewer are.)
|
|
1312
|
+
jobs = @settings.fit.include?(:jobs) ? [pairs / processes, 1].max : @settings.jobs
|
|
1313
|
+
fewer = [("#{Runner.count(jobs, "benchmark")} at a time (not #{@settings.jobs})" if jobs < @settings.jobs),
|
|
1314
|
+
("#{Runner.count(processes, "worker")} of each version (not #{@settings.processes})" if processes < @settings.processes)].compact
|
|
1315
|
+
return if fewer.empty?
|
|
1316
|
+
|
|
1317
|
+
@out.puts "A worker of each version takes #{gigabytes(taken)} GB, with its resources, of #{gigabytes(available)} GB available, " \
|
|
1318
|
+
"so to fit: #{fewer.join(", and ")}"
|
|
1319
|
+
@settings.jobs = jobs
|
|
1320
|
+
@settings.processes = processes
|
|
1321
|
+
end
|
|
1322
|
+
|
|
1323
|
+
# +bytes+ in gigabytes, to a tenth.
|
|
1324
|
+
def gigabytes(bytes) = format("%.1f", bytes.fdiv(1 << 30))
|
|
1325
|
+
|
|
997
1326
|
def with_lanes
|
|
998
1327
|
# @type var lanes: Array[Lane]
|
|
999
1328
|
lanes = []
|
|
@@ -1007,7 +1336,7 @@ module Duel
|
|
|
1007
1336
|
# every benchmark, in fresh processes, and are confirmed if they're
|
|
1008
1337
|
# still slower.
|
|
1009
1338
|
def judge_across(results, names)
|
|
1010
|
-
verdicts =
|
|
1339
|
+
verdicts = across_of(results).verdicts
|
|
1011
1340
|
slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
|
|
1012
1341
|
return verdicts if slower.empty?
|
|
1013
1342
|
|
|
@@ -1020,11 +1349,16 @@ module Duel
|
|
|
1020
1349
|
end
|
|
1021
1350
|
|
|
1022
1351
|
@out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
|
|
1023
|
-
again =
|
|
1352
|
+
again = across_of(with_lanes { |lanes| measure_all(lanes, names) }).verdicts
|
|
1024
1353
|
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
|
|
1025
1354
|
verdicts
|
|
1026
1355
|
end
|
|
1027
1356
|
|
|
1357
|
+
# Across the board, of +results+, judged over workers where the run
|
|
1358
|
+
# judged benchmarks over them (see Lane.by_process?).
|
|
1359
|
+
# mutant:disable -- whether it's over workers is only how fast: results judged over pairs have no workers' medians to find
|
|
1360
|
+
def across_of(results) = Across.new(results, confidence: @settings.confidence, by_process: Lane.by_process?(@settings.processes, @settings.confidence))
|
|
1361
|
+
|
|
1028
1362
|
def open_lane = Lane.new(@config, @settings, @sides)
|
|
1029
1363
|
|
|
1030
1364
|
def measure_all(lanes, names)
|
|
@@ -1145,7 +1479,7 @@ module Duel
|
|
|
1145
1479
|
candidate = c.fetch("candidate").fdiv(n)
|
|
1146
1480
|
# (No unit its name ends in: "retained_objects 2", not "2 objects".)
|
|
1147
1481
|
named = c.fetch("plural")&.then { |plural| metric.end_with?(plural.tr(" ", "_")) }
|
|
1148
|
-
[format("%s
|
|
1482
|
+
[format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
|
|
1149
1483
|
(unit(candidate, c) unless named)].compact.join(" ")
|
|
1150
1484
|
end
|
|
1151
1485
|
memory = result.memory&.then { |m| ["peak_memory #{mb(m.fetch("baseline"))}→#{mb(m.fetch("candidate"))}"] }
|
|
@@ -1189,7 +1523,7 @@ module Duel
|
|
|
1189
1523
|
def amount(value, n, measured)
|
|
1190
1524
|
each = value.fdiv([n, 1].max)
|
|
1191
1525
|
ns = Metric::TIME_UNITS[measured.fetch("unit")]
|
|
1192
|
-
return [
|
|
1526
|
+
return [Runner.number(each), unit(each, measured)].compact.join(" ") unless ns
|
|
1193
1527
|
|
|
1194
1528
|
time = each * ns
|
|
1195
1529
|
shown, size = Metric::TIME_UNITS.reverse_each.find { |_, unit_ns| time >= unit_ns } || ["ns", 1]
|