duel 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/README.md +5 -5
- data/docs/ci.md +120 -25
- data/docs/configuration.md +8 -4
- data/docs/how-it-works.md +109 -35
- data/docs/json.md +16 -10
- data/docs/plugins.md +3 -1
- data/docs/troubleshooting.md +6 -6
- data/lib/duel/across.rb +100 -9
- data/lib/duel/baseline.rb +26 -2
- data/lib/duel/cli.rb +18 -7
- data/lib/duel/config.rb +4 -1
- data/lib/duel/init.rb +19 -3
- data/lib/duel/json_report.rb +2 -2
- data/lib/duel/metric.rb +32 -1
- data/lib/duel/rubies.rb +39 -0
- data/lib/duel/runner.rb +337 -49
- data/lib/duel/stats.rb +76 -0
- data/lib/duel/summary.rb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +3 -0
- data/lib/duel/templates/perf-comment.yml.erb +53 -5
- data/lib/duel/templates/perf.yml.erb +71 -33
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +5 -4
- data/sig/duel/across.rbs +24 -2
- data/sig/duel/baseline.rbs +12 -0
- data/sig/duel/cli.rbs +4 -0
- data/sig/duel/init.rbs +6 -0
- data/sig/duel/metric.rbs +19 -1
- data/sig/duel/rubies.rbs +13 -0
- data/sig/duel/runner.rbs +112 -6
- data/sig/duel/stats.rbs +25 -0
- metadata +1 -1
data/lib/duel/runner.rb
CHANGED
|
@@ -14,30 +14,56 @@ require_relative "rubies"
|
|
|
14
14
|
require_relative "process_sample"
|
|
15
15
|
|
|
16
16
|
module Duel
|
|
17
|
-
#
|
|
18
|
-
# says:
|
|
17
|
+
# The most seconds each benchmark runs before it's measured, unless
|
|
18
|
+
# --warm says: until each worker is steady (see Lane#steady), as long
|
|
19
|
+
# as that takes. JRuby's and TruffleRuby's JITs keep compiling for
|
|
20
|
+
# seconds, and some code is still several times faster at 5 than it
|
|
21
|
+
# was, so they get up to a minute.
|
|
19
22
|
# @api private
|
|
20
|
-
JIT_WARM = {"jruby" =>
|
|
23
|
+
JIT_WARM = {"jruby" => 60.0, "truffleruby" => 60.0}.fetch(RUBY_ENGINE, 0.0)
|
|
21
24
|
|
|
22
|
-
#
|
|
23
|
-
# YJIT on (an app may turn it on as it boots, as Rails
|
|
24
|
-
# --warm says. A small library's methods are compiled
|
|
25
|
-
# warm-up samples, but a big app's only after 120 calls
|
|
26
|
-
# a slow benchmark's warm-up samples make.
|
|
25
|
+
# The most seconds each benchmark runs before it's measured where a
|
|
26
|
+
# worker has YJIT on (an app may turn it on as it boots, as Rails
|
|
27
|
+
# does), unless --warm says. A small library's methods are compiled
|
|
28
|
+
# within the warm-up samples, but a big app's only after 120 calls
|
|
29
|
+
# each, more than a slow benchmark's warm-up samples make.
|
|
27
30
|
# @api private
|
|
28
|
-
YJIT_WARM =
|
|
31
|
+
YJIT_WARM = 10.0
|
|
32
|
+
|
|
33
|
+
# Worker processes of each version, by default: enough for an interval
|
|
34
|
+
# over processes (see Lane#judge_processes), which takes 5 at 99%, and
|
|
35
|
+
# more, as it narrows with each. JRuby's and TruffleRuby's are JVMs,
|
|
36
|
+
# slow to start and big (some 500 MB), so they get fewer.
|
|
37
|
+
# @api private
|
|
38
|
+
PROCESSES = {"jruby" => 8, "truffleruby" => 8}.fetch(RUBY_ENGINE, 12)
|
|
39
|
+
|
|
40
|
+
# The CPUs for each lane of workers, by default (see CLI#parse): half
|
|
41
|
+
# the CPUs make lanes, so that each lane's pair of workers has a CPU of
|
|
42
|
+
# its own. JRuby's and TruffleRuby's JIT compilers keep other CPUs busy,
|
|
43
|
+
# and each of their workers is a big JVM, so they get a lane for every
|
|
44
|
+
# four: more lanes are no faster there, and take more memory.
|
|
45
|
+
# @api private
|
|
46
|
+
CPUS_PER_LANE = {"jruby" => 4, "truffleruby" => 4}.fetch(RUBY_ENGINE, 2)
|
|
47
|
+
|
|
48
|
+
# The seconds a worker may take to load, or to answer a request, by
|
|
49
|
+
# default (see Settings#timeout). JRuby's and TruffleRuby's JVMs take a
|
|
50
|
+
# while to start, and with -Xbatch (see Rubies::JRUBY_WORKER), they
|
|
51
|
+
# compile as they load too: 16 at once can take over a minute.
|
|
52
|
+
# @api private
|
|
53
|
+
TIMEOUT = {"jruby" => 300.0, "truffleruby" => 300.0}.fetch(RUBY_ENGINE, 60.0)
|
|
29
54
|
|
|
30
55
|
# The seconds a run measures for, at most, by default (see
|
|
31
|
-
# Settings#time_limit and Runner#out_of_time?)
|
|
32
|
-
|
|
56
|
+
# Settings#time_limit and Runner#out_of_time?): two hours, as on JRuby,
|
|
57
|
+
# a benchmark can take half a minute to warm up.
|
|
58
|
+
TIME_LIMIT = 7200.0
|
|
33
59
|
|
|
34
60
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
35
61
|
# @api private
|
|
36
62
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
37
|
-
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
|
|
63
|
+
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit, :accepted) do
|
|
38
64
|
# @type self: singleton(Settings)
|
|
39
|
-
def self.defaults = new(filter: //, jobs: 1, processes:
|
|
40
|
-
timeout:
|
|
65
|
+
def self.defaults = new(filter: //, jobs: 1, processes: PROCESSES, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
66
|
+
timeout: TIMEOUT, time_limit: TIME_LIMIT)
|
|
41
67
|
end
|
|
42
68
|
|
|
43
69
|
# One version as a run has it (see Runner#prepare), +name+d "baseline"
|
|
@@ -45,13 +71,13 @@ module Duel
|
|
|
45
71
|
# (Config#ruby), and what they get in their environment (that Ruby's,
|
|
46
72
|
# and what its Config#prepare(:version) blocks gave it).
|
|
47
73
|
# @api private
|
|
48
|
-
Side = Struct.new(:name, :root, :ruby, :env) do
|
|
74
|
+
Side = Struct.new(:name, :root, :ruby, :env, :engine) do
|
|
49
75
|
# @type self: singleton(Side)
|
|
50
76
|
|
|
51
77
|
# Each version with nothing prepared: its tree, on the runner's Ruby.
|
|
52
78
|
def self.unprepared(config, settings, base_root)
|
|
53
|
-
{"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}),
|
|
54
|
-
"candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {})}
|
|
79
|
+
{"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}, RUBY_ENGINE),
|
|
80
|
+
"candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {}, RUBY_ENGINE)}
|
|
55
81
|
end
|
|
56
82
|
end
|
|
57
83
|
|
|
@@ -104,9 +130,12 @@ module Duel
|
|
|
104
130
|
end
|
|
105
131
|
|
|
106
132
|
# +timeout+: the seconds it may take to load, or to answer a request,
|
|
107
|
-
# before it's taken for hung (and killed). It runs on +ruby
|
|
108
|
-
# +env
|
|
109
|
-
|
|
133
|
+
# before it's taken for hung (and killed). It runs on +ruby+ (whose
|
|
134
|
+
# engine is +engine+), with +env+, and on JRuby, the JVM's options for
|
|
135
|
+
# a worker (see Rubies.worker_environment), not given to the processes
|
|
136
|
+
# it starts for a process benchmark (see #run): their boot is measured
|
|
137
|
+
# as it is.
|
|
138
|
+
def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:)
|
|
110
139
|
@label = label
|
|
111
140
|
@timeout = timeout
|
|
112
141
|
@root = root
|
|
@@ -118,7 +147,7 @@ module Duel
|
|
|
118
147
|
# (A nil config_path is left out. And the worker gets no other
|
|
119
148
|
# worker's connection: TruffleRuby's sockets aren't close-on-exec,
|
|
120
149
|
# and one left open in another worker would keep it from ending.)
|
|
121
|
-
@pid = Process.spawn(env, ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
|
|
150
|
+
@pid = Process.spawn(Rubies.worker_environment(env, engine), ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
|
|
122
151
|
"-e", "Duel::Worker.main(*ARGV)", root, candidate_root, control, *[config_path].compact, in: File::NULL, close_others: true)
|
|
123
152
|
@socket = accept(server, control)
|
|
124
153
|
hello = read
|
|
@@ -210,12 +239,13 @@ module Duel
|
|
|
210
239
|
# a worker that died or hung.
|
|
211
240
|
class Failed < Error; end
|
|
212
241
|
|
|
213
|
-
# Workers of both versions, taking turns:
|
|
214
|
-
#
|
|
215
|
-
# the very same code a few percent faster or slower for that
|
|
216
|
-
# whole life
|
|
217
|
-
#
|
|
218
|
-
# its own resources
|
|
242
|
+
# Workers of both versions, taking turns: Settings#processes of each
|
|
243
|
+
# (PROCESSES, by default). Where things landed in one process's memory
|
|
244
|
+
# can make the very same code a few percent faster or slower for that
|
|
245
|
+
# process's whole life, so the interval is over the workers, where
|
|
246
|
+
# they're enough for one (see #judge_processes), and not over the
|
|
247
|
+
# pairs, which share them. Each worker has its own resources
|
|
248
|
+
# (Config#prepare(:worker)).
|
|
219
249
|
# @api private
|
|
220
250
|
class Lane
|
|
221
251
|
# CPU time (or a plugin's ratio metric's equivalent) a run aims for:
|
|
@@ -238,8 +268,20 @@ module Duel
|
|
|
238
268
|
# process benchmark's: see #judge).
|
|
239
269
|
STEP = 10
|
|
240
270
|
MAX_PAIRS = 60
|
|
241
|
-
#
|
|
271
|
+
# A benchmark judged by process (see #judge_processes) is decided from
|
|
272
|
+
# its MIN_ROUNDS-th round of a sample from every worker, a sample
|
|
273
|
+
# each takes, and up to PROCESS_PAIRS pairs.
|
|
274
|
+
MIN_ROUNDS = 4
|
|
275
|
+
PROCESS_PAIRS = 120
|
|
276
|
+
# Counts are each version's fewest in COUNTS samples, or where a
|
|
277
|
+
# version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
|
|
278
|
+
# takes one of two values, some 31 bytes an iteration apart, sample by
|
|
279
|
+
# sample, and the fewest of more are the lower one.
|
|
242
280
|
COUNTS = 3
|
|
281
|
+
JRUBY_COUNTS = 10
|
|
282
|
+
# A count by process (see #count_by_process) is each worker's fewest
|
|
283
|
+
# in COUNT_ROUNDS rounds of a sample from every worker.
|
|
284
|
+
COUNT_ROUNDS = 3
|
|
243
285
|
# Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
|
|
244
286
|
MEMORY_SAMPLES = 3
|
|
245
287
|
# A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
|
|
@@ -252,6 +294,7 @@ module Duel
|
|
|
252
294
|
# server takes seconds), and stop at once. +sides+ are the versions as
|
|
253
295
|
# the run prepared them (Runner#prepare), by name.
|
|
254
296
|
def initialize(config, settings, sides, processes: settings.processes)
|
|
297
|
+
@id = Lane.next_id
|
|
255
298
|
@dir = Lane.tmpdir
|
|
256
299
|
@resources = []
|
|
257
300
|
@workers = []
|
|
@@ -271,6 +314,18 @@ module Duel
|
|
|
271
314
|
raise
|
|
272
315
|
end
|
|
273
316
|
|
|
317
|
+
@ids = 0
|
|
318
|
+
@ids_lock = Mutex.new
|
|
319
|
+
|
|
320
|
+
# A number of its own for each lane, from 1: what tells one lane's
|
|
321
|
+
# workers from another's (see Result#processes).
|
|
322
|
+
# mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
|
|
323
|
+
def self.next_id = @ids_lock.synchronize { @ids += 1 }
|
|
324
|
+
|
|
325
|
+
# Whether +processes+ workers of each version are enough for a
|
|
326
|
+
# +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
|
|
327
|
+
def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
|
|
328
|
+
|
|
274
329
|
# A directory for a lane's sockets, short as a Unix socket's path must
|
|
275
330
|
# be (see .short_tmp).
|
|
276
331
|
def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
|
|
@@ -355,7 +410,7 @@ module Duel
|
|
|
355
410
|
env = Resource.environment(label, env, block.call(resource))
|
|
356
411
|
end
|
|
357
412
|
worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
|
|
358
|
-
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby)
|
|
413
|
+
config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
|
|
359
414
|
keep(@workers, worker)
|
|
360
415
|
worker
|
|
361
416
|
end
|
|
@@ -506,30 +561,89 @@ module Duel
|
|
|
506
561
|
PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
|
|
507
562
|
end
|
|
508
563
|
|
|
509
|
-
# Every worker runs the benchmark a few times, and
|
|
510
|
-
#
|
|
511
|
-
#
|
|
564
|
+
# Every worker runs the benchmark a few times, and then, all at once
|
|
565
|
+
# (none of it is measured, so they needn't take turns), for the
|
|
566
|
+
# seconds --warm says (Settings#warm), or else until it's steady (see
|
|
567
|
+
# #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
|
|
512
568
|
def warm(name, n)
|
|
513
569
|
warm_until = monotonic + @warm
|
|
514
570
|
in_parallel(@bases + @candidates) do |worker|
|
|
515
571
|
2.times { worker.sample(name, n) }
|
|
516
|
-
|
|
572
|
+
if @settings.warm
|
|
573
|
+
worker.sample(name, n) until monotonic > warm_until
|
|
574
|
+
else
|
|
575
|
+
steady(worker, name, n, warm_until)
|
|
576
|
+
end
|
|
577
|
+
end
|
|
578
|
+
end
|
|
579
|
+
|
|
580
|
+
# A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
|
|
581
|
+
# seconds in a row each have a median of its samples no faster, by
|
|
582
|
+
# STEADY_GAIN, than the last window that was (so code that creeps
|
|
583
|
+
# faster by less a window isn't steady): a JIT has compiled what it
|
|
584
|
+
# will. (Not once its fastest sample stops getting
|
|
585
|
+
# faster: the more samples, the faster the fastest of them, by
|
|
586
|
+
# chance. And not once one window is no faster: a pause, a collection
|
|
587
|
+
# say, can slow one.) A small library's code is steady in a few
|
|
588
|
+
# seconds, where a big app's JIT keeps compiling, for up to the
|
|
589
|
+
# warm-up's seconds.
|
|
590
|
+
STEADY = 0.5
|
|
591
|
+
STEADY_GAIN = 0.01
|
|
592
|
+
STEADY_WINDOWS = 3
|
|
593
|
+
|
|
594
|
+
# Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
|
|
595
|
+
# until it's steady, or it's +warm_until+.
|
|
596
|
+
def steady(worker, name, n, warm_until)
|
|
597
|
+
window, now = window(worker, name, n, warm_until)
|
|
598
|
+
return if now > warm_until
|
|
599
|
+
|
|
600
|
+
fastest = Stats.median(window)
|
|
601
|
+
calm = 0
|
|
602
|
+
loop do
|
|
603
|
+
window, now = window(worker, name, n, warm_until)
|
|
604
|
+
return if now > warm_until
|
|
605
|
+
|
|
606
|
+
median = Stats.median(window)
|
|
607
|
+
if median > fastest * (1 - STEADY_GAIN)
|
|
608
|
+
calm += 1
|
|
609
|
+
return if calm == STEADY_WINDOWS
|
|
610
|
+
else
|
|
611
|
+
calm = 0
|
|
612
|
+
fastest = median
|
|
613
|
+
end
|
|
614
|
+
end
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
# STEADY seconds of +worker+'s samples' CPU time (a sample at least),
|
|
618
|
+
# or until +warm_until+, and when they ended.
|
|
619
|
+
def window(worker, name, n, warm_until)
|
|
620
|
+
window_until = monotonic + STEADY
|
|
621
|
+
samples = [] #: Array[Metric::value]
|
|
622
|
+
loop do
|
|
623
|
+
samples << worker.sample(name, n).fetch("timed").fetch("cpu")
|
|
624
|
+
now = monotonic
|
|
625
|
+
return [samples, now] if now > window_until || now > warm_until
|
|
517
626
|
end
|
|
518
627
|
end
|
|
519
628
|
|
|
520
629
|
# Takes pairs until every ratio metric worth judging (measured on
|
|
521
630
|
# both versions in the warm-up, and above its floors on either: one
|
|
522
631
|
# that rises from next to nothing is judged too) is decided, or
|
|
523
|
-
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs
|
|
524
|
-
#
|
|
525
|
-
#
|
|
526
|
-
#
|
|
527
|
-
#
|
|
632
|
+
# MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
|
|
633
|
+
# and each interval over the pairs. That's for a process benchmark, or
|
|
634
|
+
# too few workers for an interval over them: otherwise, see
|
|
635
|
+
# #judge_processes. A process benchmark's pairs take a process each,
|
|
636
|
+
# so it's decided not slower (Result::CLEARED) after any pair too.
|
|
637
|
+
# That can only stop it before it would have been found slower, never
|
|
638
|
+
# make it slower: it's found slower only at the end of a round, as any
|
|
639
|
+
# benchmark is.
|
|
528
640
|
def judge(name, n, floors, result)
|
|
529
641
|
judged = metrics.select do |spec|
|
|
530
642
|
floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
|
|
531
643
|
floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
|
|
532
644
|
end
|
|
645
|
+
return judge_processes(name, n, judged, result) if by_process?(name)
|
|
646
|
+
|
|
533
647
|
ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
534
648
|
# @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
|
|
535
649
|
raw = judged.to_h { |spec| [spec.fetch("name"), []] }
|
|
@@ -561,6 +675,78 @@ module Duel
|
|
|
561
675
|
end
|
|
562
676
|
end
|
|
563
677
|
|
|
678
|
+
# Whether a benchmark's timings are judged by process (see
|
|
679
|
+
# #judge_processes): unless it's a process benchmark (each of its
|
|
680
|
+
# samples is a fresh process already, so its pairs are independent),
|
|
681
|
+
# or there are too few workers of each version for an interval over
|
|
682
|
+
# them (Lane.by_process?).
|
|
683
|
+
def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
|
|
684
|
+
|
|
685
|
+
# Takes rounds of pairs, each a sample from every worker of each
|
|
686
|
+
# version (the baseline's in order, paired with the candidate's in
|
|
687
|
+
# turn: each round pairs each with another), until every ratio metric
|
|
688
|
+
# judged (see #judge) is decided, from the second round on, or
|
|
689
|
+
# #rounds run out. Each worker's median is a sample, and the interval
|
|
690
|
+
# is over workers (Stats.shift_interval): where a process's luck
|
|
691
|
+
# makes it faster or slower than another for its whole life (where
|
|
692
|
+
# things landed in its memory, or what a JIT compiled), the interval
|
|
693
|
+
# is as wide as that makes it. An interval over pairs would be as
|
|
694
|
+
# narrow as if every pair had processes of its own. Each worker's
|
|
695
|
+
# medians are kept, by metric, for across the board (Result#processes).
|
|
696
|
+
def judge_processes(name, n, judged, result)
|
|
697
|
+
k = @bases.size
|
|
698
|
+
# @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
|
|
699
|
+
values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
|
|
700
|
+
pending = judged.dup
|
|
701
|
+
pairs = 0
|
|
702
|
+
rounds(k).times do |round|
|
|
703
|
+
break if pending.empty?
|
|
704
|
+
|
|
705
|
+
k.times do |i|
|
|
706
|
+
j = (i + round) % k
|
|
707
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
|
|
708
|
+
result.note(base, candidate)
|
|
709
|
+
values.each do |metric, sides|
|
|
710
|
+
sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
|
|
711
|
+
sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
|
|
712
|
+
end
|
|
713
|
+
pairs += 1
|
|
714
|
+
end
|
|
715
|
+
next if round + 1 < MIN_ROUNDS
|
|
716
|
+
|
|
717
|
+
pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
|
|
718
|
+
end
|
|
719
|
+
judged.each do |spec|
|
|
720
|
+
metric = spec.fetch("name")
|
|
721
|
+
each = medians(values.fetch(metric))
|
|
722
|
+
bases, candidates = clamped(each)
|
|
723
|
+
result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
|
|
724
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
|
|
725
|
+
"interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
|
|
726
|
+
"pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
|
|
727
|
+
"baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
|
|
728
|
+
result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
|
|
729
|
+
end
|
|
730
|
+
end
|
|
731
|
+
|
|
732
|
+
# The most rounds of +k+ workers' pairs: as many as make
|
|
733
|
+
# PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
|
|
734
|
+
def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
|
|
735
|
+
|
|
736
|
+
# Each worker's median, by version, of +values+ (each worker's
|
|
737
|
+
# measures, by version).
|
|
738
|
+
def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
|
|
739
|
+
|
|
740
|
+
# The baseline's medians (of +each+) at least 1, as the ratio's
|
|
741
|
+
# denominator (so a ratio to nothing is one to 1, as a pair's is),
|
|
742
|
+
# and the candidate's.
|
|
743
|
+
def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
|
|
744
|
+
|
|
745
|
+
def process_verdict(each, spec)
|
|
746
|
+
bases, candidates = clamped(each)
|
|
747
|
+
Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
|
|
748
|
+
end
|
|
749
|
+
|
|
564
750
|
# Whether both versions' CPU time in +floors+ is below its floor: then
|
|
565
751
|
# it wouldn't be judged at all.
|
|
566
752
|
def below_cpu_floor?(floors)
|
|
@@ -632,9 +818,14 @@ module Duel
|
|
|
632
818
|
# out, and weighs on both alike.
|
|
633
819
|
def pair(name, n, index, runs: RUNS, count: false)
|
|
634
820
|
k = @bases.size
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
821
|
+
sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
|
|
822
|
+
end
|
|
823
|
+
|
|
824
|
+
# A sample (of +n+ iterations) from +base+, a baseline worker, and
|
|
825
|
+
# from +candidate+, a candidate worker, the baseline's first if
|
|
826
|
+
# +base_first+ (see #pair).
|
|
827
|
+
def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
|
|
828
|
+
order = base_first ? [base, candidate] : [candidate, base]
|
|
638
829
|
process = benchmarks.fetch(name).fetch("process")
|
|
639
830
|
replies = if process
|
|
640
831
|
# (Each run of one version's next to one of the other's.)
|
|
@@ -657,7 +848,9 @@ module Duel
|
|
|
657
848
|
# Each counted measure's fewest, on each version, over COUNTS pairs of
|
|
658
849
|
# samples, and what its metric said of those samples.
|
|
659
850
|
def count(name, n, result)
|
|
660
|
-
|
|
851
|
+
return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
|
|
852
|
+
|
|
853
|
+
samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
|
|
661
854
|
metrics.each do |spec|
|
|
662
855
|
metric = spec.fetch("name")
|
|
663
856
|
base, candidate = samples.transpose.map do |side|
|
|
@@ -672,6 +865,61 @@ module Duel
|
|
|
672
865
|
end
|
|
673
866
|
end
|
|
674
867
|
|
|
868
|
+
# Each counted measure, with workers enough for an interval over them
|
|
869
|
+
# (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
|
|
870
|
+
# of a sample from every worker, and how much more the candidate's
|
|
871
|
+
# are than the baseline's, at least (the low end of an interval for
|
|
872
|
+
# it, Stats.difference_interval), which is what's judged (see
|
|
873
|
+
# Result#worse). Where every worker's is the same, as CRuby's counts
|
|
874
|
+
# of objects are, that's exactly how many more. Where they differ, as
|
|
875
|
+
# JRuby's bytes do where one JVM's JIT does away with an allocation
|
|
876
|
+
# and another's doesn't, it's as little as the workers can tell.
|
|
877
|
+
# Each version's own is the median of its workers'.
|
|
878
|
+
def count_by_process(name, n, result)
|
|
879
|
+
k = @bases.size
|
|
880
|
+
# @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
|
|
881
|
+
fewest = Hash.new do |h, metric|
|
|
882
|
+
none = {} #: Hash[Integer, Metric::value]
|
|
883
|
+
h[metric] = {"baseline" => none, "candidate" => none.dup}
|
|
884
|
+
end
|
|
885
|
+
COUNT_ROUNDS.times do |round|
|
|
886
|
+
k.times do |i|
|
|
887
|
+
j = (i + round) % k
|
|
888
|
+
base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
|
|
889
|
+
fewest!(fewest, "baseline", i, base)
|
|
890
|
+
fewest!(fewest, "candidate", j, candidate)
|
|
891
|
+
result.note(base, candidate)
|
|
892
|
+
end
|
|
893
|
+
end
|
|
894
|
+
metrics.each do |spec|
|
|
895
|
+
metric = spec.fetch("name")
|
|
896
|
+
next unless fewest.key?(metric)
|
|
897
|
+
|
|
898
|
+
found = fewest.fetch(metric)
|
|
899
|
+
bases = found.fetch("baseline").values
|
|
900
|
+
candidates = found.fetch("candidate").values
|
|
901
|
+
next if bases.empty? || candidates.empty?
|
|
902
|
+
|
|
903
|
+
result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
|
|
904
|
+
"more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
|
|
905
|
+
"n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
|
|
906
|
+
"unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
|
|
907
|
+
end
|
|
908
|
+
end
|
|
909
|
+
|
|
910
|
+
# Keeps each count of +sample+ (from +side+'s +worker+th worker) in
|
|
911
|
+
# +fewest+, where it's that worker's fewest so far.
|
|
912
|
+
def fewest!(fewest, side, worker, sample)
|
|
913
|
+
sample.fetch("counted").each do |metric, value|
|
|
914
|
+
least = fewest[metric].fetch(side)
|
|
915
|
+
least[worker] = [least[worker], value].compact.min
|
|
916
|
+
end
|
|
917
|
+
end
|
|
918
|
+
|
|
919
|
+
# The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
|
|
920
|
+
# or else COUNTS.
|
|
921
|
+
def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
|
|
922
|
+
|
|
675
923
|
# Each version's least peak memory over MEMORY_SAMPLES fresh workers
|
|
676
924
|
# (a long-lived one has room left over from earlier work), turn about.
|
|
677
925
|
# Fresh means no trial run either: the allocator keeps what that freed,
|
|
@@ -743,6 +991,13 @@ module Duel
|
|
|
743
991
|
# @api private
|
|
744
992
|
attr_reader :profile
|
|
745
993
|
|
|
994
|
+
# Each worker's median of each measure judged by process (see
|
|
995
|
+
# Lane#judge_processes): {metric => {"lane" => its lane's number,
|
|
996
|
+
# "baseline" => [each baseline worker's], "candidate" => [...]}},
|
|
997
|
+
# for across the board.
|
|
998
|
+
# @api private
|
|
999
|
+
attr_reader :processes
|
|
1000
|
+
|
|
746
1001
|
# The measurements again that were too close to call (see #confirm).
|
|
747
1002
|
# @api private
|
|
748
1003
|
attr_reader :undecided
|
|
@@ -758,6 +1013,7 @@ module Duel
|
|
|
758
1013
|
@details = {}
|
|
759
1014
|
@confirmations = []
|
|
760
1015
|
@undecided = []
|
|
1016
|
+
@processes = {}
|
|
761
1017
|
end
|
|
762
1018
|
|
|
763
1019
|
# Keeps what metrics said of a pair of samples (Metric#details): each
|
|
@@ -773,11 +1029,12 @@ module Duel
|
|
|
773
1029
|
|
|
774
1030
|
# The metrics on which the candidate did worse: a timed one slower, or
|
|
775
1031
|
# a counted one up by more than its tolerance per iteration and its
|
|
776
|
-
# allowance per sample
|
|
1032
|
+
# allowance per sample (counted by process, up by that much at least:
|
|
1033
|
+
# see Lane#count_by_process).
|
|
777
1034
|
# @api private
|
|
778
1035
|
def worse
|
|
779
1036
|
more = @counted.select do |_, c|
|
|
780
|
-
!Metric.within?(c.fetch("candidate") - c.fetch("baseline"), c.fetch("n"), tolerance: c.fetch("tolerance"),
|
|
1037
|
+
!Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
|
|
781
1038
|
allowance: c.fetch("allowance"))
|
|
782
1039
|
end
|
|
783
1040
|
slower + more.keys + (more_memory? ? ["peak_memory"] : [])
|
|
@@ -871,6 +1128,14 @@ module Duel
|
|
|
871
1128
|
# benchmark", "2 benchmarks").
|
|
872
1129
|
def self.count(number, noun, plural = "#{noun}s") = "#{number} #{(number == 1) ? noun : plural}"
|
|
873
1130
|
|
|
1131
|
+
# A count, an iteration's (a Float, often): to four figures, or one of
|
|
1132
|
+
# NUMBER or more (where four figures would be 1e+04) as a whole number
|
|
1133
|
+
# (25736 bytes, not 2.574e+04).
|
|
1134
|
+
def self.number(value) = (value.abs >= NUMBER) ? value.round.to_s : format("%.4g", value)
|
|
1135
|
+
|
|
1136
|
+
# The least count #number shows as a whole number.
|
|
1137
|
+
NUMBER = 9_999.5
|
|
1138
|
+
|
|
874
1139
|
# Across the board (see Across): {metric => verdict}, "confirmed" in
|
|
875
1140
|
# those slower in a second measurement too. Set by #run.
|
|
876
1141
|
attr_reader :across
|
|
@@ -962,7 +1227,7 @@ module Duel
|
|
|
962
1227
|
@benchmarks = lanes.first.benchmarks
|
|
963
1228
|
@out.puts "Comparing #{Runner.count(found.size, "benchmark")}, #{@settings.jobs} at a time: baseline #{@base_root} " \
|
|
964
1229
|
"against candidate #{File.expand_path(@settings.candidate_dir || @config.root)} (slower: a " \
|
|
965
|
-
"#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit})"
|
|
1230
|
+
"#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit}, #{interval_over})"
|
|
966
1231
|
rubies = @rubies = lanes.first.rubies
|
|
967
1232
|
rubies.each { |side, ruby| @out.puts " #{side} on #{ruby}" } if rubies.values.uniq.size > 1
|
|
968
1233
|
@out.puts " with plugins #{plugins.join(", ")}" unless plugins.empty?
|
|
@@ -975,6 +1240,23 @@ module Duel
|
|
|
975
1240
|
results
|
|
976
1241
|
end
|
|
977
1242
|
|
|
1243
|
+
# What a benchmark's interval is over (see Lane#judge_processes): the
|
|
1244
|
+
# workers, or the pairs, where they're too few (and says how many it
|
|
1245
|
+
# takes, of the first PROCESSES_CHECKED).
|
|
1246
|
+
# mutant:disable -- fewer than 2 of each never take an interval over them, and the last checked is only a bound
|
|
1247
|
+
def interval_over
|
|
1248
|
+
processes = @settings.processes
|
|
1249
|
+
workers = Runner.count(processes, "worker process", "worker processes")
|
|
1250
|
+
return "over #{workers} of each version" if Lane.by_process?(processes, @settings.confidence)
|
|
1251
|
+
|
|
1252
|
+
least = (1..PROCESSES_CHECKED).find { |k| Lane.by_process?(k, @settings.confidence) }
|
|
1253
|
+
"over pairs: #{workers} of each version are too few for an interval over them" + (least ? ", which takes #{least}" : "")
|
|
1254
|
+
end
|
|
1255
|
+
|
|
1256
|
+
# The most workers of each version #interval_over looks at, for how
|
|
1257
|
+
# many it takes (more would take long to check, and longer to run).
|
|
1258
|
+
PROCESSES_CHECKED = 20
|
|
1259
|
+
|
|
978
1260
|
# Each version, prepared once for the run, before any of its workers
|
|
979
1261
|
# start: its Ruby found (Config#ruby, or --ruby), and its
|
|
980
1262
|
# Config#prepare(:version) blocks run, each given a Resource (named for
|
|
@@ -984,6 +1266,7 @@ module Duel
|
|
|
984
1266
|
Side.unprepared(@config, @settings, @base_root).transform_values do |side|
|
|
985
1267
|
side.ruby = Rubies.find(choice, side.root, side.name)
|
|
986
1268
|
side.env = Rubies.environment(side.ruby)
|
|
1269
|
+
side.engine = Rubies.engine(side.ruby)
|
|
987
1270
|
@config.prepares.fetch(:version).each_with_index do |block, i|
|
|
988
1271
|
resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
|
|
989
1272
|
cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
|
|
@@ -1007,7 +1290,7 @@ module Duel
|
|
|
1007
1290
|
# every benchmark, in fresh processes, and are confirmed if they're
|
|
1008
1291
|
# still slower.
|
|
1009
1292
|
def judge_across(results, names)
|
|
1010
|
-
verdicts =
|
|
1293
|
+
verdicts = across_of(results).verdicts
|
|
1011
1294
|
slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
|
|
1012
1295
|
return verdicts if slower.empty?
|
|
1013
1296
|
|
|
@@ -1020,11 +1303,16 @@ module Duel
|
|
|
1020
1303
|
end
|
|
1021
1304
|
|
|
1022
1305
|
@out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
|
|
1023
|
-
again =
|
|
1306
|
+
again = across_of(with_lanes { |lanes| measure_all(lanes, names) }).verdicts
|
|
1024
1307
|
slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
|
|
1025
1308
|
verdicts
|
|
1026
1309
|
end
|
|
1027
1310
|
|
|
1311
|
+
# Across the board, of +results+, judged over workers where the run
|
|
1312
|
+
# judged benchmarks over them (see Lane.by_process?).
|
|
1313
|
+
# mutant:disable -- whether it's over workers is only how fast: results judged over pairs have no workers' medians to find
|
|
1314
|
+
def across_of(results) = Across.new(results, confidence: @settings.confidence, by_process: Lane.by_process?(@settings.processes, @settings.confidence))
|
|
1315
|
+
|
|
1028
1316
|
def open_lane = Lane.new(@config, @settings, @sides)
|
|
1029
1317
|
|
|
1030
1318
|
def measure_all(lanes, names)
|
|
@@ -1145,7 +1433,7 @@ module Duel
|
|
|
1145
1433
|
candidate = c.fetch("candidate").fdiv(n)
|
|
1146
1434
|
# (No unit its name ends in: "retained_objects 2", not "2 objects".)
|
|
1147
1435
|
named = c.fetch("plural")&.then { |plural| metric.end_with?(plural.tr(" ", "_")) }
|
|
1148
|
-
[format("%s
|
|
1436
|
+
[format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
|
|
1149
1437
|
(unit(candidate, c) unless named)].compact.join(" ")
|
|
1150
1438
|
end
|
|
1151
1439
|
memory = result.memory&.then { |m| ["peak_memory #{mb(m.fetch("baseline"))}→#{mb(m.fetch("candidate"))}"] }
|
|
@@ -1189,7 +1477,7 @@ module Duel
|
|
|
1189
1477
|
def amount(value, n, measured)
|
|
1190
1478
|
each = value.fdiv([n, 1].max)
|
|
1191
1479
|
ns = Metric::TIME_UNITS[measured.fetch("unit")]
|
|
1192
|
-
return [
|
|
1480
|
+
return [Runner.number(each), unit(each, measured)].compact.join(" ") unless ns
|
|
1193
1481
|
|
|
1194
1482
|
time = each * ns
|
|
1195
1483
|
shown, size = Metric::TIME_UNITS.reverse_each.find { |_, unit_ns| time >= unit_ns } || ["ns", 1]
|