duel 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +146 -1
  3. data/README.md +35 -0
  4. data/docs/ci.md +96 -3
  5. data/docs/configuration.md +35 -1
  6. data/docs/how-it-works.md +14 -8
  7. data/docs/plugins.md +16 -0
  8. data/lib/duel/across.rb +1 -6
  9. data/lib/duel/baseline.rb +8 -7
  10. data/lib/duel/cli.rb +45 -19
  11. data/lib/duel/comment.rb +71 -10
  12. data/lib/duel/config.rb +113 -2
  13. data/lib/duel/context.rb +33 -34
  14. data/lib/duel/control.rb +52 -0
  15. data/lib/duel/init.rb +142 -18
  16. data/lib/duel/lane.rb +734 -0
  17. data/lib/duel/lock.rb +34 -4
  18. data/lib/duel/memory.rb +14 -5
  19. data/lib/duel/metric.rb +97 -26
  20. data/lib/duel/native.rb +199 -32
  21. data/lib/duel/platform.rb +31 -0
  22. data/lib/duel/process_sample.rb +74 -49
  23. data/lib/duel/resource.rb +71 -42
  24. data/lib/duel/result.rb +165 -0
  25. data/lib/duel/rubies.rb +21 -3
  26. data/lib/duel/runner.rb +13 -1039
  27. data/lib/duel/spawn.rb +327 -0
  28. data/lib/duel/summary.rb +10 -10
  29. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  30. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  31. data/lib/duel/templates/perf-comment.yml.erb +45 -3
  32. data/lib/duel/templates/perf.yml.erb +143 -7
  33. data/lib/duel/testing.rb +3 -0
  34. data/lib/duel/version.rb +1 -1
  35. data/lib/duel/worker.rb +13 -4
  36. data/lib/duel/worker_process.rb +142 -0
  37. data/lib/duel.rb +27 -3
  38. data/sig/duel/cli.rbs +8 -0
  39. data/sig/duel/comment.rbs +16 -1
  40. data/sig/duel/config.rbs +52 -2
  41. data/sig/duel/context.rbs +4 -31
  42. data/sig/duel/control.rbs +27 -0
  43. data/sig/duel/init.rbs +56 -0
  44. data/sig/duel/lane.rbs +202 -0
  45. data/sig/duel/lock.rbs +8 -1
  46. data/sig/duel/memory.rbs +1 -1
  47. data/sig/duel/metric.rbs +53 -10
  48. data/sig/duel/native.rbs +109 -9
  49. data/sig/duel/platform.rbs +13 -0
  50. data/sig/duel/process_sample.rbs +7 -9
  51. data/sig/duel/resource.rbs +28 -9
  52. data/sig/duel/result.rbs +164 -0
  53. data/sig/duel/rubies.rbs +4 -0
  54. data/sig/duel/runner.rbs +3 -460
  55. data/sig/duel/spawn.rbs +111 -0
  56. data/sig/duel/summary.rbs +3 -2
  57. data/sig/duel/worker_process.rbs +90 -0
  58. data/sig/duel.rbs +17 -3
  59. metadata +121 -107
data/lib/duel/lane.rb ADDED
@@ -0,0 +1,734 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "fileutils"
4
+ require "tmpdir"
5
+ require_relative "../duel"
6
+ require_relative "platform"
7
+ require_relative "process_sample"
8
+ require_relative "resource"
9
+ require_relative "result"
10
+ require_relative "stats"
11
+ require_relative "worker_process"
12
+
13
+ module Duel
14
+ # Workers of both versions, taking turns: Settings#processes of each
15
+ # (PROCESSES, by default). Where things landed in one process's memory
16
+ # can make the very same code a few percent faster or slower for that
17
+ # process's whole life, so the interval is over the workers, where
18
+ # they're enough for one (see #judge_processes), and not over the
19
+ # pairs, which share them. Each worker has its own resources
20
+ # (Config#prepare(:worker)).
21
+ # @api private
22
+ class Lane
23
+ # CPU time (or a plugin's ratio metric's equivalent) a run aims for:
24
+ # long enough that a clock's resolution is lost in it. A sample is the
25
+ # fastest of RUNS runs.
26
+ TARGET_NS = 1_000_000
27
+ RUNS = 5
28
+ WARMUP = 5
29
+ # A process benchmark's pairs before any are measured (of one run
30
+ # each), and the most runs a sample of it takes (unless it says: see
31
+ # DSL#duel_process): as many as take PROCESS_SAMPLE_NS of CPU time,
32
+ # each sample the fastest of them.
33
+ PROCESS_WARMUP = 1
34
+ PROCESS_RUNS = 3
35
+ PROCESS_SAMPLE_NS = 1_000_000_000
36
+ # The most a run may allocate, in bytes (see Sample#heap): it all stays
37
+ # until the run is over, as the garbage collector is held off.
38
+ HEAP_CAP = 64 << 20
39
+ # Pairs are taken in rounds of STEP until the verdict is clear (a
40
+ # process benchmark's: see #judge).
41
+ STEP = 10
42
+ MAX_PAIRS = 60
43
+ # A benchmark judged by process (see #judge_processes) is decided from
44
+ # its MIN_ROUNDS-th round of a sample from every worker, a sample
45
+ # each takes, and up to PROCESS_PAIRS pairs.
46
+ MIN_ROUNDS = 4
47
+ PROCESS_PAIRS = 120
48
+ # Counts are each version's fewest in COUNTS samples, or where a
49
+ # version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
50
+ # takes one of two values, some 31 bytes an iteration apart, sample by
51
+ # sample, and the fewest of more are the lower one.
52
+ COUNTS = 3
53
+ JRUBY_COUNTS = 10
54
+ # A count by process (see #count_by_process) is each worker's fewest
55
+ # in COUNT_ROUNDS rounds of a sample from every worker.
56
+ COUNT_ROUNDS = 3
57
+ # Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
58
+ MEMORY_SAMPLES = 3
59
+ # A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
60
+ # StackProf's a version: enough to tell a few percent apart).
61
+ PROFILE_SAMPLES = 500
62
+
63
+ attr_reader :profiler
64
+
65
+ # Its workers start at once, each with its resources (a database
66
+ # server takes seconds), and stop at once. +sides+ are the versions as
67
+ # the run prepared them (Runner#prepare), by name.
68
+ def initialize(config, settings, sides, processes: settings.processes)
69
+ @id = Lane.next_id
70
+ @dir = Lane.tmpdir
71
+ @resources = []
72
+ @workers = []
73
+ @lock = Mutex.new
74
+ @config = config
75
+ @settings = settings
76
+ @sides = sides
77
+ workers = %w[baseline candidate].product(Array.new(processes) { |i| i }) #: Array[[String, Integer]]
78
+ started = in_parallel(workers) { |side, index| spawn(side, index) }
79
+ @bases = started.first(processes)
80
+ @candidates = started.drop(processes)
81
+ @profiler = @candidates.first.profiler
82
+ @warm = settings.warm || (started.any?(&:yjit) ? YJIT_WARM : JIT_WARM)
83
+ check_builds
84
+ rescue Exception # rubocop:disable Lint/RescueException
85
+ stop
86
+ raise
87
+ end
88
+
89
+ @ids = 0
90
+ @ids_lock = Mutex.new
91
+
92
+ # A number of its own for each lane, from 1: what tells one lane's
93
+ # workers from another's (see Result#processes).
94
+ # mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
95
+ def self.next_id = @ids_lock.synchronize { @ids += 1 }
96
+
97
+ # Whether +processes+ workers of each version are enough for a
98
+ # +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
99
+ def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
100
+
101
+ # A directory for a lane's sockets, short as a Unix socket's path must
102
+ # be (see .short_tmp).
103
+ def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
104
+
105
+ # The longest +tmpdir+ a lane's directory goes in: a resource's socket
106
+ # there (Resource#path) must stay within a Unix socket path's 104 bytes.
107
+ SHORT = 40
108
+
109
+ # Where lanes' directories go: +tmpdir+ (TMPDIR) if it's that short,
110
+ # else /tmp (macOS's TMPDIR is long). Windows has no /tmp, so its own
111
+ # TMP, however long.
112
+ def self.short_tmp(tmpdir = Dir.tmpdir, windows: Platform::WINDOWS) = (windows || tmpdir.size <= SHORT) ? tmpdir : "/tmp"
113
+
114
+ # Opens a lane, yields it and always closes it.
115
+ def self.open(config, settings, sides, processes: settings.processes)
116
+ # @type var lane: Lane
117
+ lane = new(config, settings, sides, processes:)
118
+ yield lane
119
+ ensure
120
+ lane&.stop
121
+ end
122
+
123
+ def benchmarks = @candidates.first.benchmarks
124
+
125
+ # The Ruby each version runs on (RUBY_DESCRIPTION), by version.
126
+ def rubies = {"baseline" => @bases.first.description, "candidate" => @candidates.first.description}
127
+
128
+ # Stops its workers, then their resources: each worker's last first
129
+ # (one may need another), the workers' at once.
130
+ def stop
131
+ in_parallel(@workers) { |worker| worker.stop }
132
+ in_parallel(@resources.group_by(&:name).values) { |resources| resources.reverse_each(&:stop) }
133
+ FileUtils.rm_rf(@dir)
134
+ end
135
+
136
+ # Measures one benchmark on both versions, and returns its Result. (A
137
+ # process benchmark's workers have nothing to set up: its samples are
138
+ # processes of their own.)
139
+ def compare(name)
140
+ result = Result.new(name)
141
+ started = monotonic
142
+ if benchmarks.fetch(name).fetch("process")
143
+ measure_process(name, result)
144
+ else
145
+ set_up(name, result)
146
+ return result if result.skipped
147
+
148
+ begin
149
+ measure(name, result)
150
+ ensure
151
+ each_worker { |worker| worker.request("teardown", name) }
152
+ end
153
+ end
154
+ result.seconds = monotonic - started
155
+ result
156
+ end
157
+
158
+ # Profiles of the benchmark on each version (+n+ iterations a
159
+ # sample): {"baseline" =>, "candidate" =>}, or nil without StackProf.
160
+ def profile(name, n)
161
+ return unless @profiler
162
+
163
+ # @type var sides: Hash[String, WorkerProcess]?
164
+ sides = {"baseline" => @bases.first, "candidate" => @candidates.first}
165
+ sides.each_value { |worker| worker.request("setup", name) }
166
+ sides.transform_values { |worker| worker.request("profile", name, n: n, runs: PROFILE_SAMPLES) }
167
+ ensure
168
+ sides&.each_value { |worker| worker.request("teardown", name) }
169
+ end
170
+
171
+ private
172
+
173
+ # A worker of +side+, with its own resources, and its version's Ruby
174
+ # and environment.
175
+ def spawn(side, index)
176
+ version = @sides.fetch(side)
177
+ label = "#{version.label}-#{index + 1}"
178
+ env = Resource.prepare(@config.prepares.fetch(:worker), File.join(@dir, label), label, version.root, env: version.env,
179
+ ruby: version.ruby, timeout: @settings.timeout, cache: @config.cache) { |resource| keep(@resources, resource) }
180
+ worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
181
+ config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
182
+ keep(@workers, worker)
183
+ worker
184
+ end
185
+
186
+ # Fails if a version's workers loaded a native extension that isn't a
187
+ # build of its own: one in the other version's tree, or one where the
188
+ # other version loaded the same from its own tree (an installed gem's,
189
+ # say). A checkout without a build of its own (see Config#build)
190
+ # quietly loads whichever build it finds, and the comparison wouldn't
191
+ # be of its code. An installed gem's extension (in the directories
192
+ # gems are installed in, by the worker that loaded it) is in no
193
+ # version's tree, even in a bundle there (vendor/bundle, say, which
194
+ # both versions may load from). And fails if a version's workers
195
+ # loaded one twice (see Worker.twice): the candidate's, that something
196
+ # required before Duel loaded the version, and then its own over it.
197
+ def check_builds
198
+ workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
199
+ workers.each do |side, worker|
200
+ worker.twice.each do |feature, paths|
201
+ raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
202
+ "Duel loaded the #{side}, and both stay loaded. Require what loads it in a block that runs once the version " \
203
+ "has loaded (c.in_worker, in .duel.rb), not as .duel.rb or a plugin loads"
204
+ end
205
+ other = (side == "baseline") ? "candidate" : "baseline"
206
+ theirs = workers.fetch(other).then { |w| w.extensions.select { |_, path| inside?(path, other, w) } }
207
+ worker.extensions.each do |feature, path|
208
+ next if inside?(path, side, worker) || !(inside?(path, other, worker) || theirs.key?(feature))
209
+
210
+ raise Failed, "the #{side} loaded #{path}, not a build of its own: #{feature} isn't built in " \
211
+ "#{@sides.fetch(side).root}. Set c.build in .duel.rb to build each version (by default, it's " \
212
+ "`bundle exec rake compile` for a project with ext/ and a Rakefile)"
213
+ end
214
+ end
215
+ end
216
+
217
+ # Whether +path+ (a real path, that +worker+ loaded) is in +side+'s
218
+ # tree, and not in an installed gem (in +worker+'s gem directories).
219
+ def inside?(path, side, worker)
220
+ root = @sides.fetch(side).root
221
+ path.start_with?(File.join(File.exist?(root) ? File.realpath(root) : root, "")) &&
222
+ worker.gem_dirs.none? { |dir| path.start_with?(dir) }
223
+ end
224
+
225
+ # Adds +item+ to +list+, from a thread of its own (see #in_parallel).
226
+ # mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
227
+ def keep(list, item) = @lock.synchronize { list << item }
228
+
229
+ def each_worker(&) = (@bases + @candidates).each(&)
230
+
231
+ # Sets the benchmark up in every worker, the first candidate first (a
232
+ # benchmark that fails there is broken) and the first baseline next
233
+ # (one that fails there is new: skipped), then the rest at once.
234
+ def set_up(name, result)
235
+ @candidates.first.request("setup", name)
236
+ begin
237
+ @bases.first.request("setup", name)
238
+ rescue Failed => e
239
+ @candidates.first.request("teardown", name)
240
+ result.skipped = "not on the baseline (#{e.message.lines.first.chomp})"
241
+ return
242
+ end
243
+ in_parallel(@bases.drop(1) + @candidates.drop(1)) { |worker| worker.request("setup", name) }
244
+ end
245
+
246
+ # What's measured: ratio metrics' measures are in a sample's "timed",
247
+ # counted ones' in its "counted".
248
+ def metrics = @candidates.first.metrics
249
+
250
+ # Whether workers start (and stop) at once: not on TruffleRuby, whose
251
+ # Process.spawn, closing other descriptors (see WorkerProcess), fails
252
+ # when another thread closes one meanwhile.
253
+ PARALLEL = RUBY_ENGINE != "truffleruby"
254
+
255
+ # What the block returns for each item, each on a thread of its own
256
+ # (one after another, unless PARALLEL). Every one finishes before the
257
+ # first that failed, if any, raises.
258
+ def in_parallel(items, &)
259
+ return items.map(&) unless PARALLEL
260
+
261
+ threads = items.map do |item|
262
+ Thread.new do
263
+ Thread.current.report_on_exception = false
264
+ yield item
265
+ end
266
+ end
267
+ threads.each do |thread|
268
+ thread.join
269
+ rescue
270
+ nil # (raised by #value, once they're all done)
271
+ end
272
+ threads.map(&:value)
273
+ end
274
+
275
+ def measure(name, result)
276
+ benchmark = benchmarks.fetch(name)
277
+ only = benchmark.fetch("only")
278
+ if only == "count"
279
+ count(name, result.n = COUNTS_ONLY, result)
280
+ return
281
+ end
282
+
283
+ timings_only = !only.nil? # (the only other choice)
284
+ target = timings_only ? TARGET_NS * 10 : TARGET_NS
285
+ n = calibrate(name, target, 1)
286
+ warm(name, n)
287
+ n = calibrate(name, target, n) if @warm.positive? # it may have got faster
288
+ floors = Array.new(WARMUP) { |i| pair(name, n, i) }
289
+ # (It got faster still, a JIT got to it, say: sized again, unless
290
+ # what it allocates holds it there.)
291
+ if below_cpu_floor?(floors) && (sized = calibrate(name, target, n)) > n
292
+ n = sized
293
+ floors = Array.new(WARMUP) { |i| pair(name, n, i) }
294
+ end
295
+ result.n = n
296
+ peak_memory(name, n, result) if benchmark.fetch("peak_memory")
297
+ judge(name, n, floors, result)
298
+ # (Counted once it's timed: on JRuby, the bytes a benchmark allocates
299
+ # can fall as its JIT compiles it, so a count right after a short
300
+ # warm-up could find one version's workers further along than the
301
+ # other's.)
302
+ count(name, n, result) unless timings_only
303
+ end
304
+
305
+ # A process benchmark's samples (see DSL#duel_process): each the
306
+ # fastest of a few fresh processes (see #pair and #process_runs), as
307
+ # its one iteration, paired as any other's, after PROCESS_WARMUP pairs
308
+ # (the first runs read files the disk's cache hasn't yet), and each
309
+ # version's least peak memory, where it can be told.
310
+ def measure_process(name, result)
311
+ @peaks = {"baseline" => [], "candidate" => []}
312
+ result.n = 1
313
+ floors = Array.new(PROCESS_WARMUP) { |i| pair(name, 1, i) }
314
+ # (A process benchmark's.)
315
+ process = benchmarks.fetch(name).fetch("process") #: Definition::process
316
+ judge(name, process.fetch("runs") || process_runs(floors), floors, result)
317
+ return if @peaks.values.flatten.include?(nil)
318
+
319
+ # (Each told, so each least is.)
320
+ least = @peaks.transform_values(&:min) #: Hash[String, Integer]
321
+ result.memory = least
322
+ end
323
+
324
+ # The runs a process benchmark's sample takes, by its warm-up's
325
+ # samples (+floors+, of a run each): as many as take
326
+ # PROCESS_SAMPLE_NS of CPU time on the slower version, up to
327
+ # PROCESS_RUNS, and one at least. A short process varies the most, and
328
+ # the fastest of a few is steadier, but a long one's pairs take long
329
+ # enough as they are.
330
+ def process_runs(floors)
331
+ longest = floors.flatten.map { |sample| sample.fetch("timed").fetch("cpu") }.max #: Integer
332
+ # (Any number, for one that took none at all.)
333
+ PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
334
+ end
335
+
336
+ # Every worker runs the benchmark a few times, and then, all at once
337
+ # (none of it is measured, so they needn't take turns), for the
338
+ # seconds --warm says (Settings#warm), or else until it's steady (see
339
+ # #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
340
+ def warm(name, n)
341
+ warm_until = monotonic + @warm
342
+ in_parallel(@bases + @candidates) do |worker|
343
+ 2.times { worker.sample(name, n) }
344
+ if @settings.warm
345
+ worker.sample(name, n) until monotonic > warm_until
346
+ else
347
+ steady(worker, name, n, warm_until)
348
+ end
349
+ end
350
+ end
351
+
352
+ # A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
353
+ # seconds in a row each have a median of its samples no faster, by
354
+ # STEADY_GAIN, than the last window that was (so code that creeps
355
+ # faster by less a window isn't steady): a JIT has compiled what it
356
+ # will. (Not once its fastest sample stops getting
357
+ # faster: the more samples, the faster the fastest of them, by
358
+ # chance. And not once one window is no faster: a pause, a collection
359
+ # say, can slow one.) A small library's code is steady in a few
360
+ # seconds, where a big app's JIT keeps compiling, for up to the
361
+ # warm-up's seconds.
362
+ STEADY = 0.5
363
+ STEADY_GAIN = 0.01
364
+ STEADY_WINDOWS = 3
365
+
366
+ # Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
367
+ # until it's steady, or it's +warm_until+.
368
+ def steady(worker, name, n, warm_until)
369
+ window, now = window(worker, name, n, warm_until)
370
+ return if now > warm_until
371
+
372
+ fastest = Stats.median(window)
373
+ calm = 0
374
+ loop do
375
+ window, now = window(worker, name, n, warm_until)
376
+ return if now > warm_until
377
+
378
+ median = Stats.median(window)
379
+ if median > fastest * (1 - STEADY_GAIN)
380
+ calm += 1
381
+ return if calm == STEADY_WINDOWS
382
+ else
383
+ calm = 0
384
+ fastest = median
385
+ end
386
+ end
387
+ end
388
+
389
+ # STEADY seconds of +worker+'s samples' CPU time (a sample at least),
390
+ # or until +warm_until+, and when they ended.
391
+ def window(worker, name, n, warm_until)
392
+ window_until = monotonic + STEADY
393
+ samples = [] #: Array[Metric::value]
394
+ loop do
395
+ samples << worker.sample(name, n).fetch("timed").fetch("cpu")
396
+ now = monotonic
397
+ return [samples, now] if now > window_until || now > warm_until
398
+ end
399
+ end
400
+
401
+ # Takes pairs until every ratio metric worth judging (measured on
402
+ # both versions in the warm-up, and above its floors on either: one
403
+ # that rises from next to nothing is judged too) is decided, or
404
+ # MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
405
+ # and each interval over the pairs. That's for a process benchmark, or
406
+ # too few workers for an interval over them: otherwise, see
407
+ # #judge_processes. A process benchmark's pairs take a process each,
408
+ # so it's decided not slower (Result::CLEARED) after any pair too.
409
+ # That can only stop it before it would have been found slower, never
410
+ # make it slower: it's found slower only at the end of a round, as any
411
+ # benchmark is.
412
+ def judge(name, n, floors, result)
413
+ judged = metrics.select do |spec|
414
+ floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
415
+ floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
416
+ end
417
+ return judge_processes(name, n, judged, result) if by_process?(name)
418
+
419
+ ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
420
+ # @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
421
+ raw = judged.to_h { |spec| [spec.fetch("name"), []] }
422
+ pending = judged.dup
423
+ every = benchmarks.fetch(name).fetch("process")
424
+ (0...MAX_PAIRS).each do |i|
425
+ break if pending.empty?
426
+
427
+ base, candidate = pair(name, n, i)
428
+ result.note(base, candidate)
429
+ ratios.each do |metric, list|
430
+ values = [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] #: [Metric::value, Metric::value]
431
+ raw.fetch(metric) << values
432
+ list << values.last.fdiv([values.first, 1].max)
433
+ end
434
+ round = ((i + 1) % STEP).zero?
435
+ next unless round || every
436
+
437
+ pending.reject! { |spec| decided?(verdict(ratios.fetch(spec.fetch("name")), spec), round) }
438
+ end
439
+ judged.each do |spec|
440
+ list = ratios.fetch(spec.fetch("name"))
441
+ medians = raw.fetch(spec.fetch("name")).transpose.map { |values| Stats.median(values) }
442
+ result.timed[spec.fetch("name")] = {"median" => Stats.median(list), "limit" => limit(spec), "across" => spec.fetch("across"),
443
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
444
+ "interval" => Stats.median_interval(list, @settings.confidence),
445
+ "pairs" => list.size, "verdict" => verdict(list, spec).to_s,
446
+ "baseline" => medians.fetch(0), "candidate" => medians.fetch(1)}
447
+ end
448
+ end
449
+
450
+ # Whether a benchmark's timings are judged by process (see
451
+ # #judge_processes): unless it's a process benchmark (each of its
452
+ # samples is a fresh process already, so its pairs are independent),
453
+ # or there are too few workers of each version for an interval over
454
+ # them (Lane.by_process?).
455
+ def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
456
+
457
+ # Takes rounds of pairs, each a sample from every worker of each
458
+ # version (the baseline's in order, paired with the candidate's in
459
+ # turn: each round pairs each with another), until every ratio metric
460
+ # judged (see #judge) is decided, from the second round on, or
461
+ # #rounds run out. Each worker's median is a sample, and the interval
462
+ # is over workers (Stats.shift_interval): where a process's luck
463
+ # makes it faster or slower than another for its whole life (where
464
+ # things landed in its memory, or what a JIT compiled), the interval
465
+ # is as wide as that makes it. An interval over pairs would be as
466
+ # narrow as if every pair had processes of its own. Each worker's
467
+ # medians are kept, by metric, for across the board (Result#processes).
468
+ def judge_processes(name, n, judged, result)
469
+ k = @bases.size
470
+ # @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
471
+ values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
472
+ pending = judged.dup
473
+ pairs = 0
474
+ rounds(k).times do |round|
475
+ break if pending.empty?
476
+
477
+ k.times do |i|
478
+ j = (i + round) % k
479
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
480
+ result.note(base, candidate)
481
+ values.each do |metric, sides|
482
+ sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
483
+ sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
484
+ end
485
+ pairs += 1
486
+ end
487
+ next if round + 1 < MIN_ROUNDS
488
+
489
+ pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
490
+ end
491
+ judged.each do |spec|
492
+ metric = spec.fetch("name")
493
+ each = medians(values.fetch(metric))
494
+ bases, candidates = clamped(each)
495
+ result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
496
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
497
+ "interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
498
+ "pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
499
+ "baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
500
+ result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
501
+ end
502
+ end
503
+
504
+ # The most rounds of +k+ workers' pairs: as many as make
505
+ # PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
506
+ def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
507
+
508
+ # Each worker's median, by version, of +values+ (each worker's
509
+ # measures, by version).
510
+ def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
511
+
512
+ # The baseline's medians (of +each+) at least 1, as the ratio's
513
+ # denominator (so a ratio to nothing is one to 1, as a pair's is),
514
+ # and the candidate's.
515
+ def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
516
+
517
+ def process_verdict(each, spec)
518
+ bases, candidates = clamped(each)
519
+ Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
520
+ end
521
+
522
+ # Whether both versions' CPU time in +floors+ is below its floor: then
523
+ # it wouldn't be judged at all.
524
+ def below_cpu_floor?(floors)
525
+ cpu = metrics.find { |spec| spec.fetch("name") == "cpu" } #: Metric::spec
526
+ floors.transpose.none? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, cpu) }
527
+ end
528
+
529
+ # Whether +samples+ (one version's timed measures) are above +spec+'s
530
+ # floors (the median): its own, and its share of CPU time.
531
+ def above_floors?(samples, spec)
532
+ values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
533
+ cpu = samples.map { |sample| sample.fetch("cpu") }
534
+ median = Stats.median(values)
535
+ median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
536
+ end
537
+
538
+ def limit(spec) = spec.fetch("limit") || @settings.limit
539
+
540
+ # The verdicts that decide a measure at the end of a round (see
541
+ # #judge), and after any other pair: only those that clear it.
542
+ DECISIVE = {true => [:slower, :faster, :same], false => [:faster, :same]}.freeze #: Hash[bool, Array[Stats::verdict]]
543
+
544
+ # Whether +verdict+ decides a measure, at the end of a +round+ or not.
545
+ def decided?(verdict, round) = DECISIVE.fetch(round).include?(verdict)
546
+
547
+ def verdict(ratios, spec) = Stats.verdict(ratios, limit: limit(spec), confidence: @settings.confidence)
548
+
549
+ # The iterations per sample (from +n+ up) whose fastest run takes
550
+ # about +target+ on the first candidate, by its largest timed metric
551
+ # that sizes samples (CPU time does), found while warming up (twice in
552
+ # a row: a cold run may look like enough), and that allocate no more
553
+ # than HEAP_CAP a run.
554
+ def calibrate(name, target, n)
555
+ units = metrics.select { |spec| spec.fetch("sizes_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
556
+ reached = 0
557
+ until reached == 2
558
+ sample = pair(name, n, 0, runs: 3).last
559
+ # (CPU time sizes samples, and is always measured.)
560
+ timed = sample.fetch("timed")
561
+ work = units.filter_map { |metric, unit| timed[metric]&.*(unit) }.max #: Metric::value
562
+ most = most_iterations(n, sample.fetch("heap"))
563
+ if most <= n
564
+ # (Finite, when it's at most n.)
565
+ return most #: Integer
566
+ end
567
+
568
+ reached = (work >= target) ? reached + 1 : 0
569
+ next unless reached.zero?
570
+
571
+ # (An Integer: most is finite, or more.)
572
+ n = [(n * (target * 1.2 / work).clamp(2, 100)).ceil, most].min #: Integer
573
+ end
574
+ n
575
+ end
576
+
577
+ # The most iterations whose allocations fit HEAP_CAP, by +heap+ bytes
578
+ # over +n+ (at least one, and any number when it allocated nothing).
579
+ def most_iterations(n, heap)
580
+ return Float::INFINITY unless heap.positive?
581
+
582
+ [n * HEAP_CAP / heap, 1].max
583
+ end
584
+
585
+ # The +index+th pair: a sample (of +n+ iterations) from a baseline
586
+ # worker and from a candidate worker, each pair of workers in turn,
587
+ # baseline first on even pairs. A process benchmark's sample is the
588
+ # fastest of +n+ runs (see #measure_process), the two versions' taking
589
+ # turns: so interference, which only ever adds time, is mostly left
590
+ # out, and weighs on both alike.
591
+ def pair(name, n, index, runs: RUNS, count: false)
592
+ k = @bases.size
593
+ sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
594
+ end
595
+
596
+ # A sample (of +n+ iterations) from +base+, a baseline worker, and
597
+ # from +candidate+, a candidate worker, the baseline's first if
598
+ # +base_first+ (see #pair).
599
+ def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
600
+ order = base_first ? [base, candidate] : [candidate, base]
601
+ process = benchmarks.fetch(name).fetch("process")
602
+ replies = if process
603
+ # (Each run of one version's next to one of the other's.)
604
+ each_run = Array.new(n) { order.map { |worker| worker.run(process) } }
605
+ order.zip(each_run.transpose).to_h do |worker, samples|
606
+ # (Never nil: each worker in order has its runs.)
607
+ ran = samples #: Array[ProcessSample::sampled]
608
+ [worker, ProcessSample.fastest(ran)]
609
+ end
610
+ else
611
+ order.to_h { |worker| [worker, worker.sample(name, n, runs: runs, count: count)] }
612
+ end
613
+ if process
614
+ @peaks.fetch("baseline") << replies.fetch(base).fetch("peak")
615
+ @peaks.fetch("candidate") << replies.fetch(candidate).fetch("peak")
616
+ end
617
+ [replies.fetch(base), replies.fetch(candidate)]
618
+ end
619
+
620
+ # Each counted measure's fewest, on each version, over COUNTS pairs of
621
+ # samples, and what its metric said of those samples.
622
+ def count(name, n, result)
623
+ return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
624
+
625
+ samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
626
+ metrics.each do |spec|
627
+ metric = spec.fetch("name")
628
+ base, candidate = samples.transpose.map do |side|
629
+ side.select { |sample| sample.fetch("counted").key?(metric) }.min_by { |sample| sample.fetch("counted").fetch(metric) }
630
+ end
631
+ next unless base && candidate
632
+
633
+ result.counted[metric] = {"baseline" => base.fetch("counted").fetch(metric), "candidate" => candidate.fetch("counted").fetch(metric),
634
+ "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
635
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
636
+ result.note(base, candidate, metric)
637
+ end
638
+ end
639
+
640
+ # Each counted measure, with workers enough for an interval over them
641
+ # (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
642
+ # of a sample from every worker, and how much more the candidate's
643
+ # are than the baseline's, at least (the low end of an interval for
644
+ # it, Stats.difference_interval), which is what's judged (see
645
+ # Result#worse). Where every worker's is the same, as CRuby's counts
646
+ # of objects are, that's exactly how many more. Where they differ, as
647
+ # JRuby's bytes do where one JVM's JIT does away with an allocation
648
+ # and another's doesn't, it's as little as the workers can tell.
649
+ # Each version's own is the median of its workers'.
650
+ def count_by_process(name, n, result)
651
+ k = @bases.size
652
+ # @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
653
+ fewest = Hash.new do |h, metric|
654
+ none = {} #: Hash[Integer, Metric::value]
655
+ h[metric] = {"baseline" => none, "candidate" => none.dup}
656
+ end
657
+ COUNT_ROUNDS.times do |round|
658
+ k.times do |i|
659
+ j = (i + round) % k
660
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
661
+ fewest!(fewest, "baseline", i, base)
662
+ fewest!(fewest, "candidate", j, candidate)
663
+ # (What metrics said of its counts alone: a timing's is the timed
664
+ # pairs'.)
665
+ base.fetch("counted").each_key { |metric| result.note(base, candidate, metric) }
666
+ end
667
+ end
668
+ metrics.each do |spec|
669
+ metric = spec.fetch("name")
670
+ next unless fewest.key?(metric)
671
+
672
+ found = fewest.fetch(metric)
673
+ bases = found.fetch("baseline").values
674
+ candidates = found.fetch("candidate").values
675
+ next if bases.empty? || candidates.empty?
676
+
677
+ result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
678
+ "more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
679
+ "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
680
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
681
+ end
682
+ end
683
+
684
+ # Keeps each count of +sample+ (from +side+'s +worker+th worker) in
685
+ # +fewest+, where it's that worker's fewest so far.
686
+ def fewest!(fewest, side, worker, sample)
687
+ sample.fetch("counted").each do |metric, value|
688
+ least = fewest[metric].fetch(side)
689
+ least[worker] = [least[worker], value].compact.min
690
+ end
691
+ end
692
+
693
+ # The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
694
+ # or else COUNTS.
695
+ def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
696
+
697
+ # Each version's least peak memory over MEMORY_SAMPLES fresh workers
698
+ # (a long-lived one has room left over from earlier work), turn about.
699
+ # Fresh means no trial run either: the allocator keeps what that freed,
700
+ # and the measured run would reuse it rather than grow.
701
+ def peak_memory(name, n, result)
702
+ return unless @candidates.first.memory
703
+
704
+ # @type var peaks: Hash[String, Array[Integer]]
705
+ peaks = {"baseline" => [], "candidate" => []}
706
+ MEMORY_SAMPLES.times do |i|
707
+ sides = i.even? ? %w[baseline candidate] : %w[candidate baseline]
708
+ sides.each do |side|
709
+ peaks.fetch(side) << fresh(side) do |worker|
710
+ worker.request("setup", name, trial: false)
711
+ worker.request("memory", name, n: n).fetch("peak")
712
+ end
713
+ end
714
+ end
715
+ result.memory = peaks.transform_values(&:min)
716
+ end
717
+
718
+ # Yields a worker of +side+ of its own, set up for nothing yet, then
719
+ # stops it and its resources.
720
+ def fresh(side)
721
+ # @type var workers: Integer
722
+ # @type var resources: Integer
723
+ workers = @workers.size
724
+ resources = @resources.size
725
+ worker = spawn(side, @workers.size)
726
+ yield worker
727
+ ensure
728
+ @workers.pop(@workers.size - workers).each(&:stop)
729
+ @resources.pop(@resources.size - resources).reverse_each(&:stop)
730
+ end
731
+
732
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
733
+ end
734
+ end