duel 0.1.8 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +146 -1
  3. data/README.md +35 -0
  4. data/docs/ci.md +96 -3
  5. data/docs/configuration.md +35 -1
  6. data/docs/how-it-works.md +14 -8
  7. data/docs/plugins.md +16 -0
  8. data/lib/duel/across.rb +1 -6
  9. data/lib/duel/baseline.rb +8 -7
  10. data/lib/duel/cli.rb +45 -19
  11. data/lib/duel/comment.rb +71 -10
  12. data/lib/duel/config.rb +113 -2
  13. data/lib/duel/context.rb +33 -34
  14. data/lib/duel/control.rb +52 -0
  15. data/lib/duel/init.rb +142 -18
  16. data/lib/duel/lane.rb +734 -0
  17. data/lib/duel/lock.rb +34 -4
  18. data/lib/duel/memory.rb +14 -5
  19. data/lib/duel/metric.rb +97 -26
  20. data/lib/duel/native.rb +199 -32
  21. data/lib/duel/platform.rb +31 -0
  22. data/lib/duel/process_sample.rb +74 -49
  23. data/lib/duel/resource.rb +71 -42
  24. data/lib/duel/result.rb +165 -0
  25. data/lib/duel/rubies.rb +21 -3
  26. data/lib/duel/runner.rb +13 -1039
  27. data/lib/duel/spawn.rb +327 -0
  28. data/lib/duel/summary.rb +10 -10
  29. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  30. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  31. data/lib/duel/templates/perf-comment.yml.erb +45 -3
  32. data/lib/duel/templates/perf.yml.erb +143 -7
  33. data/lib/duel/testing.rb +3 -0
  34. data/lib/duel/version.rb +1 -1
  35. data/lib/duel/worker.rb +13 -4
  36. data/lib/duel/worker_process.rb +142 -0
  37. data/lib/duel.rb +27 -3
  38. data/sig/duel/cli.rbs +8 -0
  39. data/sig/duel/comment.rbs +16 -1
  40. data/sig/duel/config.rbs +52 -2
  41. data/sig/duel/context.rbs +4 -31
  42. data/sig/duel/control.rbs +27 -0
  43. data/sig/duel/init.rbs +56 -0
  44. data/sig/duel/lane.rbs +202 -0
  45. data/sig/duel/lock.rbs +8 -1
  46. data/sig/duel/memory.rbs +1 -1
  47. data/sig/duel/metric.rbs +53 -10
  48. data/sig/duel/native.rbs +109 -9
  49. data/sig/duel/platform.rbs +13 -0
  50. data/sig/duel/process_sample.rbs +7 -9
  51. data/sig/duel/resource.rbs +28 -9
  52. data/sig/duel/result.rbs +164 -0
  53. data/sig/duel/rubies.rbs +4 -0
  54. data/sig/duel/runner.rbs +3 -460
  55. data/sig/duel/spawn.rbs +111 -0
  56. data/sig/duel/summary.rbs +3 -2
  57. data/sig/duel/worker_process.rbs +90 -0
  58. data/sig/duel.rbs +17 -3
  59. metadata +121 -107
data/lib/duel/runner.rb CHANGED
@@ -2,17 +2,14 @@
2
2
 
3
3
  require "fileutils"
4
4
  require "rbconfig"
5
- require "socket"
6
- require "tmpdir"
7
5
  require_relative "../duel"
8
- require_relative "stats"
9
- require_relative "profile"
10
6
  require_relative "across"
11
- require_relative "message"
7
+ require_relative "lane"
8
+ require_relative "memory"
9
+ require_relative "profile"
12
10
  require_relative "resource"
11
+ require_relative "result"
13
12
  require_relative "rubies"
14
- require_relative "process_sample"
15
- require_relative "memory"
16
13
 
17
14
  module Duel
18
15
  # The most seconds each benchmark runs before it's measured, unless
@@ -56,6 +53,7 @@ module Duel
56
53
  # The seconds a run measures for, at most, by default (see
57
54
  # Settings#time_limit and Runner#out_of_time?): two hours, as on JRuby,
58
55
  # a benchmark can take half a minute to warm up.
56
+ # @api private
59
57
  TIME_LIMIT = 7200.0
60
58
 
61
59
  # How a run is set up (see Duel::CLI for where each comes from).
@@ -100,1028 +98,6 @@ module Duel
100
98
  def label = LABELS.fetch(name)
101
99
  end
102
100
 
103
- # A worker process (Duel::Worker) running one version of the code.
104
- # @api private
105
- class WorkerProcess
106
- LIB = File.expand_path("..", __dir__)
107
- # Seconds between looks at a worker that's loading, or ending.
108
- POLL = 0.01
109
- # Seconds a worker may take to end, once its connection is closed,
110
- # before it's killed.
111
- STOP_WAIT = 10
112
- # The gems that are Duel's: Duel, and its plugins (duel-redis, say).
113
- DUEL_GEM = /\Aduel(-|\z)/
114
-
115
- attr_reader :label, :benchmarks, :metrics, :memory, :profiler, :yjit
116
-
117
- # The Ruby it runs on (RUBY_DESCRIPTION), the native extensions it
118
- # loaded (see Worker.extensions), those it loaded twice (see
119
- # Worker.twice), and the directories gems are installed in (see
120
- # Worker.gem_dirs).
121
- attr_reader :description, :extensions, :twice, :gem_dirs
122
-
123
- # Where a worker on +ruby+ loads Duel from, ahead of anything else
124
- # (a version's own bundle, which may not have Duel, or an older one):
125
- # the runner's Duel, and every Duel gem (+specs+) the runner loaded
126
- # (its plugins), so both versions are measured with the same. On the
127
- # runner's own Ruby, Duel's dependencies (Fiddle) too: on another,
128
- # their builds wouldn't load.
129
- def self.load_path(ruby, specs: Gem.loaded_specs)
130
- duel = specs.select { |name, _| name.match?(DUEL_GEM) }.values
131
- none = [] #: Array[Gem::BasicSpecification]
132
- dependencies = Rubies.runner?(ruby) ? specs["duel"]&.runtime_dependencies.to_a.filter_map { |d| specs[d.name] } : none
133
- [LIB, *(duel + dependencies).flat_map(&:full_require_paths)].uniq
134
- end
135
-
136
- # +timeout+: the seconds it may take to load, or to answer a request,
137
- # before it's taken for hung (and killed). It runs on +ruby+ (whose
138
- # engine is +engine+), with +env+, and on JRuby, the JVM's options for
139
- # a worker (see Rubies.worker_environment), not given to the processes
140
- # it starts for a process benchmark (see #run): their boot is measured
141
- # as it is.
142
- def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:)
143
- @label = label
144
- @timeout = timeout
145
- @root = root
146
- @dir = dir
147
- @env = env
148
- @ruby = ruby
149
- control = File.join(dir, "#{label}.sock")
150
- server = UNIXServer.new(control)
151
- # (A nil config_path is left out. And the worker gets no other
152
- # worker's connection: TruffleRuby's sockets aren't close-on-exec,
153
- # and one left open in another worker would keep it from ending.)
154
- @pid = Process.spawn(Rubies.worker_environment(env, engine), ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
155
- "-e", "Duel::Worker.main(*ARGV)", root, candidate_root, control, *[config_path].compact, in: File::NULL, close_others: true)
156
- @socket = accept(server, control)
157
- hello = read
158
- @benchmarks = hello.fetch("benchmarks").to_h { |b| [b.fetch("name"), b] }
159
- @metrics = hello.fetch("metrics")
160
- @memory = hello.fetch("memory")
161
- @profiler = hello.fetch("profiler")
162
- @yjit = hello.fetch("yjit")
163
- @description = hello.fetch("ruby")
164
- @extensions = hello.fetch("extensions")
165
- @twice = hello.fetch("twice")
166
- @gem_dirs = hello.fetch("gem_dirs")
167
- end
168
-
169
- # Sends a request, and returns the reply, or raises Failed with its
170
- # error.
171
- def request(op, name, **args)
172
- # (By name: +args+ come by keyword, as Symbols.)
173
- Message.write(@socket, {"op" => op, "name" => name, **args.transform_keys(&:to_s)})
174
- reply = read
175
- raise Failed, "#{@label}: #{reply.fetch("error")}\n #{reply.fetch("backtrace").join("\n ")}" if reply.key?("error")
176
-
177
- reply
178
- end
179
-
180
- def sample(name, n, runs: 1, count: false) = request("sample", name, n: n, runs: runs, count: count)
181
-
182
- # A run of a process benchmark (see DSL#duel_process): +process+, a
183
- # fresh process, started as this worker was (on its Ruby, in its
184
- # version's tree, with its environment), and measured (see
185
- # ProcessSample#run).
186
- def run(process)
187
- # (Code, without a command.)
188
- code = process.fetch("ruby") #: String
189
- command = process.fetch("command") || [@ruby, "-I", File.join(@root, "lib"), "-e", code]
190
- runs = @runs ||= ProcessSample.new(@label, env: @env, chdir: @root, log: File.join(@dir, "#{@label}.log"), timeout: @timeout)
191
- runs.run(command)
192
- end
193
-
194
- # Closing the connection ends the worker. One still running +wait+
195
- # seconds later is killed.
196
- def stop(wait: STOP_WAIT)
197
- @runs&.stop
198
- @socket.close
199
- deadline = monotonic + wait
200
- sleep POLL until (gone = Process.wait(@pid, Process::WNOHANG)) || monotonic > deadline
201
- kill unless gone
202
- rescue Errno::ECHILD # gone already
203
- nil
204
- end
205
-
206
- private
207
-
208
- # The worker's connection, once it has loaded (or Failed if it died,
209
- # or took too long, first).
210
- def accept(server, path)
211
- deadline = monotonic + @timeout
212
- until server.wait_readable(POLL)
213
- raise Failed, "#{@label} worker exited while loading" if Process.wait(@pid, Process::WNOHANG)
214
- hung("load") if monotonic > deadline
215
- end
216
- server.accept
217
- ensure
218
- server.close
219
- File.delete(path)
220
- end
221
-
222
- def read
223
- hung("answer") unless @socket.wait_readable(@timeout)
224
- Message.read(@socket) or raise Failed, "#{@label} worker exited"
225
- end
226
-
227
- # Kills a worker that didn't +what+ within the timeout, and fails.
228
- def hung(what)
229
- kill
230
- raise Failed, "#{@label} worker didn't #{what} within #{@timeout}s (killed: a benchmark that hangs?)"
231
- end
232
-
233
- def kill
234
- Process.kill("KILL", @pid)
235
- Process.wait(@pid)
236
- end
237
-
238
- def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
239
- end
240
-
241
- # What Duel raises when something a run needs fails: a resource that
242
- # didn't start (Resource#failed), two that set a variable differently,
243
- # a worker that died or hung.
244
- class Failed < Error; end
245
-
246
- # Workers of both versions, taking turns: Settings#processes of each
247
- # (PROCESSES, by default). Where things landed in one process's memory
248
- # can make the very same code a few percent faster or slower for that
249
- # process's whole life, so the interval is over the workers, where
250
- # they're enough for one (see #judge_processes), and not over the
251
- # pairs, which share them. Each worker has its own resources
252
- # (Config#prepare(:worker)).
253
- # @api private
254
- class Lane
255
- # CPU time (or a plugin's ratio metric's equivalent) a run aims for:
256
- # long enough that a clock's resolution is lost in it. A sample is the
257
- # fastest of RUNS runs.
258
- TARGET_NS = 1_000_000
259
- RUNS = 5
260
- WARMUP = 5
261
- # A process benchmark's pairs before any are measured (of one run
262
- # each), and the most runs a sample of it takes (unless it says: see
263
- # DSL#duel_process): as many as take PROCESS_SAMPLE_NS of CPU time,
264
- # each sample the fastest of them.
265
- PROCESS_WARMUP = 1
266
- PROCESS_RUNS = 3
267
- PROCESS_SAMPLE_NS = 1_000_000_000
268
- # The most a run may allocate, in bytes (see Sample#heap): it all stays
269
- # until the run is over, as the garbage collector is held off.
270
- HEAP_CAP = 64 << 20
271
- # Pairs are taken in rounds of STEP until the verdict is clear (a
272
- # process benchmark's: see #judge).
273
- STEP = 10
274
- MAX_PAIRS = 60
275
- # A benchmark judged by process (see #judge_processes) is decided from
276
- # its MIN_ROUNDS-th round of a sample from every worker, a sample
277
- # each takes, and up to PROCESS_PAIRS pairs.
278
- MIN_ROUNDS = 4
279
- PROCESS_PAIRS = 120
280
- # Counts are each version's fewest in COUNTS samples, or where a
281
- # version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
282
- # takes one of two values, some 31 bytes an iteration apart, sample by
283
- # sample, and the fewest of more are the lower one.
284
- COUNTS = 3
285
- JRUBY_COUNTS = 10
286
- # A count by process (see #count_by_process) is each worker's fewest
287
- # in COUNT_ROUNDS rounds of a sample from every worker.
288
- COUNT_ROUNDS = 3
289
- # Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
290
- MEMORY_SAMPLES = 3
291
- # A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
292
- # StackProf's a version: enough to tell a few percent apart).
293
- PROFILE_SAMPLES = 500
294
-
295
- attr_reader :profiler
296
-
297
- # Its workers start at once, each with its resources (a database
298
- # server takes seconds), and stop at once. +sides+ are the versions as
299
- # the run prepared them (Runner#prepare), by name.
300
- def initialize(config, settings, sides, processes: settings.processes)
301
- @id = Lane.next_id
302
- @dir = Lane.tmpdir
303
- @resources = []
304
- @workers = []
305
- @lock = Mutex.new
306
- @config = config
307
- @settings = settings
308
- @sides = sides
309
- workers = %w[baseline candidate].product(Array.new(processes) { |i| i }) #: Array[[String, Integer]]
310
- started = in_parallel(workers) { |side, index| spawn(side, index) }
311
- @bases = started.first(processes)
312
- @candidates = started.drop(processes)
313
- @profiler = @candidates.first.profiler
314
- @warm = settings.warm || (started.any?(&:yjit) ? YJIT_WARM : JIT_WARM)
315
- check_builds
316
- rescue Exception # rubocop:disable Lint/RescueException
317
- stop
318
- raise
319
- end
320
-
321
- @ids = 0
322
- @ids_lock = Mutex.new
323
-
324
- # A number of its own for each lane, from 1: what tells one lane's
325
- # workers from another's (see Result#processes).
326
- # mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
327
- def self.next_id = @ids_lock.synchronize { @ids += 1 }
328
-
329
- # Whether +processes+ workers of each version are enough for a
330
- # +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
331
- def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
332
-
333
- # A directory for a lane's sockets, short as a Unix socket's path must
334
- # be (see .short_tmp).
335
- def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
336
-
337
- # The longest +tmpdir+ a lane's directory goes in: a resource's socket
338
- # there (Resource#path) must stay within a Unix socket path's 104 bytes.
339
- SHORT = 40
340
-
341
- # Where lanes' directories go: +tmpdir+ (TMPDIR) if it's that short,
342
- # else /tmp (macOS's TMPDIR is long).
343
- def self.short_tmp(tmpdir = Dir.tmpdir) = (tmpdir.size <= SHORT) ? tmpdir : "/tmp"
344
-
345
- # Opens a lane, yields it and always closes it.
346
- def self.open(config, settings, sides, processes: settings.processes)
347
- # @type var lane: Lane
348
- lane = new(config, settings, sides, processes:)
349
- yield lane
350
- ensure
351
- lane&.stop
352
- end
353
-
354
- def benchmarks = @candidates.first.benchmarks
355
-
356
- # The Ruby each version runs on (RUBY_DESCRIPTION), by version.
357
- def rubies = {"baseline" => @bases.first.description, "candidate" => @candidates.first.description}
358
-
359
- # Stops its workers, then their resources: each worker's last first
360
- # (one may need another), the workers' at once.
361
- def stop
362
- in_parallel(@workers) { |worker| worker.stop }
363
- in_parallel(@resources.group_by(&:name).values) { |resources| resources.reverse_each(&:stop) }
364
- FileUtils.rm_rf(@dir)
365
- end
366
-
367
- # Measures one benchmark on both versions, and returns its Result. (A
368
- # process benchmark's workers have nothing to set up: its samples are
369
- # processes of their own.)
370
- def compare(name)
371
- result = Result.new(name)
372
- started = monotonic
373
- if benchmarks.fetch(name).fetch("process")
374
- measure_process(name, result)
375
- else
376
- set_up(name, result)
377
- return result if result.skipped
378
-
379
- begin
380
- measure(name, result)
381
- ensure
382
- each_worker { |worker| worker.request("teardown", name) }
383
- end
384
- end
385
- result.seconds = monotonic - started
386
- result
387
- end
388
-
389
- # Profiles of the benchmark on each version (+n+ iterations a
390
- # sample): {"baseline" =>, "candidate" =>}, or nil without StackProf.
391
- def profile(name, n)
392
- return unless @profiler
393
-
394
- # @type var sides: Hash[String, WorkerProcess]?
395
- sides = {"baseline" => @bases.first, "candidate" => @candidates.first}
396
- sides.each_value { |worker| worker.request("setup", name) }
397
- sides.transform_values { |worker| worker.request("profile", name, n: n, runs: PROFILE_SAMPLES) }
398
- ensure
399
- sides&.each_value { |worker| worker.request("teardown", name) }
400
- end
401
-
402
- private
403
-
404
- # A worker of +side+, with its own resources, and its version's Ruby
405
- # and environment.
406
- def spawn(side, index)
407
- version = @sides.fetch(side)
408
- label = "#{version.label}-#{index + 1}"
409
- env = version.env
410
- @config.prepares.fetch(:worker).each_with_index do |block, i|
411
- resource = Resource.new(File.join(@dir, "#{label}-#{i + 1}"), label, version.root, timeout: @settings.timeout,
412
- cache: File.join(@config.state, "cache"), ruby: version.ruby, env: version.env)
413
- keep(@resources, resource)
414
- env = Resource.environment(label, env, block.call(resource))
415
- end
416
- worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
417
- config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
418
- keep(@workers, worker)
419
- worker
420
- end
421
-
422
- # Fails if a version's workers loaded a native extension that isn't a
423
- # build of its own: one in the other version's tree, or one where the
424
- # other version loaded the same from its own tree (an installed gem's,
425
- # say). A checkout without a build of its own (see Config#build)
426
- # quietly loads whichever build it finds, and the comparison wouldn't
427
- # be of its code. An installed gem's extension (in the directories
428
- # gems are installed in, by the worker that loaded it) is in no
429
- # version's tree, even in a bundle there (vendor/bundle, say, which
430
- # both versions may load from). And fails if a version's workers
431
- # loaded one twice (see Worker.twice): the candidate's, that something
432
- # required before Duel loaded the version, and then its own over it.
433
- def check_builds
434
- workers = {"baseline" => @bases.first, "candidate" => @candidates.first}
435
- workers.each do |side, worker|
436
- worker.twice.each do |feature, paths|
437
- raise Failed, "the #{side} loaded #{feature} twice, from #{paths.join(" and ")}: something required it before " \
438
- "Duel loaded the #{side}, and both stay loaded. Require what loads it in a block that runs once the version " \
439
- "has loaded (c.in_worker, in .duel.rb), not as .duel.rb or a plugin loads"
440
- end
441
- other = (side == "baseline") ? "candidate" : "baseline"
442
- theirs = workers.fetch(other).then { |w| w.extensions.select { |_, path| inside?(path, other, w) } }
443
- worker.extensions.each do |feature, path|
444
- next if inside?(path, side, worker) || !(inside?(path, other, worker) || theirs.key?(feature))
445
-
446
- raise Failed, "the #{side} loaded #{path}, not a build of its own: #{feature} isn't built in " \
447
- "#{@sides.fetch(side).root}. Set c.build in .duel.rb to build each version (by default, it's " \
448
- "`bundle exec rake compile` for a project with ext/ and a Rakefile)"
449
- end
450
- end
451
- end
452
-
453
- # Whether +path+ (a real path, that +worker+ loaded) is in +side+'s
454
- # tree, and not in an installed gem (in +worker+'s gem directories).
455
- def inside?(path, side, worker)
456
- root = @sides.fetch(side).root
457
- path.start_with?(File.join(File.exist?(root) ? File.realpath(root) : root, "")) &&
458
- worker.gem_dirs.none? { |dir| path.start_with?(dir) }
459
- end
460
-
461
- # Adds +item+ to +list+, from a thread of its own (see #in_parallel).
462
- # mutant:disable -- CRuby's Array#<< is atomic anyway, and JRuby's isn't
463
- def keep(list, item) = @lock.synchronize { list << item }
464
-
465
- def each_worker(&) = (@bases + @candidates).each(&)
466
-
467
- # Sets the benchmark up in every worker, the first candidate first (a
468
- # benchmark that fails there is broken) and the first baseline next
469
- # (one that fails there is new: skipped), then the rest at once.
470
- def set_up(name, result)
471
- @candidates.first.request("setup", name)
472
- begin
473
- @bases.first.request("setup", name)
474
- rescue Failed => e
475
- @candidates.first.request("teardown", name)
476
- result.skipped = "not on the baseline (#{e.message.lines.first.chomp})"
477
- return
478
- end
479
- in_parallel(@bases.drop(1) + @candidates.drop(1)) { |worker| worker.request("setup", name) }
480
- end
481
-
482
- # What's measured: ratio metrics' measures are in a sample's "timed",
483
- # counted ones' in its "counted".
484
- def metrics = @candidates.first.metrics
485
-
486
- # Whether workers start (and stop) at once: not on TruffleRuby, whose
487
- # Process.spawn, closing other descriptors (see WorkerProcess), fails
488
- # when another thread closes one meanwhile.
489
- PARALLEL = RUBY_ENGINE != "truffleruby"
490
-
491
- # What the block returns for each item, each on a thread of its own
492
- # (one after another, unless PARALLEL). Every one finishes before the
493
- # first that failed, if any, raises.
494
- def in_parallel(items, &)
495
- return items.map(&) unless PARALLEL
496
-
497
- threads = items.map do |item|
498
- Thread.new do
499
- Thread.current.report_on_exception = false
500
- yield item
501
- end
502
- end
503
- threads.each do |thread|
504
- thread.join
505
- rescue
506
- nil # (raised by #value, once they're all done)
507
- end
508
- threads.map(&:value)
509
- end
510
-
511
- def measure(name, result)
512
- benchmark = benchmarks.fetch(name)
513
- only = benchmark.fetch("only")
514
- if only == "count"
515
- count(name, result.n = COUNTS_ONLY, result)
516
- return
517
- end
518
-
519
- timings_only = !only.nil? # (the only other choice)
520
- target = timings_only ? TARGET_NS * 10 : TARGET_NS
521
- n = calibrate(name, target, 1)
522
- warm(name, n)
523
- n = calibrate(name, target, n) if @warm.positive? # it may have got faster
524
- floors = Array.new(WARMUP) { |i| pair(name, n, i) }
525
- # (It got faster still, a JIT got to it, say: sized again, unless
526
- # what it allocates holds it there.)
527
- if below_cpu_floor?(floors) && (sized = calibrate(name, target, n)) > n
528
- n = sized
529
- floors = Array.new(WARMUP) { |i| pair(name, n, i) }
530
- end
531
- result.n = n
532
- count(name, n, result) unless timings_only
533
- peak_memory(name, n, result) if benchmark.fetch("peak_memory")
534
- judge(name, n, floors, result)
535
- end
536
-
537
- # A process benchmark's samples (see DSL#duel_process): each the
538
- # fastest of a few fresh processes (see #pair and #process_runs), as
539
- # its one iteration, paired as any other's, after PROCESS_WARMUP pairs
540
- # (the first runs read files the disk's cache hasn't yet), and each
541
- # version's least peak memory, where it can be told.
542
- def measure_process(name, result)
543
- @peaks = {"baseline" => [], "candidate" => []}
544
- result.n = 1
545
- floors = Array.new(PROCESS_WARMUP) { |i| pair(name, 1, i) }
546
- # (A process benchmark's.)
547
- process = benchmarks.fetch(name).fetch("process") #: Definition::process
548
- judge(name, process.fetch("runs") || process_runs(floors), floors, result)
549
- return if @peaks.values.flatten.include?(nil)
550
-
551
- # (Each told, so each least is.)
552
- least = @peaks.transform_values(&:min) #: Hash[String, Integer]
553
- result.memory = least
554
- end
555
-
556
- # The runs a process benchmark's sample takes, by its warm-up's
557
- # samples (+floors+, of a run each): as many as take
558
- # PROCESS_SAMPLE_NS of CPU time on the slower version, up to
559
- # PROCESS_RUNS, and one at least. A short process varies the most, and
560
- # the fastest of a few is steadier, but a long one's pairs take long
561
- # enough as they are.
562
- def process_runs(floors)
563
- longest = floors.flatten.map { |sample| sample.fetch("timed").fetch("cpu") }.max #: Integer
564
- # (Any number, for one that took none at all.)
565
- PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
566
- end
567
-
568
- # Every worker runs the benchmark a few times, and then, all at once
569
- # (none of it is measured, so they needn't take turns), for the
570
- # seconds --warm says (Settings#warm), or else until it's steady (see
571
- # #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
572
- def warm(name, n)
573
- warm_until = monotonic + @warm
574
- in_parallel(@bases + @candidates) do |worker|
575
- 2.times { worker.sample(name, n) }
576
- if @settings.warm
577
- worker.sample(name, n) until monotonic > warm_until
578
- else
579
- steady(worker, name, n, warm_until)
580
- end
581
- end
582
- end
583
-
584
- # A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
585
- # seconds in a row each have a median of its samples no faster, by
586
- # STEADY_GAIN, than the last window that was (so code that creeps
587
- # faster by less a window isn't steady): a JIT has compiled what it
588
- # will. (Not once its fastest sample stops getting
589
- # faster: the more samples, the faster the fastest of them, by
590
- # chance. And not once one window is no faster: a pause, a collection
591
- # say, can slow one.) A small library's code is steady in a few
592
- # seconds, where a big app's JIT keeps compiling, for up to the
593
- # warm-up's seconds.
594
- STEADY = 0.5
595
- STEADY_GAIN = 0.01
596
- STEADY_WINDOWS = 3
597
-
598
- # Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
599
- # until it's steady, or it's +warm_until+.
600
- def steady(worker, name, n, warm_until)
601
- window, now = window(worker, name, n, warm_until)
602
- return if now > warm_until
603
-
604
- fastest = Stats.median(window)
605
- calm = 0
606
- loop do
607
- window, now = window(worker, name, n, warm_until)
608
- return if now > warm_until
609
-
610
- median = Stats.median(window)
611
- if median > fastest * (1 - STEADY_GAIN)
612
- calm += 1
613
- return if calm == STEADY_WINDOWS
614
- else
615
- calm = 0
616
- fastest = median
617
- end
618
- end
619
- end
620
-
621
- # STEADY seconds of +worker+'s samples' CPU time (a sample at least),
622
- # or until +warm_until+, and when they ended.
623
- def window(worker, name, n, warm_until)
624
- window_until = monotonic + STEADY
625
- samples = [] #: Array[Metric::value]
626
- loop do
627
- samples << worker.sample(name, n).fetch("timed").fetch("cpu")
628
- now = monotonic
629
- return [samples, now] if now > window_until || now > warm_until
630
- end
631
- end
632
-
633
- # Takes pairs until every ratio metric worth judging (measured on
634
- # both versions in the warm-up, and above its floors on either: one
635
- # that rises from next to nothing is judged too) is decided, or
636
- # MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
637
- # and each interval over the pairs. That's for a process benchmark, or
638
- # too few workers for an interval over them: otherwise, see
639
- # #judge_processes. A process benchmark's pairs take a process each,
640
- # so it's decided not slower (Result::CLEARED) after any pair too.
641
- # That can only stop it before it would have been found slower, never
642
- # make it slower: it's found slower only at the end of a round, as any
643
- # benchmark is.
644
- def judge(name, n, floors, result)
645
- judged = metrics.select do |spec|
646
- floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
647
- floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
648
- end
649
- return judge_processes(name, n, judged, result) if by_process?(name)
650
-
651
- ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
652
- # @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
653
- raw = judged.to_h { |spec| [spec.fetch("name"), []] }
654
- pending = judged.dup
655
- every = benchmarks.fetch(name).fetch("process")
656
- (0...MAX_PAIRS).each do |i|
657
- break if pending.empty?
658
-
659
- base, candidate = pair(name, n, i)
660
- result.note(base, candidate)
661
- ratios.each do |metric, list|
662
- values = [base.fetch("timed").fetch(metric), candidate.fetch("timed").fetch(metric)] #: [Metric::value, Metric::value]
663
- raw.fetch(metric) << values
664
- list << values.last.fdiv([values.first, 1].max)
665
- end
666
- round = ((i + 1) % STEP).zero?
667
- next unless round || every
668
-
669
- pending.reject! { |spec| decided?(verdict(ratios.fetch(spec.fetch("name")), spec), round) }
670
- end
671
- judged.each do |spec|
672
- list = ratios.fetch(spec.fetch("name"))
673
- medians = raw.fetch(spec.fetch("name")).transpose.map { |values| Stats.median(values) }
674
- result.timed[spec.fetch("name")] = {"median" => Stats.median(list), "limit" => limit(spec), "across" => spec.fetch("across"),
675
- "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
676
- "interval" => Stats.median_interval(list, @settings.confidence),
677
- "pairs" => list.size, "verdict" => verdict(list, spec).to_s,
678
- "baseline" => medians.fetch(0), "candidate" => medians.fetch(1)}
679
- end
680
- end
681
-
682
- # Whether a benchmark's timings are judged by process (see
683
- # #judge_processes): unless it's a process benchmark (each of its
684
- # samples is a fresh process already, so its pairs are independent),
685
- # or there are too few workers of each version for an interval over
686
- # them (Lane.by_process?).
687
- def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
688
-
689
- # Takes rounds of pairs, each a sample from every worker of each
690
- # version (the baseline's in order, paired with the candidate's in
691
- # turn: each round pairs each with another), until every ratio metric
692
- # judged (see #judge) is decided, from the second round on, or
693
- # #rounds run out. Each worker's median is a sample, and the interval
694
- # is over workers (Stats.shift_interval): where a process's luck
695
- # makes it faster or slower than another for its whole life (where
696
- # things landed in its memory, or what a JIT compiled), the interval
697
- # is as wide as that makes it. An interval over pairs would be as
698
- # narrow as if every pair had processes of its own. Each worker's
699
- # medians are kept, by metric, for across the board (Result#processes).
700
- def judge_processes(name, n, judged, result)
701
- k = @bases.size
702
- # @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
703
- values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
704
- pending = judged.dup
705
- pairs = 0
706
- rounds(k).times do |round|
707
- break if pending.empty?
708
-
709
- k.times do |i|
710
- j = (i + round) % k
711
- base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
712
- result.note(base, candidate)
713
- values.each do |metric, sides|
714
- sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
715
- sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
716
- end
717
- pairs += 1
718
- end
719
- next if round + 1 < MIN_ROUNDS
720
-
721
- pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
722
- end
723
- judged.each do |spec|
724
- metric = spec.fetch("name")
725
- each = medians(values.fetch(metric))
726
- bases, candidates = clamped(each)
727
- result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
728
- "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
729
- "interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
730
- "pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
731
- "baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
732
- result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
733
- end
734
- end
735
-
736
- # The most rounds of +k+ workers' pairs: as many as make
737
- # PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
738
- def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
739
-
740
- # Each worker's median, by version, of +values+ (each worker's
741
- # measures, by version).
742
- def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
743
-
744
- # The baseline's medians (of +each+) at least 1, as the ratio's
745
- # denominator (so a ratio to nothing is one to 1, as a pair's is),
746
- # and the candidate's.
747
- def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
748
-
749
- def process_verdict(each, spec)
750
- bases, candidates = clamped(each)
751
- Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
752
- end
753
-
754
- # Whether both versions' CPU time in +floors+ is below its floor: then
755
- # it wouldn't be judged at all.
756
- def below_cpu_floor?(floors)
757
- cpu = metrics.find { |spec| spec.fetch("name") == "cpu" } #: Metric::spec
758
- floors.transpose.none? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, cpu) }
759
- end
760
-
761
- # Whether +samples+ (one version's timed measures) are above +spec+'s
762
- # floors (the median): its own, and its share of CPU time.
763
- def above_floors?(samples, spec)
764
- values = samples.map { |sample| sample.fetch(spec.fetch("name")) }
765
- cpu = samples.map { |sample| sample.fetch("cpu") }
766
- median = Stats.median(values)
767
- median >= spec.fetch("floor") && median * spec.fetch("unit_ns") >= spec.fetch("cpu_floor") * Stats.median(cpu)
768
- end
769
-
770
- def limit(spec) = spec.fetch("limit") || @settings.limit
771
-
772
- # The verdicts that decide a measure at the end of a round (see
773
- # #judge), and after any other pair: only those that clear it.
774
- DECISIVE = {true => [:slower, :faster, :same], false => [:faster, :same]}.freeze #: Hash[bool, Array[Stats::verdict]]
775
-
776
- # Whether +verdict+ decides a measure, at the end of a +round+ or not.
777
- def decided?(verdict, round) = DECISIVE.fetch(round).include?(verdict)
778
-
779
- def verdict(ratios, spec) = Stats.verdict(ratios, limit: limit(spec), confidence: @settings.confidence)
780
-
781
- # The iterations per sample (from +n+ up) whose fastest run takes
782
- # about +target+ on the first candidate, by its largest timed metric
783
- # that sizes samples (CPU time does), found while warming up (twice in
784
- # a row: a cold run may look like enough), and that allocate no more
785
- # than HEAP_CAP a run.
786
- def calibrate(name, target, n)
787
- units = metrics.select { |spec| spec.fetch("sizes_samples") }.to_h { |spec| [spec.fetch("name"), spec.fetch("unit_ns")] }
788
- reached = 0
789
- until reached == 2
790
- sample = pair(name, n, 0, runs: 3).last
791
- # (CPU time sizes samples, and is always measured.)
792
- timed = sample.fetch("timed")
793
- work = units.filter_map { |metric, unit| timed[metric]&.*(unit) }.max #: Metric::value
794
- most = most_iterations(n, sample.fetch("heap"))
795
- if most <= n
796
- # (Finite, when it's at most n.)
797
- return most #: Integer
798
- end
799
-
800
- reached = (work >= target) ? reached + 1 : 0
801
- next unless reached.zero?
802
-
803
- # (An Integer: most is finite, or more.)
804
- n = [(n * (target * 1.2 / work).clamp(2, 100)).ceil, most].min #: Integer
805
- end
806
- n
807
- end
808
-
809
- # The most iterations whose allocations fit HEAP_CAP, by +heap+ bytes
810
- # over +n+ (at least one, and any number when it allocated nothing).
811
- def most_iterations(n, heap)
812
- return Float::INFINITY unless heap.positive?
813
-
814
- [n * HEAP_CAP / heap, 1].max
815
- end
816
-
817
- # The +index+th pair: a sample (of +n+ iterations) from a baseline
818
- # worker and from a candidate worker, each pair of workers in turn,
819
- # baseline first on even pairs. A process benchmark's sample is the
820
- # fastest of +n+ runs (see #measure_process), the two versions' taking
821
- # turns: so interference, which only ever adds time, is mostly left
822
- # out, and weighs on both alike.
823
- def pair(name, n, index, runs: RUNS, count: false)
824
- k = @bases.size
825
- sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
826
- end
827
-
828
- # A sample (of +n+ iterations) from +base+, a baseline worker, and
829
- # from +candidate+, a candidate worker, the baseline's first if
830
- # +base_first+ (see #pair).
831
- def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
832
- order = base_first ? [base, candidate] : [candidate, base]
833
- process = benchmarks.fetch(name).fetch("process")
834
- replies = if process
835
- # (Each run of one version's next to one of the other's.)
836
- each_run = Array.new(n) { order.map { |worker| worker.run(process) } }
837
- order.zip(each_run.transpose).to_h do |worker, samples|
838
- # (Never nil: each worker in order has its runs.)
839
- ran = samples #: Array[ProcessSample::sampled]
840
- [worker, ProcessSample.fastest(ran)]
841
- end
842
- else
843
- order.to_h { |worker| [worker, worker.sample(name, n, runs: runs, count: count)] }
844
- end
845
- if process
846
- @peaks.fetch("baseline") << replies.fetch(base).fetch("peak")
847
- @peaks.fetch("candidate") << replies.fetch(candidate).fetch("peak")
848
- end
849
- [replies.fetch(base), replies.fetch(candidate)]
850
- end
851
-
852
- # Each counted measure's fewest, on each version, over COUNTS pairs of
853
- # samples, and what its metric said of those samples.
854
- def count(name, n, result)
855
- return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
856
-
857
- samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
858
- metrics.each do |spec|
859
- metric = spec.fetch("name")
860
- base, candidate = samples.transpose.map do |side|
861
- side.select { |sample| sample.fetch("counted").key?(metric) }.min_by { |sample| sample.fetch("counted").fetch(metric) }
862
- end
863
- next unless base && candidate
864
-
865
- result.counted[metric] = {"baseline" => base.fetch("counted").fetch(metric), "candidate" => candidate.fetch("counted").fetch(metric),
866
- "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
867
- "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
868
- result.note(base, candidate, metric)
869
- end
870
- end
871
-
872
- # Each counted measure, with workers enough for an interval over them
873
- # (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
874
- # of a sample from every worker, and how much more the candidate's
875
- # are than the baseline's, at least (the low end of an interval for
876
- # it, Stats.difference_interval), which is what's judged (see
877
- # Result#worse). Where every worker's is the same, as CRuby's counts
878
- # of objects are, that's exactly how many more. Where they differ, as
879
- # JRuby's bytes do where one JVM's JIT does away with an allocation
880
- # and another's doesn't, it's as little as the workers can tell.
881
- # Each version's own is the median of its workers'.
882
- def count_by_process(name, n, result)
883
- k = @bases.size
884
- # @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
885
- fewest = Hash.new do |h, metric|
886
- none = {} #: Hash[Integer, Metric::value]
887
- h[metric] = {"baseline" => none, "candidate" => none.dup}
888
- end
889
- COUNT_ROUNDS.times do |round|
890
- k.times do |i|
891
- j = (i + round) % k
892
- base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
893
- fewest!(fewest, "baseline", i, base)
894
- fewest!(fewest, "candidate", j, candidate)
895
- result.note(base, candidate)
896
- end
897
- end
898
- metrics.each do |spec|
899
- metric = spec.fetch("name")
900
- next unless fewest.key?(metric)
901
-
902
- found = fewest.fetch(metric)
903
- bases = found.fetch("baseline").values
904
- candidates = found.fetch("candidate").values
905
- next if bases.empty? || candidates.empty?
906
-
907
- result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
908
- "more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
909
- "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
910
- "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
911
- end
912
- end
913
-
914
- # Keeps each count of +sample+ (from +side+'s +worker+th worker) in
915
- # +fewest+, where it's that worker's fewest so far.
916
- def fewest!(fewest, side, worker, sample)
917
- sample.fetch("counted").each do |metric, value|
918
- least = fewest[metric].fetch(side)
919
- least[worker] = [least[worker], value].compact.min
920
- end
921
- end
922
-
923
- # The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
924
- # or else COUNTS.
925
- def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
926
-
927
- # Each version's least peak memory over MEMORY_SAMPLES fresh workers
928
- # (a long-lived one has room left over from earlier work), turn about.
929
- # Fresh means no trial run either: the allocator keeps what that freed,
930
- # and the measured run would reuse it rather than grow.
931
- def peak_memory(name, n, result)
932
- return unless @candidates.first.memory
933
-
934
- # @type var peaks: Hash[String, Array[Integer]]
935
- peaks = {"baseline" => [], "candidate" => []}
936
- MEMORY_SAMPLES.times do |i|
937
- sides = i.even? ? %w[baseline candidate] : %w[candidate baseline]
938
- sides.each do |side|
939
- peaks.fetch(side) << fresh(side) do |worker|
940
- worker.request("setup", name, trial: false)
941
- worker.request("memory", name, n: n).fetch("peak")
942
- end
943
- end
944
- end
945
- result.memory = peaks.transform_values(&:min)
946
- end
947
-
948
- # Yields a worker of +side+ of its own, set up for nothing yet, then
949
- # stops it and its resources.
950
- def fresh(side)
951
- # @type var workers: Integer
952
- # @type var resources: Integer
953
- workers = @workers.size
954
- resources = @resources.size
955
- worker = spawn(side, @workers.size)
956
- yield worker
957
- ensure
958
- @workers.pop(@workers.size - workers).each(&:stop)
959
- @resources.pop(@resources.size - resources).reverse_each(&:stop)
960
- end
961
-
962
- def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
963
- end
964
-
965
- # What one comparison of a benchmark found (Testing::Comparison#results
966
- # has them): its #name, whether it #passed?, the measures it #regressed
967
- # on, each #timed measure's verdict and each #counted one's counts,
968
- # what metrics said of it (#details), its peak #memory, its iterations
969
- # a sample (#n), the #seconds it took, why it was #skipped or its
970
- # #error, and its #confirmations (the measurements that confirmed it
971
- # was worse, not those too close to call: see #confirm). Its writers,
972
- # and the rest, are @api private.
973
- class Result
974
- # Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
975
- # bytes at least, before it's worse (it's measured as a whole, with the
976
- # garbage collector working, so it isn't exact).
977
- # @api private
978
- MEMORY_RATIO = 1.1
979
-
980
- # @api private
981
- MEMORY_FLOOR = 1 << 20
982
-
983
- # What a sample says when no metric said anything (and what a measure
984
- # it said nothing of did).
985
- # @api private
986
- NOTHING_SAID = {} #: Hash[String, String]
987
-
988
- # The verdicts that say a timing isn't slower.
989
- # @api private
990
- CLEARED = %w[same faster].freeze
991
-
992
- attr_reader :name, :timed, :counted, :details, :confirmations, :n, :seconds, :skipped, :error, :memory
993
-
994
- # Where a regression's time went (see Profile).
995
- # @api private
996
- attr_reader :profile
997
-
998
- # Each worker's median of each measure judged by process (see
999
- # Lane#judge_processes): {metric => {"lane" => its lane's number,
1000
- # "baseline" => [each baseline worker's], "candidate" => [...]}},
1001
- # for across the board.
1002
- # @api private
1003
- attr_reader :processes
1004
-
1005
- # The measurements again that were too close to call (see #confirm).
1006
- # @api private
1007
- attr_reader :undecided
1008
-
1009
- # @api private
1010
- attr_writer :n, :seconds, :skipped, :error, :memory, :profile
1011
-
1012
- # @api private
1013
- def initialize(name)
1014
- @name = name
1015
- @timed = {}
1016
- @counted = {}
1017
- @details = {}
1018
- @confirmations = []
1019
- @undecided = []
1020
- @processes = {}
1021
- end
1022
-
1023
- # Keeps what metrics said of a pair of samples (Metric#details): each
1024
- # measure's latest, by version (only +measure+'s, if given).
1025
- # @api private
1026
- def note(base, candidate, measure = nil)
1027
- {"baseline" => base, "candidate" => candidate}.each do |side, sample|
1028
- said = sample["details"] || NOTHING_SAID
1029
- said = said.slice(measure) if measure
1030
- said.each { |name, text| (@details[name] ||= {}).store(side, text) }
1031
- end
1032
- end
1033
-
1034
- # The metrics on which the candidate did worse: a timed one slower, or
1035
- # a counted one up by more than its tolerance per iteration and its
1036
- # allowance per sample (counted by process, up by that much at least:
1037
- # see Lane#count_by_process).
1038
- # @api private
1039
- def worse
1040
- more = @counted.select do |_, c|
1041
- !Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
1042
- allowance: c.fetch("allowance"))
1043
- end
1044
- slower + more.keys + (more_memory? ? ["peak_memory"] : [])
1045
- end
1046
-
1047
- # The timed metrics on which the candidate was slower.
1048
- # @api private
1049
- def slower = @timed.filter_map { |metric, t| metric if t.fetch("verdict") == "slower" }
1050
-
1051
- # Whether its timings say slower: one is, and none says it isn't
1052
- # (same, or faster).
1053
- # @api private
1054
- def slower_in_time? = slower.any? && @timed.none? { |_, t| CLEARED.include?(t.fetch("verdict")) }
1055
-
1056
- # Whether it's to be measured again (see Runner#confirm): it looks
1057
- # worse in every measurement so far, and there are fewer confirmations
1058
- # than it takes to fail (Runner::CONFIRM, or one more where it's
1059
- # #mixed?).
1060
- # @api private
1061
- def confirming?
1062
- return false if regressed.empty?
1063
-
1064
- @confirmations.size < (mixed? ? Runner::CONFIRM + 1 : Runner::CONFIRM)
1065
- end
1066
-
1067
- # Keeps +again+, a measurement of it again from scratch (see
1068
- # Runner#confirm), as a confirmation, or sets it aside where it's too
1069
- # close to call, up to Runner::CONFIRM times: every timing is still
1070
- # undecided there (none slower, none the same or faster), and each
1071
- # measure it regressed on so far is one of them. One that can't tell
1072
- # neither confirms it nor clears it, so it's measured again instead.
1073
- # Returns whether +again+ was set aside.
1074
- # @api private
1075
- def confirm(again)
1076
- timed = again.timed
1077
- close = timed.all? { |_, t| t.fetch("verdict") == "undecided" } && regressed.all? { |metric| timed.key?(metric) } &&
1078
- @undecided.size < Runner::CONFIRM
1079
- (close ? @undecided : @confirmations) << again
1080
- close
1081
- end
1082
-
1083
- # Whether its peak memory grew by too much (see MEMORY_RATIO).
1084
- # @api private
1085
- def more_memory?
1086
- memory = @memory
1087
- return false unless memory
1088
-
1089
- base = memory.fetch("baseline")
1090
- memory.fetch("candidate") - base > [MEMORY_FLOOR, base * (MEMORY_RATIO - 1)].max
1091
- end
1092
-
1093
- # Worse on a metric in the first measurement and in every confirmation.
1094
- # Or, where no one metric is, slower in time in all of them (see
1095
- # #mixed?): the timed metrics slower in any.
1096
- def regressed
1097
- return [] if @error || @skipped
1098
-
1099
- agreed = measurements.map(&:worse).reduce(:&)
1100
- return agreed unless agreed.empty? && measurements.all?(&:slower_in_time?)
1101
-
1102
- measurements.flat_map(&:slower).uniq
1103
- end
1104
-
1105
- # Whether it regressed in time on one timed metric in some
1106
- # measurements and on another in the rest (CPU time in one,
1107
- # instructions in the next, say), slower in every one with none saying
1108
- # it isn't: that takes one measurement more than one metric does.
1109
- # @api private
1110
- def mixed? = measurements.map(&:worse).reduce(:&).empty? && !regressed.empty?
1111
-
1112
- # Whether it regressed on nothing, and didn't fail.
1113
- def passed? = regressed.empty? && !@error
1114
-
1115
- # Whether a timed measure's verdict was too close to the limit to settle.
1116
- # @api private
1117
- def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "undecided" }
1118
-
1119
- private
1120
-
1121
- # The first measurement and its confirmations.
1122
- def measurements = [self, *@confirmations]
1123
- end
1124
-
1125
101
  # Measures every benchmark, confirms what looks worse, and reports.
1126
102
  # @api private
1127
103
  class Runner
@@ -1140,6 +116,9 @@ module Duel
1140
116
  # The least count #number shows as a whole number.
1141
117
  NUMBER = 9_999.5
1142
118
 
119
+ # Bytes of memory, in megabytes to a tenth ("41.5MB").
120
+ def self.mb(bytes) = format("%.1fMB", bytes / 1_048_576.0)
121
+
1143
122
  # Across the board (see Across): {metric => verdict}, "confirmed" in
1144
123
  # those slower in a second measurement too. Set by #run.
1145
124
  attr_reader :across
@@ -1175,6 +154,7 @@ module Duel
1175
154
 
1176
155
  # Returns the Results.
1177
156
  def run
157
+ CycleClock.prepare
1178
158
  started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
1179
159
  @dir = Lane.tmpdir
1180
160
  begin
@@ -1182,7 +162,7 @@ module Duel
1182
162
  ensure
1183
163
  @prepared.reverse_each(&:stop)
1184
164
  FileUtils.rm_rf(@dir)
1185
- Resource.evict(File.join(@config.state, "cache"))
165
+ Resource.evict(@config.cache)
1186
166
  end
1187
167
  end
1188
168
 
@@ -1272,12 +252,8 @@ module Duel
1272
252
  side.ruby = Rubies.find(choice, side.root, side.name)
1273
253
  side.env = Rubies.environment(side.ruby)
1274
254
  side.engine = Rubies.engine(side.ruby)
1275
- @config.prepares.fetch(:version).each_with_index do |block, i|
1276
- resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
1277
- cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
1278
- @prepared << resource
1279
- side.env = Resource.environment(side.label, side.env, block.call(resource))
1280
- end
255
+ side.env = Resource.prepare(@config.prepares.fetch(:version), File.join(@dir, side.label), side.label, side.root,
256
+ env: side.env, ruby: side.ruby, timeout: @settings.timeout, cache: @config.cache, so_far: true) { |resource| @prepared << resource }
1281
257
  side
1282
258
  end
1283
259
  end
@@ -1497,13 +473,11 @@ module Duel
1497
473
  [format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
1498
474
  (unit(candidate, c) unless named)].compact.join(" ")
1499
475
  end
1500
- memory = result.memory&.then { |m| ["peak_memory #{mb(m.fetch("baseline"))}→#{mb(m.fetch("candidate"))}"] }
476
+ memory = result.memory&.then { |m| ["peak_memory #{Runner.mb(m.fetch("baseline"))}→#{Runner.mb(m.fetch("candidate"))}"] }
1501
477
  format("%s %-48s %5.1fs n=%-6d %s", result.worse.empty? ? " " : "!", result.name, result.seconds, result.n,
1502
478
  (timed + counted + memory.to_a).join(" "))
1503
479
  end
1504
480
 
1505
- def mb(bytes) = format("%.1fMB", bytes / 1_048_576.0)
1506
-
1507
481
  # The unit of +measured+ (a result's measure), for +number+ of it: the
1508
482
  # singular for one, the plural for any other.
1509
483
  def unit(number, measured) = (number == 1) ? measured.fetch("unit") : measured.fetch("plural")