duel 0.1.4 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/duel/runner.rb CHANGED
@@ -14,30 +14,56 @@ require_relative "rubies"
14
14
  require_relative "process_sample"
15
15
 
16
16
  module Duel
17
- # Seconds each benchmark runs before it's measured, unless --warm
18
- # says: JRuby's and TruffleRuby's JITs keep compiling for seconds.
17
+ # The most seconds each benchmark runs before it's measured, unless
18
+ # --warm says: until each worker is steady (see Lane#steady), as long
19
+ # as that takes. JRuby's and TruffleRuby's JITs keep compiling for
20
+ # seconds, and some code is still several times faster at 5 than it
21
+ # was, so they get up to a minute.
19
22
  # @api private
20
- JIT_WARM = {"jruby" => 5.0, "truffleruby" => 5.0}.fetch(RUBY_ENGINE, 0.0)
23
+ JIT_WARM = {"jruby" => 60.0, "truffleruby" => 60.0}.fetch(RUBY_ENGINE, 0.0)
21
24
 
22
- # Seconds each benchmark runs before it's measured where a worker has
23
- # YJIT on (an app may turn it on as it boots, as Rails does), unless
24
- # --warm says. A small library's methods are compiled within the
25
- # warm-up samples, but a big app's only after 120 calls each, more than
26
- # a slow benchmark's warm-up samples make.
25
+ # The most seconds each benchmark runs before it's measured where a
26
+ # worker has YJIT on (an app may turn it on as it boots, as Rails
27
+ # does), unless --warm says. A small library's methods are compiled
28
+ # within the warm-up samples, but a big app's only after 120 calls
29
+ # each, more than a slow benchmark's warm-up samples make.
27
30
  # @api private
28
- YJIT_WARM = 2.0
31
+ YJIT_WARM = 10.0
32
+
33
+ # Worker processes of each version, by default: enough for an interval
34
+ # over processes (see Lane#judge_processes), which takes 5 at 99%, and
35
+ # more, as it narrows with each. JRuby's and TruffleRuby's are JVMs,
36
+ # slow to start and big (some 500 MB), so they get fewer.
37
+ # @api private
38
+ PROCESSES = {"jruby" => 8, "truffleruby" => 8}.fetch(RUBY_ENGINE, 12)
39
+
40
+ # The CPUs for each lane of workers, by default (see CLI#parse): half
41
+ # the CPUs make lanes, so that each lane's pair of workers has a CPU of
42
+ # its own. JRuby's and TruffleRuby's JIT compilers keep other CPUs busy,
43
+ # and each of their workers is a big JVM, so they get a lane for every
44
+ # four: more lanes are no faster there, and take more memory.
45
+ # @api private
46
+ CPUS_PER_LANE = {"jruby" => 4, "truffleruby" => 4}.fetch(RUBY_ENGINE, 2)
47
+
48
+ # The seconds a worker may take to load, or to answer a request, by
49
+ # default (see Settings#timeout). JRuby's and TruffleRuby's JVMs take a
50
+ # while to start, and with -Xbatch (see Rubies::JRUBY_WORKER), they
51
+ # compile as they load too: 16 at once can take over a minute.
52
+ # @api private
53
+ TIMEOUT = {"jruby" => 300.0, "truffleruby" => 300.0}.fetch(RUBY_ENGINE, 60.0)
29
54
 
30
55
  # The seconds a run measures for, at most, by default (see
31
- # Settings#time_limit and Runner#out_of_time?).
32
- TIME_LIMIT = 1800.0
56
+ # Settings#time_limit and Runner#out_of_time?): two hours, as on JRuby,
57
+ # a benchmark can take half a minute to warm up.
58
+ TIME_LIMIT = 7200.0
33
59
 
34
60
  # How a run is set up (see Duel::CLI for where each comes from).
35
61
  # @api private
36
62
  Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
37
- :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
63
+ :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit, :accepted) do
38
64
  # @type self: singleton(Settings)
39
- def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
40
- timeout: 60.0, time_limit: TIME_LIMIT)
65
+ def self.defaults = new(filter: //, jobs: 1, processes: PROCESSES, limit: 1.02, confidence: 0.99, warm: nil, force: false,
66
+ timeout: TIMEOUT, time_limit: TIME_LIMIT)
41
67
  end
42
68
 
43
69
  # One version as a run has it (see Runner#prepare), +name+d "baseline"
@@ -45,13 +71,13 @@ module Duel
45
71
  # (Config#ruby), and what they get in their environment (that Ruby's,
46
72
  # and what its Config#prepare(:version) blocks gave it).
47
73
  # @api private
48
- Side = Struct.new(:name, :root, :ruby, :env) do
74
+ Side = Struct.new(:name, :root, :ruby, :env, :engine) do
49
75
  # @type self: singleton(Side)
50
76
 
51
77
  # Each version with nothing prepared: its tree, on the runner's Ruby.
52
78
  def self.unprepared(config, settings, base_root)
53
- {"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}),
54
- "candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {})}
79
+ {"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}, RUBY_ENGINE),
80
+ "candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {}, RUBY_ENGINE)}
55
81
  end
56
82
  end
57
83
 
@@ -104,9 +130,12 @@ module Duel
104
130
  end
105
131
 
106
132
  # +timeout+: the seconds it may take to load, or to answer a request,
107
- # before it's taken for hung (and killed). It runs on +ruby+, with
108
- # +env+.
109
- def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:)
133
+ # before it's taken for hung (and killed). It runs on +ruby+ (whose
134
+ # engine is +engine+), with +env+, and on JRuby, the JVM's options for
135
+ # a worker (see Rubies.worker_environment), not given to the processes
136
+ # it starts for a process benchmark (see #run): their boot is measured
137
+ # as it is.
138
+ def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:)
110
139
  @label = label
111
140
  @timeout = timeout
112
141
  @root = root
@@ -118,7 +147,7 @@ module Duel
118
147
  # (A nil config_path is left out. And the worker gets no other
119
148
  # worker's connection: TruffleRuby's sockets aren't close-on-exec,
120
149
  # and one left open in another worker would keep it from ending.)
121
- @pid = Process.spawn(env, ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
150
+ @pid = Process.spawn(Rubies.worker_environment(env, engine), ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
122
151
  "-e", "Duel::Worker.main(*ARGV)", root, candidate_root, control, *[config_path].compact, in: File::NULL, close_others: true)
123
152
  @socket = accept(server, control)
124
153
  hello = read
@@ -210,12 +239,13 @@ module Duel
210
239
  # a worker that died or hung.
211
240
  class Failed < Error; end
212
241
 
213
- # Workers of both versions, taking turns: PROCESSES of each (Settings),
214
- # so that where things landed in one process's memory (which can make
215
- # the very same code a few percent faster or slower for that process's
216
- # whole life) is spread over many pairs as noise the confidence
217
- # interval accounts for, not a bias every pair shares. Each worker has
218
- # its own resources (Config#prepare(:worker)).
242
+ # Workers of both versions, taking turns: Settings#processes of each
243
+ # (PROCESSES, by default). Where things landed in one process's memory
244
+ # can make the very same code a few percent faster or slower for that
245
+ # process's whole life, so the interval is over the workers, where
246
+ # they're enough for one (see #judge_processes), and not over the
247
+ # pairs, which share them. Each worker has its own resources
248
+ # (Config#prepare(:worker)).
219
249
  # @api private
220
250
  class Lane
221
251
  # CPU time (or a plugin's ratio metric's equivalent) a run aims for:
@@ -238,8 +268,20 @@ module Duel
238
268
  # process benchmark's: see #judge).
239
269
  STEP = 10
240
270
  MAX_PAIRS = 60
241
- # Counts are each version's fewest in COUNTS samples.
271
+ # A benchmark judged by process (see #judge_processes) is decided from
272
+ # its MIN_ROUNDS-th round of a sample from every worker, a sample
273
+ # each takes, and up to PROCESS_PAIRS pairs.
274
+ MIN_ROUNDS = 4
275
+ PROCESS_PAIRS = 120
276
+ # Counts are each version's fewest in COUNTS samples, or where a
277
+ # version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
278
+ # takes one of two values, some 31 bytes an iteration apart, sample by
279
+ # sample, and the fewest of more are the lower one.
242
280
  COUNTS = 3
281
+ JRUBY_COUNTS = 10
282
+ # A count by process (see #count_by_process) is each worker's fewest
283
+ # in COUNT_ROUNDS rounds of a sample from every worker.
284
+ COUNT_ROUNDS = 3
243
285
  # Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
244
286
  MEMORY_SAMPLES = 3
245
287
  # A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
@@ -252,6 +294,7 @@ module Duel
252
294
  # server takes seconds), and stop at once. +sides+ are the versions as
253
295
  # the run prepared them (Runner#prepare), by name.
254
296
  def initialize(config, settings, sides, processes: settings.processes)
297
+ @id = Lane.next_id
255
298
  @dir = Lane.tmpdir
256
299
  @resources = []
257
300
  @workers = []
@@ -271,6 +314,18 @@ module Duel
271
314
  raise
272
315
  end
273
316
 
317
+ @ids = 0
318
+ @ids_lock = Mutex.new
319
+
320
+ # A number of its own for each lane, from 1: what tells one lane's
321
+ # workers from another's (see Result#processes).
322
+ # mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
323
+ def self.next_id = @ids_lock.synchronize { @ids += 1 }
324
+
325
+ # Whether +processes+ workers of each version are enough for a
326
+ # +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
327
+ def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
328
+
274
329
  # A directory for a lane's sockets, short as a Unix socket's path must
275
330
  # be (see .short_tmp).
276
331
  def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
@@ -355,7 +410,7 @@ module Duel
355
410
  env = Resource.environment(label, env, block.call(resource))
356
411
  end
357
412
  worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
358
- config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby)
413
+ config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
359
414
  keep(@workers, worker)
360
415
  worker
361
416
  end
@@ -506,30 +561,89 @@ module Duel
506
561
  PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
507
562
  end
508
563
 
509
- # Every worker runs the benchmark a few times, and for the warm-up's
510
- # seconds (Settings#warm, or JIT_WARM, or YJIT_WARM), all at once:
511
- # none of it is measured, so they needn't take turns.
564
+ # Every worker runs the benchmark a few times, and then, all at once
565
+ # (none of it is measured, so they needn't take turns), for the
566
+ # seconds --warm says (Settings#warm), or else until it's steady (see
567
+ # #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
512
568
  def warm(name, n)
513
569
  warm_until = monotonic + @warm
514
570
  in_parallel(@bases + @candidates) do |worker|
515
571
  2.times { worker.sample(name, n) }
516
- worker.sample(name, n) until monotonic > warm_until
572
+ if @settings.warm
573
+ worker.sample(name, n) until monotonic > warm_until
574
+ else
575
+ steady(worker, name, n, warm_until)
576
+ end
577
+ end
578
+ end
579
+
580
+ # A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
581
+ # seconds in a row each have a median of its samples no faster, by
582
+ # STEADY_GAIN, than the last window that was (so code that creeps
583
+ # faster by less a window isn't steady): a JIT has compiled what it
584
+ # will. (Not once its fastest sample stops getting
585
+ # faster: the more samples, the faster the fastest of them, by
586
+ # chance. And not once one window is no faster: a pause, a collection
587
+ # say, can slow one.) A small library's code is steady in a few
588
+ # seconds, where a big app's JIT keeps compiling, for up to the
589
+ # warm-up's seconds.
590
+ STEADY = 0.5
591
+ STEADY_GAIN = 0.01
592
+ STEADY_WINDOWS = 3
593
+
594
+ # Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
595
+ # until it's steady, or it's +warm_until+.
596
+ def steady(worker, name, n, warm_until)
597
+ window, now = window(worker, name, n, warm_until)
598
+ return if now > warm_until
599
+
600
+ fastest = Stats.median(window)
601
+ calm = 0
602
+ loop do
603
+ window, now = window(worker, name, n, warm_until)
604
+ return if now > warm_until
605
+
606
+ median = Stats.median(window)
607
+ if median > fastest * (1 - STEADY_GAIN)
608
+ calm += 1
609
+ return if calm == STEADY_WINDOWS
610
+ else
611
+ calm = 0
612
+ fastest = median
613
+ end
614
+ end
615
+ end
616
+
617
+ # STEADY seconds of +worker+'s samples' CPU time (a sample at least),
618
+ # or until +warm_until+, and when they ended.
619
+ def window(worker, name, n, warm_until)
620
+ window_until = monotonic + STEADY
621
+ samples = [] #: Array[Metric::value]
622
+ loop do
623
+ samples << worker.sample(name, n).fetch("timed").fetch("cpu")
624
+ now = monotonic
625
+ return [samples, now] if now > window_until || now > warm_until
517
626
  end
518
627
  end
519
628
 
520
629
  # Takes pairs until every ratio metric worth judging (measured on
521
630
  # both versions in the warm-up, and above its floors on either: one
522
631
  # that rises from next to nothing is judged too) is decided, or
523
- # MAX_PAIRS run out: each decided at the end of a round of STEP pairs.
524
- # A process benchmark's pairs take a process each, so it's decided not
525
- # slower (Result::CLEARED) after any pair too. That can only stop it
526
- # before it would have been found slower, never make it slower: it's
527
- # found slower only at the end of a round, as any benchmark is.
632
+ # MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
633
+ # and each interval over the pairs. That's for a process benchmark, or
634
+ # too few workers for an interval over them: otherwise, see
635
+ # #judge_processes. A process benchmark's pairs take a process each,
636
+ # so it's decided not slower (Result::CLEARED) after any pair too.
637
+ # That can only stop it before it would have been found slower, never
638
+ # make it slower: it's found slower only at the end of a round, as any
639
+ # benchmark is.
528
640
  def judge(name, n, floors, result)
529
641
  judged = metrics.select do |spec|
530
642
  floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
531
643
  floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
532
644
  end
645
+ return judge_processes(name, n, judged, result) if by_process?(name)
646
+
533
647
  ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
534
648
  # @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
535
649
  raw = judged.to_h { |spec| [spec.fetch("name"), []] }
@@ -561,6 +675,78 @@ module Duel
561
675
  end
562
676
  end
563
677
 
678
+ # Whether a benchmark's timings are judged by process (see
679
+ # #judge_processes): unless it's a process benchmark (each of its
680
+ # samples is a fresh process already, so its pairs are independent),
681
+ # or there are too few workers of each version for an interval over
682
+ # them (Lane.by_process?).
683
+ def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
684
+
685
+ # Takes rounds of pairs, each a sample from every worker of each
686
+ # version (the baseline's in order, paired with the candidate's in
687
+ # turn: each round pairs each with another), until every ratio metric
688
+ # judged (see #judge) is decided, from the second round on, or
689
+ # #rounds run out. Each worker's median is a sample, and the interval
690
+ # is over workers (Stats.shift_interval): where a process's luck
691
+ # makes it faster or slower than another for its whole life (where
692
+ # things landed in its memory, or what a JIT compiled), the interval
693
+ # is as wide as that makes it. An interval over pairs would be as
694
+ # narrow as if every pair had processes of its own. Each worker's
695
+ # medians are kept, by metric, for across the board (Result#processes).
696
+ def judge_processes(name, n, judged, result)
697
+ k = @bases.size
698
+ # @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
699
+ values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
700
+ pending = judged.dup
701
+ pairs = 0
702
+ rounds(k).times do |round|
703
+ break if pending.empty?
704
+
705
+ k.times do |i|
706
+ j = (i + round) % k
707
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
708
+ result.note(base, candidate)
709
+ values.each do |metric, sides|
710
+ sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
711
+ sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
712
+ end
713
+ pairs += 1
714
+ end
715
+ next if round + 1 < MIN_ROUNDS
716
+
717
+ pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
718
+ end
719
+ judged.each do |spec|
720
+ metric = spec.fetch("name")
721
+ each = medians(values.fetch(metric))
722
+ bases, candidates = clamped(each)
723
+ result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
724
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
725
+ "interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
726
+ "pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
727
+ "baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
728
+ result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
729
+ end
730
+ end
731
+
732
+ # The most rounds of +k+ workers' pairs: as many as make
733
+ # PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
734
+ def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
735
+
736
+ # Each worker's median, by version, of +values+ (each worker's
737
+ # measures, by version).
738
+ def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
739
+
740
+ # The baseline's medians (of +each+) at least 1, as the ratio's
741
+ # denominator (so a ratio to nothing is one to 1, as a pair's is),
742
+ # and the candidate's.
743
+ def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
744
+
745
+ def process_verdict(each, spec)
746
+ bases, candidates = clamped(each)
747
+ Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
748
+ end
749
+
564
750
  # Whether both versions' CPU time in +floors+ is below its floor: then
565
751
  # it wouldn't be judged at all.
566
752
  def below_cpu_floor?(floors)
@@ -632,9 +818,14 @@ module Duel
632
818
  # out, and weighs on both alike.
633
819
  def pair(name, n, index, runs: RUNS, count: false)
634
820
  k = @bases.size
635
- base = @bases.fetch(index % k)
636
- candidate = @candidates.fetch((index + (index / k)) % k)
637
- order = index.even? ? [base, candidate] : [candidate, base]
821
+ sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
822
+ end
823
+
824
+ # A sample (of +n+ iterations) from +base+, a baseline worker, and
825
+ # from +candidate+, a candidate worker, the baseline's first if
826
+ # +base_first+ (see #pair).
827
+ def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
828
+ order = base_first ? [base, candidate] : [candidate, base]
638
829
  process = benchmarks.fetch(name).fetch("process")
639
830
  replies = if process
640
831
  # (Each run of one version's next to one of the other's.)
@@ -657,7 +848,9 @@ module Duel
657
848
  # Each counted measure's fewest, on each version, over COUNTS pairs of
658
849
  # samples, and what its metric said of those samples.
659
850
  def count(name, n, result)
660
- samples = Array.new(COUNTS) { |i| pair(name, n, i, runs: 1, count: true) }
851
+ return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
852
+
853
+ samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
661
854
  metrics.each do |spec|
662
855
  metric = spec.fetch("name")
663
856
  base, candidate = samples.transpose.map do |side|
@@ -672,6 +865,61 @@ module Duel
672
865
  end
673
866
  end
674
867
 
868
+ # Each counted measure, with workers enough for an interval over them
869
+ # (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
870
+ # of a sample from every worker, and how much more the candidate's
871
+ # are than the baseline's, at least (the low end of an interval for
872
+ # it, Stats.difference_interval), which is what's judged (see
873
+ # Result#worse). Where every worker's is the same, as CRuby's counts
874
+ # of objects are, that's exactly how many more. Where they differ, as
875
+ # JRuby's bytes do where one JVM's JIT does away with an allocation
876
+ # and another's doesn't, it's as little as the workers can tell.
877
+ # Each version's own is the median of its workers'.
878
+ def count_by_process(name, n, result)
879
+ k = @bases.size
880
+ # @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
881
+ fewest = Hash.new do |h, metric|
882
+ none = {} #: Hash[Integer, Metric::value]
883
+ h[metric] = {"baseline" => none, "candidate" => none.dup}
884
+ end
885
+ COUNT_ROUNDS.times do |round|
886
+ k.times do |i|
887
+ j = (i + round) % k
888
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
889
+ fewest!(fewest, "baseline", i, base)
890
+ fewest!(fewest, "candidate", j, candidate)
891
+ result.note(base, candidate)
892
+ end
893
+ end
894
+ metrics.each do |spec|
895
+ metric = spec.fetch("name")
896
+ next unless fewest.key?(metric)
897
+
898
+ found = fewest.fetch(metric)
899
+ bases = found.fetch("baseline").values
900
+ candidates = found.fetch("candidate").values
901
+ next if bases.empty? || candidates.empty?
902
+
903
+ result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
904
+ "more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
905
+ "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
906
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
907
+ end
908
+ end
909
+
910
+ # Keeps each count of +sample+ (from +side+'s +worker+th worker) in
911
+ # +fewest+, where it's that worker's fewest so far.
912
+ def fewest!(fewest, side, worker, sample)
913
+ sample.fetch("counted").each do |metric, value|
914
+ least = fewest[metric].fetch(side)
915
+ least[worker] = [least[worker], value].compact.min
916
+ end
917
+ end
918
+
919
+ # The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
920
+ # or else COUNTS.
921
+ def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
922
+
675
923
  # Each version's least peak memory over MEMORY_SAMPLES fresh workers
676
924
  # (a long-lived one has room left over from earlier work), turn about.
677
925
  # Fresh means no trial run either: the allocator keeps what that freed,
@@ -743,6 +991,13 @@ module Duel
743
991
  # @api private
744
992
  attr_reader :profile
745
993
 
994
+ # Each worker's median of each measure judged by process (see
995
+ # Lane#judge_processes): {metric => {"lane" => its lane's number,
996
+ # "baseline" => [each baseline worker's], "candidate" => [...]}},
997
+ # for across the board.
998
+ # @api private
999
+ attr_reader :processes
1000
+
746
1001
  # The measurements again that were too close to call (see #confirm).
747
1002
  # @api private
748
1003
  attr_reader :undecided
@@ -758,6 +1013,7 @@ module Duel
758
1013
  @details = {}
759
1014
  @confirmations = []
760
1015
  @undecided = []
1016
+ @processes = {}
761
1017
  end
762
1018
 
763
1019
  # Keeps what metrics said of a pair of samples (Metric#details): each
@@ -773,11 +1029,12 @@ module Duel
773
1029
 
774
1030
  # The metrics on which the candidate did worse: a timed one slower, or
775
1031
  # a counted one up by more than its tolerance per iteration and its
776
- # allowance per sample.
1032
+ # allowance per sample (counted by process, up by that much at least:
1033
+ # see Lane#count_by_process).
777
1034
  # @api private
778
1035
  def worse
779
1036
  more = @counted.select do |_, c|
780
- !Metric.within?(c.fetch("candidate") - c.fetch("baseline"), c.fetch("n"), tolerance: c.fetch("tolerance"),
1037
+ !Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
781
1038
  allowance: c.fetch("allowance"))
782
1039
  end
783
1040
  slower + more.keys + (more_memory? ? ["peak_memory"] : [])
@@ -871,6 +1128,14 @@ module Duel
871
1128
  # benchmark", "2 benchmarks").
872
1129
  def self.count(number, noun, plural = "#{noun}s") = "#{number} #{(number == 1) ? noun : plural}"
873
1130
 
1131
+ # A count, an iteration's (a Float, often): to four figures, or one of
1132
+ # NUMBER or more (where four figures would be 1e+04) as a whole number
1133
+ # (25736 bytes, not 2.574e+04).
1134
+ def self.number(value) = (value.abs >= NUMBER) ? value.round.to_s : format("%.4g", value)
1135
+
1136
+ # The least count #number shows as a whole number.
1137
+ NUMBER = 9_999.5
1138
+
874
1139
  # Across the board (see Across): {metric => verdict}, "confirmed" in
875
1140
  # those slower in a second measurement too. Set by #run.
876
1141
  attr_reader :across
@@ -962,7 +1227,7 @@ module Duel
962
1227
  @benchmarks = lanes.first.benchmarks
963
1228
  @out.puts "Comparing #{Runner.count(found.size, "benchmark")}, #{@settings.jobs} at a time: baseline #{@base_root} " \
964
1229
  "against candidate #{File.expand_path(@settings.candidate_dir || @config.root)} (slower: a " \
965
- "#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit})"
1230
+ "#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit}, #{interval_over})"
966
1231
  rubies = @rubies = lanes.first.rubies
967
1232
  rubies.each { |side, ruby| @out.puts " #{side} on #{ruby}" } if rubies.values.uniq.size > 1
968
1233
  @out.puts " with plugins #{plugins.join(", ")}" unless plugins.empty?
@@ -975,6 +1240,23 @@ module Duel
975
1240
  results
976
1241
  end
977
1242
 
1243
+ # What a benchmark's interval is over (see Lane#judge_processes): the
1244
+ # workers, or the pairs, where they're too few (and says how many it
1245
+ # takes, of the first PROCESSES_CHECKED).
1246
+ # mutant:disable -- fewer than 2 of each never take an interval over them, and the last checked is only a bound
1247
+ def interval_over
1248
+ processes = @settings.processes
1249
+ workers = Runner.count(processes, "worker process", "worker processes")
1250
+ return "over #{workers} of each version" if Lane.by_process?(processes, @settings.confidence)
1251
+
1252
+ least = (1..PROCESSES_CHECKED).find { |k| Lane.by_process?(k, @settings.confidence) }
1253
+ "over pairs: #{workers} of each version are too few for an interval over them" + (least ? ", which takes #{least}" : "")
1254
+ end
1255
+
1256
+ # The most workers of each version #interval_over looks at, for how
1257
+ # many it takes (more would take long to check, and longer to run).
1258
+ PROCESSES_CHECKED = 20
1259
+
978
1260
  # Each version, prepared once for the run, before any of its workers
979
1261
  # start: its Ruby found (Config#ruby, or --ruby), and its
980
1262
  # Config#prepare(:version) blocks run, each given a Resource (named for
@@ -984,6 +1266,7 @@ module Duel
984
1266
  Side.unprepared(@config, @settings, @base_root).transform_values do |side|
985
1267
  side.ruby = Rubies.find(choice, side.root, side.name)
986
1268
  side.env = Rubies.environment(side.ruby)
1269
+ side.engine = Rubies.engine(side.ruby)
987
1270
  @config.prepares.fetch(:version).each_with_index do |block, i|
988
1271
  resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
989
1272
  cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
@@ -1007,7 +1290,7 @@ module Duel
1007
1290
  # every benchmark, in fresh processes, and are confirmed if they're
1008
1291
  # still slower.
1009
1292
  def judge_across(results, names)
1010
- verdicts = Across.new(results, confidence: @settings.confidence).verdicts
1293
+ verdicts = across_of(results).verdicts
1011
1294
  slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
1012
1295
  return verdicts if slower.empty?
1013
1296
 
@@ -1020,11 +1303,16 @@ module Duel
1020
1303
  end
1021
1304
 
1022
1305
  @out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
1023
- again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
1306
+ again = across_of(with_lanes { |lanes| measure_all(lanes, names) }).verdicts
1024
1307
  slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
1025
1308
  verdicts
1026
1309
  end
1027
1310
 
1311
+ # Across the board, of +results+, judged over workers where the run
1312
+ # judged benchmarks over them (see Lane.by_process?).
1313
+ # mutant:disable -- whether it's over workers is only how fast: results judged over pairs have no workers' medians to find
1314
+ def across_of(results) = Across.new(results, confidence: @settings.confidence, by_process: Lane.by_process?(@settings.processes, @settings.confidence))
1315
+
1028
1316
  def open_lane = Lane.new(@config, @settings, @sides)
1029
1317
 
1030
1318
  def measure_all(lanes, names)
@@ -1145,7 +1433,7 @@ module Duel
1145
1433
  candidate = c.fetch("candidate").fdiv(n)
1146
1434
  # (No unit its name ends in: "retained_objects 2", not "2 objects".)
1147
1435
  named = c.fetch("plural")&.then { |plural| metric.end_with?(plural.tr(" ", "_")) }
1148
- [format("%s %.4g%s", metric, base, (format("→%.4g", candidate) unless base == candidate)),
1436
+ [format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
1149
1437
  (unit(candidate, c) unless named)].compact.join(" ")
1150
1438
  end
1151
1439
  memory = result.memory&.then { |m| ["peak_memory #{mb(m.fetch("baseline"))}→#{mb(m.fetch("candidate"))}"] }
@@ -1189,7 +1477,7 @@ module Duel
1189
1477
  def amount(value, n, measured)
1190
1478
  each = value.fdiv([n, 1].max)
1191
1479
  ns = Metric::TIME_UNITS[measured.fetch("unit")]
1192
- return [format("%.4g", each), unit(each, measured)].compact.join(" ") unless ns
1480
+ return [Runner.number(each), unit(each, measured)].compact.join(" ") unless ns
1193
1481
 
1194
1482
  time = each * ns
1195
1483
  shown, size = Metric::TIME_UNITS.reverse_each.find { |_, unit_ns| time >= unit_ns } || ["ns", 1]