duel 0.1.5 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/duel/runner.rb CHANGED
@@ -12,32 +12,62 @@ require_relative "message"
12
12
  require_relative "resource"
13
13
  require_relative "rubies"
14
14
  require_relative "process_sample"
15
+ require_relative "memory"
15
16
 
16
17
  module Duel
17
- # Seconds each benchmark runs before it's measured, unless --warm
18
- # says: JRuby's and TruffleRuby's JITs keep compiling for seconds.
18
+ # The most seconds each benchmark runs before it's measured, unless
19
+ # --warm says: until each worker is steady (see Lane#steady), as long
20
+ # as that takes. JRuby's and TruffleRuby's JITs keep compiling for
21
+ # seconds, and some code is still several times faster at 5 than it
22
+ # was, so they get up to a minute.
19
23
  # @api private
20
- JIT_WARM = {"jruby" => 5.0, "truffleruby" => 5.0}.fetch(RUBY_ENGINE, 0.0)
24
+ JIT_WARM = {"jruby" => 60.0, "truffleruby" => 60.0}.fetch(RUBY_ENGINE, 0.0)
21
25
 
22
- # Seconds each benchmark runs before it's measured where a worker has
23
- # YJIT on (an app may turn it on as it boots, as Rails does), unless
24
- # --warm says. A small library's methods are compiled within the
25
- # warm-up samples, but a big app's only after 120 calls each, more than
26
- # a slow benchmark's warm-up samples make.
26
+ # The most seconds each benchmark runs before it's measured where a
27
+ # worker has YJIT on (an app may turn it on as it boots, as Rails
28
+ # does), unless --warm says. A small library's methods are compiled
29
+ # within the warm-up samples, but a big app's only after 120 calls
30
+ # each, more than a slow benchmark's warm-up samples make.
27
31
  # @api private
28
- YJIT_WARM = 2.0
32
+ YJIT_WARM = 10.0
33
+
34
+ # Worker processes of each version, by default: enough for an interval
35
+ # over processes (see Lane#judge_processes), which takes 5 at 99%, and
36
+ # more, as it narrows with each. JRuby's and TruffleRuby's are JVMs,
37
+ # slow to start and big (some 500 MB), so they get fewer.
38
+ # @api private
39
+ PROCESSES = {"jruby" => 8, "truffleruby" => 8}.fetch(RUBY_ENGINE, 12)
40
+
41
+ # The CPUs for each lane of workers, by default (see CLI#parse): half
42
+ # the CPUs make lanes, so that each lane's pair of workers has a CPU of
43
+ # its own. JRuby's and TruffleRuby's JIT compilers keep other CPUs busy,
44
+ # and each of their workers is a big JVM, so they get a lane for every
45
+ # four: more lanes are no faster there, and take more memory.
46
+ # @api private
47
+ CPUS_PER_LANE = {"jruby" => 4, "truffleruby" => 4}.fetch(RUBY_ENGINE, 2)
48
+
49
+ # The seconds a worker may take to load, or to answer a request, by
50
+ # default (see Settings#timeout). JRuby's and TruffleRuby's JVMs take a
51
+ # while to start, and with -Xbatch (see Rubies::JRUBY_WORKER), they
52
+ # compile as they load too: 16 at once can take over a minute.
53
+ # @api private
54
+ TIMEOUT = {"jruby" => 300.0, "truffleruby" => 300.0}.fetch(RUBY_ENGINE, 60.0)
29
55
 
30
56
  # The seconds a run measures for, at most, by default (see
31
- # Settings#time_limit and Runner#out_of_time?).
32
- TIME_LIMIT = 1800.0
57
+ # Settings#time_limit and Runner#out_of_time?): two hours, as on JRuby,
58
+ # a benchmark can take half a minute to warm up.
59
+ TIME_LIMIT = 7200.0
33
60
 
34
61
  # How a run is set up (see Duel::CLI for where each comes from).
35
62
  # @api private
63
+ # Of +jobs+ and +processes+, +fit+ names those Duel may lower to fit
64
+ # the memory available (see Runner#fit): the defaults, but not what
65
+ # an option set.
36
66
  Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
37
- :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit) do
67
+ :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit, :accepted, :fit) do
38
68
  # @type self: singleton(Settings)
39
- def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
40
- timeout: 60.0, time_limit: TIME_LIMIT)
69
+ def self.defaults = new(filter: //, jobs: 1, processes: PROCESSES, limit: 1.02, confidence: 0.99, warm: nil, force: false,
70
+ timeout: TIMEOUT, time_limit: TIME_LIMIT, fit: [])
41
71
  end
42
72
 
43
73
  # One version as a run has it (see Runner#prepare), +name+d "baseline"
@@ -45,13 +75,13 @@ module Duel
45
75
  # (Config#ruby), and what they get in their environment (that Ruby's,
46
76
  # and what its Config#prepare(:version) blocks gave it).
47
77
  # @api private
48
- Side = Struct.new(:name, :root, :ruby, :env) do
78
+ Side = Struct.new(:name, :root, :ruby, :env, :engine) do
49
79
  # @type self: singleton(Side)
50
80
 
51
81
  # Each version with nothing prepared: its tree, on the runner's Ruby.
52
82
  def self.unprepared(config, settings, base_root)
53
- {"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}),
54
- "candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {})}
83
+ {"baseline" => new("baseline", File.expand_path(base_root), RbConfig.ruby, {}, RUBY_ENGINE),
84
+ "candidate" => new("candidate", File.expand_path(settings.candidate_dir || config.root), RbConfig.ruby, {}, RUBY_ENGINE)}
55
85
  end
56
86
  end
57
87
 
@@ -104,9 +134,12 @@ module Duel
104
134
  end
105
135
 
106
136
  # +timeout+: the seconds it may take to load, or to answer a request,
107
- # before it's taken for hung (and killed). It runs on +ruby+, with
108
- # +env+.
109
- def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:)
137
+ # before it's taken for hung (and killed). It runs on +ruby+ (whose
138
+ # engine is +engine+), with +env+, and on JRuby, the JVM's options for
139
+ # a worker (see Rubies.worker_environment), not given to the processes
140
+ # it starts for a process benchmark (see #run): their boot is measured
141
+ # as it is.
142
+ def initialize(label, root, candidate_root, dir, config_path:, env:, timeout:, ruby:, engine:)
110
143
  @label = label
111
144
  @timeout = timeout
112
145
  @root = root
@@ -118,7 +151,7 @@ module Duel
118
151
  # (A nil config_path is left out. And the worker gets no other
119
152
  # worker's connection: TruffleRuby's sockets aren't close-on-exec,
120
153
  # and one left open in another worker would keep it from ending.)
121
- @pid = Process.spawn(env, ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
154
+ @pid = Process.spawn(Rubies.worker_environment(env, engine), ruby, *WorkerProcess.load_path(ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/worker",
122
155
  "-e", "Duel::Worker.main(*ARGV)", root, candidate_root, control, *[config_path].compact, in: File::NULL, close_others: true)
123
156
  @socket = accept(server, control)
124
157
  hello = read
@@ -210,12 +243,13 @@ module Duel
210
243
  # a worker that died or hung.
211
244
  class Failed < Error; end
212
245
 
213
- # Workers of both versions, taking turns: PROCESSES of each (Settings),
214
- # so that where things landed in one process's memory (which can make
215
- # the very same code a few percent faster or slower for that process's
216
- # whole life) is spread over many pairs as noise the confidence
217
- # interval accounts for, not a bias every pair shares. Each worker has
218
- # its own resources (Config#prepare(:worker)).
246
+ # Workers of both versions, taking turns: Settings#processes of each
247
+ # (PROCESSES, by default). Where things landed in one process's memory
248
+ # can make the very same code a few percent faster or slower for that
249
+ # process's whole life, so the interval is over the workers, where
250
+ # they're enough for one (see #judge_processes), and not over the
251
+ # pairs, which share them. Each worker has its own resources
252
+ # (Config#prepare(:worker)).
219
253
  # @api private
220
254
  class Lane
221
255
  # CPU time (or a plugin's ratio metric's equivalent) a run aims for:
@@ -238,8 +272,20 @@ module Duel
238
272
  # process benchmark's: see #judge).
239
273
  STEP = 10
240
274
  MAX_PAIRS = 60
241
- # Counts are each version's fewest in COUNTS samples.
275
+ # A benchmark judged by process (see #judge_processes) is decided from
276
+ # its MIN_ROUNDS-th round of a sample from every worker, a sample
277
+ # each takes, and up to PROCESS_PAIRS pairs.
278
+ MIN_ROUNDS = 4
279
+ PROCESS_PAIRS = 120
280
+ # Counts are each version's fewest in COUNTS samples, or where a
281
+ # version is on JRuby, JRUBY_COUNTS: its count of bytes allocated
282
+ # takes one of two values, some 31 bytes an iteration apart, sample by
283
+ # sample, and the fewest of more are the lower one.
242
284
  COUNTS = 3
285
+ JRUBY_COUNTS = 10
286
+ # A count by process (see #count_by_process) is each worker's fewest
287
+ # in COUNT_ROUNDS rounds of a sample from every worker.
288
+ COUNT_ROUNDS = 3
243
289
  # Peak memory is each version's least in MEMORY_SAMPLES fresh workers.
244
290
  MEMORY_SAMPLES = 3
245
291
  # A profile: PROFILE_SAMPLES samples of the benchmark (some 5,000 of
@@ -252,6 +298,7 @@ module Duel
252
298
  # server takes seconds), and stop at once. +sides+ are the versions as
253
299
  # the run prepared them (Runner#prepare), by name.
254
300
  def initialize(config, settings, sides, processes: settings.processes)
301
+ @id = Lane.next_id
255
302
  @dir = Lane.tmpdir
256
303
  @resources = []
257
304
  @workers = []
@@ -271,6 +318,18 @@ module Duel
271
318
  raise
272
319
  end
273
320
 
321
+ @ids = 0
322
+ @ids_lock = Mutex.new
323
+
324
+ # A number of its own for each lane, from 1: what tells one lane's
325
+ # workers from another's (see Result#processes).
326
+ # mutant:disable -- CRuby doesn't switch threads in the middle of += on an Integer, and JRuby can
327
+ def self.next_id = @ids_lock.synchronize { @ids += 1 }
328
+
329
+ # Whether +processes+ workers of each version are enough for a
330
+ # +confidence+ interval over them (see Stats.shift_rank): 5 at 99%.
331
+ def self.by_process?(processes, confidence) = !Stats.shift_rank(processes, processes, confidence).nil?
332
+
274
333
  # A directory for a lane's sockets, short as a Unix socket's path must
275
334
  # be (see .short_tmp).
276
335
  def self.tmpdir = Dir.mktmpdir("duel", short_tmp)
@@ -355,7 +414,7 @@ module Duel
355
414
  env = Resource.environment(label, env, block.call(resource))
356
415
  end
357
416
  worker = WorkerProcess.new(label, version.root, @sides.fetch("candidate").root, @dir,
358
- config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby)
417
+ config_path: @settings.config_path, env: env, timeout: @settings.timeout, ruby: version.ruby, engine: version.engine)
359
418
  keep(@workers, worker)
360
419
  worker
361
420
  end
@@ -506,30 +565,89 @@ module Duel
506
565
  PROCESS_SAMPLE_NS.fdiv(longest).clamp(1, PROCESS_RUNS).floor
507
566
  end
508
567
 
509
- # Every worker runs the benchmark a few times, and for the warm-up's
510
- # seconds (Settings#warm, or JIT_WARM, or YJIT_WARM), all at once:
511
- # none of it is measured, so they needn't take turns.
568
+ # Every worker runs the benchmark a few times, and then, all at once
569
+ # (none of it is measured, so they needn't take turns), for the
570
+ # seconds --warm says (Settings#warm), or else until it's steady (see
571
+ # #steady), for JIT_WARM's or YJIT_WARM's seconds at most.
512
572
  def warm(name, n)
513
573
  warm_until = monotonic + @warm
514
574
  in_parallel(@bases + @candidates) do |worker|
515
575
  2.times { worker.sample(name, n) }
516
- worker.sample(name, n) until monotonic > warm_until
576
+ if @settings.warm
577
+ worker.sample(name, n) until monotonic > warm_until
578
+ else
579
+ steady(worker, name, n, warm_until)
580
+ end
581
+ end
582
+ end
583
+
584
+ # A worker's warm-up ends once STEADY_WINDOWS windows of STEADY
585
+ # seconds in a row each have a median of its samples no faster, by
586
+ # STEADY_GAIN, than the last window that was (so code that creeps
587
+ # faster by less a window isn't steady): a JIT has compiled what it
588
+ # will. (Not once its fastest sample stops getting
589
+ # faster: the more samples, the faster the fastest of them, by
590
+ # chance. And not once one window is no faster: a pause, a collection
591
+ # say, can slow one.) A small library's code is steady in a few
592
+ # seconds, where a big app's JIT keeps compiling, for up to the
593
+ # warm-up's seconds.
594
+ STEADY = 0.5
595
+ STEADY_GAIN = 0.01
596
+ STEADY_WINDOWS = 3
597
+
598
+ # Samples +worker+ (+n+ iterations each), STEADY seconds at a time,
599
+ # until it's steady, or it's +warm_until+.
600
+ def steady(worker, name, n, warm_until)
601
+ window, now = window(worker, name, n, warm_until)
602
+ return if now > warm_until
603
+
604
+ fastest = Stats.median(window)
605
+ calm = 0
606
+ loop do
607
+ window, now = window(worker, name, n, warm_until)
608
+ return if now > warm_until
609
+
610
+ median = Stats.median(window)
611
+ if median > fastest * (1 - STEADY_GAIN)
612
+ calm += 1
613
+ return if calm == STEADY_WINDOWS
614
+ else
615
+ calm = 0
616
+ fastest = median
617
+ end
618
+ end
619
+ end
620
+
621
+ # STEADY seconds of +worker+'s samples' CPU time (a sample at least),
622
+ # or until +warm_until+, and when they ended.
623
+ def window(worker, name, n, warm_until)
624
+ window_until = monotonic + STEADY
625
+ samples = [] #: Array[Metric::value]
626
+ loop do
627
+ samples << worker.sample(name, n).fetch("timed").fetch("cpu")
628
+ now = monotonic
629
+ return [samples, now] if now > window_until || now > warm_until
517
630
  end
518
631
  end
519
632
 
520
633
  # Takes pairs until every ratio metric worth judging (measured on
521
634
  # both versions in the warm-up, and above its floors on either: one
522
635
  # that rises from next to nothing is judged too) is decided, or
523
- # MAX_PAIRS run out: each decided at the end of a round of STEP pairs.
524
- # A process benchmark's pairs take a process each, so it's decided not
525
- # slower (Result::CLEARED) after any pair too. That can only stop it
526
- # before it would have been found slower, never make it slower: it's
527
- # found slower only at the end of a round, as any benchmark is.
636
+ # MAX_PAIRS run out: each decided at the end of a round of STEP pairs,
637
+ # and each interval over the pairs. That's for a process benchmark, or
638
+ # too few workers for an interval over them: otherwise, see
639
+ # #judge_processes. A process benchmark's pairs take a process each,
640
+ # so it's decided not slower (Result::CLEARED) after any pair too.
641
+ # That can only stop it before it would have been found slower, never
642
+ # make it slower: it's found slower only at the end of a round, as any
643
+ # benchmark is.
528
644
  def judge(name, n, floors, result)
529
645
  judged = metrics.select do |spec|
530
646
  floors.flatten.all? { |sample| sample.fetch("timed").key?(spec.fetch("name")) } &&
531
647
  floors.transpose.any? { |side| above_floors?(side.map { |sample| sample.fetch("timed") }, spec) }
532
648
  end
649
+ return judge_processes(name, n, judged, result) if by_process?(name)
650
+
533
651
  ratios = judged.to_h { |spec| [spec.fetch("name"), []] }
534
652
  # @type var raw: Hash[String, Array[[Metric::value, Metric::value]]]
535
653
  raw = judged.to_h { |spec| [spec.fetch("name"), []] }
@@ -561,6 +679,78 @@ module Duel
561
679
  end
562
680
  end
563
681
 
682
+ # Whether a benchmark's timings are judged by process (see
683
+ # #judge_processes): unless it's a process benchmark (each of its
684
+ # samples is a fresh process already, so its pairs are independent),
685
+ # or there are too few workers of each version for an interval over
686
+ # them (Lane.by_process?).
687
+ def by_process?(name) = !benchmarks.fetch(name).fetch("process") && Lane.by_process?(@bases.size, @settings.confidence)
688
+
689
+ # Takes rounds of pairs, each a sample from every worker of each
690
+ # version (the baseline's in order, paired with the candidate's in
691
+ # turn: each round pairs each with another), until every ratio metric
692
+ # judged (see #judge) is decided, from the second round on, or
693
+ # #rounds run out. Each worker's median is a sample, and the interval
694
+ # is over workers (Stats.shift_interval): where a process's luck
695
+ # makes it faster or slower than another for its whole life (where
696
+ # things landed in its memory, or what a JIT compiled), the interval
697
+ # is as wide as that makes it. An interval over pairs would be as
698
+ # narrow as if every pair had processes of its own. Each worker's
699
+ # medians are kept, by metric, for across the board (Result#processes).
700
+ def judge_processes(name, n, judged, result)
701
+ k = @bases.size
702
+ # @type var values: Hash[String, Hash[String, Array[Array[Metric::value]]]]
703
+ values = judged.to_h { |spec| [spec.fetch("name"), {"baseline" => Array.new(k) { [] }, "candidate" => Array.new(k) { [] }}] }
704
+ pending = judged.dup
705
+ pairs = 0
706
+ rounds(k).times do |round|
707
+ break if pending.empty?
708
+
709
+ k.times do |i|
710
+ j = (i + round) % k
711
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?)
712
+ result.note(base, candidate)
713
+ values.each do |metric, sides|
714
+ sides.fetch("baseline").fetch(i) << base.fetch("timed").fetch(metric)
715
+ sides.fetch("candidate").fetch(j) << candidate.fetch("timed").fetch(metric)
716
+ end
717
+ pairs += 1
718
+ end
719
+ next if round + 1 < MIN_ROUNDS
720
+
721
+ pending.reject! { |spec| decided?(process_verdict(medians(values.fetch(spec.fetch("name"))), spec), true) }
722
+ end
723
+ judged.each do |spec|
724
+ metric = spec.fetch("name")
725
+ each = medians(values.fetch(metric))
726
+ bases, candidates = clamped(each)
727
+ result.timed[metric] = {"median" => Stats.shift(bases, candidates), "limit" => limit(spec), "across" => spec.fetch("across"),
728
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural"),
729
+ "interval" => Stats.shift_interval(bases, candidates, @settings.confidence),
730
+ "pairs" => pairs, "processes" => k, "verdict" => process_verdict(each, spec).to_s,
731
+ "baseline" => Stats.median(each.fetch("baseline")), "candidate" => Stats.median(each.fetch("candidate"))}
732
+ result.processes[metric] = {"lane" => @id, "baseline" => each.fetch("baseline"), "candidate" => each.fetch("candidate")}
733
+ end
734
+ end
735
+
736
+ # The most rounds of +k+ workers' pairs: as many as make
737
+ # PROCESS_PAIRS pairs, and MIN_ROUNDS at least.
738
+ def rounds(k) = [PROCESS_PAIRS / k, MIN_ROUNDS].max
739
+
740
+ # Each worker's median, by version, of +values+ (each worker's
741
+ # measures, by version).
742
+ def medians(values) = values.transform_values { |lists| lists.map { |list| Stats.median(list) } }
743
+
744
+ # The baseline's medians (of +each+) at least 1, as the ratio's
745
+ # denominator (so a ratio to nothing is one to 1, as a pair's is),
746
+ # and the candidate's.
747
+ def clamped(each) = [each.fetch("baseline").map { |value| [value, 1].max }, each.fetch("candidate")]
748
+
749
+ def process_verdict(each, spec)
750
+ bases, candidates = clamped(each)
751
+ Stats.decide(Stats.shift_interval(bases, candidates, @settings.confidence), limit(spec))
752
+ end
753
+
564
754
  # Whether both versions' CPU time in +floors+ is below its floor: then
565
755
  # it wouldn't be judged at all.
566
756
  def below_cpu_floor?(floors)
@@ -632,9 +822,14 @@ module Duel
632
822
  # out, and weighs on both alike.
633
823
  def pair(name, n, index, runs: RUNS, count: false)
634
824
  k = @bases.size
635
- base = @bases.fetch(index % k)
636
- candidate = @candidates.fetch((index + (index / k)) % k)
637
- order = index.even? ? [base, candidate] : [candidate, base]
825
+ sampled(name, n, @bases.fetch(index % k), @candidates.fetch((index + (index / k)) % k), base_first: index.even?, runs:, count:)
826
+ end
827
+
828
+ # A sample (of +n+ iterations) from +base+, a baseline worker, and
829
+ # from +candidate+, a candidate worker, the baseline's first if
830
+ # +base_first+ (see #pair).
831
+ def sampled(name, n, base, candidate, base_first:, runs: RUNS, count: false)
832
+ order = base_first ? [base, candidate] : [candidate, base]
638
833
  process = benchmarks.fetch(name).fetch("process")
639
834
  replies = if process
640
835
  # (Each run of one version's next to one of the other's.)
@@ -657,7 +852,9 @@ module Duel
657
852
  # Each counted measure's fewest, on each version, over COUNTS pairs of
658
853
  # samples, and what its metric said of those samples.
659
854
  def count(name, n, result)
660
- samples = Array.new(COUNTS) { |i| pair(name, n, i, runs: 1, count: true) }
855
+ return count_by_process(name, n, result) if Lane.by_process?(@bases.size, @settings.confidence)
856
+
857
+ samples = Array.new(counts) { |i| pair(name, n, i, runs: 1, count: true) }
661
858
  metrics.each do |spec|
662
859
  metric = spec.fetch("name")
663
860
  base, candidate = samples.transpose.map do |side|
@@ -672,6 +869,61 @@ module Duel
672
869
  end
673
870
  end
674
871
 
872
+ # Each counted measure, with workers enough for an interval over them
873
+ # (see #judge_processes): each worker's fewest in COUNT_ROUNDS rounds
874
+ # of a sample from every worker, and how much more the candidate's
875
+ # are than the baseline's, at least (the low end of an interval for
876
+ # it, Stats.difference_interval), which is what's judged (see
877
+ # Result#worse). Where every worker's is the same, as CRuby's counts
878
+ # of objects are, that's exactly how many more. Where they differ, as
879
+ # JRuby's bytes do where one JVM's JIT does away with an allocation
880
+ # and another's doesn't, it's as little as the workers can tell.
881
+ # Each version's own is the median of its workers'.
882
+ def count_by_process(name, n, result)
883
+ k = @bases.size
884
+ # @type var fewest: Hash[String, Hash[String, Hash[Integer, Metric::value]]]
885
+ fewest = Hash.new do |h, metric|
886
+ none = {} #: Hash[Integer, Metric::value]
887
+ h[metric] = {"baseline" => none, "candidate" => none.dup}
888
+ end
889
+ COUNT_ROUNDS.times do |round|
890
+ k.times do |i|
891
+ j = (i + round) % k
892
+ base, candidate = sampled(name, n, @bases.fetch(i), @candidates.fetch(j), base_first: i.even? == round.even?, runs: 1, count: true)
893
+ fewest!(fewest, "baseline", i, base)
894
+ fewest!(fewest, "candidate", j, candidate)
895
+ result.note(base, candidate)
896
+ end
897
+ end
898
+ metrics.each do |spec|
899
+ metric = spec.fetch("name")
900
+ next unless fewest.key?(metric)
901
+
902
+ found = fewest.fetch(metric)
903
+ bases = found.fetch("baseline").values
904
+ candidates = found.fetch("candidate").values
905
+ next if bases.empty? || candidates.empty?
906
+
907
+ result.counted[metric] = {"baseline" => Stats.median(bases), "candidate" => Stats.median(candidates),
908
+ "more" => Stats.difference_interval(bases, candidates, @settings.confidence)&.first,
909
+ "n" => n, "tolerance" => spec.fetch("tolerance"), "allowance" => spec.fetch("allowance"),
910
+ "unit" => spec.fetch("unit"), "plural" => spec.fetch("plural")}
911
+ end
912
+ end
913
+
914
+ # Keeps each count of +sample+ (from +side+'s +worker+th worker) in
915
+ # +fewest+, where it's that worker's fewest so far.
916
+ def fewest!(fewest, side, worker, sample)
917
+ sample.fetch("counted").each do |metric, value|
918
+ least = fewest[metric].fetch(side)
919
+ least[worker] = [least[worker], value].compact.min
920
+ end
921
+ end
922
+
923
+ # The samples a count takes: JRUBY_COUNTS where a version is on JRuby,
924
+ # or else COUNTS.
925
+ def counts = (@sides.each_value.any? { |side| side.engine == "jruby" }) ? JRUBY_COUNTS : COUNTS
926
+
675
927
  # Each version's least peak memory over MEMORY_SAMPLES fresh workers
676
928
  # (a long-lived one has room left over from earlier work), turn about.
677
929
  # Fresh means no trial run either: the allocator keeps what that freed,
@@ -743,6 +995,13 @@ module Duel
743
995
  # @api private
744
996
  attr_reader :profile
745
997
 
998
+ # Each worker's median of each measure judged by process (see
999
+ # Lane#judge_processes): {metric => {"lane" => its lane's number,
1000
+ # "baseline" => [each baseline worker's], "candidate" => [...]}},
1001
+ # for across the board.
1002
+ # @api private
1003
+ attr_reader :processes
1004
+
746
1005
  # The measurements again that were too close to call (see #confirm).
747
1006
  # @api private
748
1007
  attr_reader :undecided
@@ -758,6 +1017,7 @@ module Duel
758
1017
  @details = {}
759
1018
  @confirmations = []
760
1019
  @undecided = []
1020
+ @processes = {}
761
1021
  end
762
1022
 
763
1023
  # Keeps what metrics said of a pair of samples (Metric#details): each
@@ -773,11 +1033,12 @@ module Duel
773
1033
 
774
1034
  # The metrics on which the candidate did worse: a timed one slower, or
775
1035
  # a counted one up by more than its tolerance per iteration and its
776
- # allowance per sample.
1036
+ # allowance per sample (counted by process, up by that much at least:
1037
+ # see Lane#count_by_process).
777
1038
  # @api private
778
1039
  def worse
779
1040
  more = @counted.select do |_, c|
780
- !Metric.within?(c.fetch("candidate") - c.fetch("baseline"), c.fetch("n"), tolerance: c.fetch("tolerance"),
1041
+ !Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
781
1042
  allowance: c.fetch("allowance"))
782
1043
  end
783
1044
  slower + more.keys + (more_memory? ? ["peak_memory"] : [])
@@ -871,6 +1132,14 @@ module Duel
871
1132
  # benchmark", "2 benchmarks").
872
1133
  def self.count(number, noun, plural = "#{noun}s") = "#{number} #{(number == 1) ? noun : plural}"
873
1134
 
1135
+ # A count, an iteration's (a Float, often): to four figures, or one of
1136
+ # NUMBER or more (where four figures would be 1e+04) as a whole number
1137
+ # (25736 bytes, not 2.574e+04).
1138
+ def self.number(value) = (value.abs >= NUMBER) ? value.round.to_s : format("%.4g", value)
1139
+
1140
+ # The least count #number shows as a whole number.
1141
+ NUMBER = 9_999.5
1142
+
874
1143
  # Across the board (see Across): {metric => verdict}, "confirmed" in
875
1144
  # those slower in a second measurement too. Set by #run.
876
1145
  attr_reader :across
@@ -957,12 +1226,13 @@ module Duel
957
1226
  # What #run does, with its versions prepared in its directory.
958
1227
  def compare_all(started)
959
1228
  @sides = prepare
1229
+ fit
960
1230
  names, results = with_lanes do |lanes|
961
1231
  found = lanes.first.benchmarks.keys.grep(@settings.filter)
962
1232
  @benchmarks = lanes.first.benchmarks
963
1233
  @out.puts "Comparing #{Runner.count(found.size, "benchmark")}, #{@settings.jobs} at a time: baseline #{@base_root} " \
964
1234
  "against candidate #{File.expand_path(@settings.candidate_dir || @config.root)} (slower: a " \
965
- "#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit})"
1235
+ "#{(@settings.confidence * 100).round(1)}% interval above #{@settings.limit}, #{interval_over})"
966
1236
  rubies = @rubies = lanes.first.rubies
967
1237
  rubies.each { |side, ruby| @out.puts " #{side} on #{ruby}" } if rubies.values.uniq.size > 1
968
1238
  @out.puts " with plugins #{plugins.join(", ")}" unless plugins.empty?
@@ -975,6 +1245,23 @@ module Duel
975
1245
  results
976
1246
  end
977
1247
 
1248
+ # What a benchmark's interval is over (see Lane#judge_processes): the
1249
+ # workers, or the pairs, where they're too few (and says how many it
1250
+ # takes, of the first PROCESSES_CHECKED).
1251
+ # mutant:disable -- fewer than 2 of each never take an interval over them, and the last checked is only a bound
1252
+ def interval_over
1253
+ processes = @settings.processes
1254
+ workers = Runner.count(processes, "worker process", "worker processes")
1255
+ return "over #{workers} of each version" if Lane.by_process?(processes, @settings.confidence)
1256
+
1257
+ least = (1..PROCESSES_CHECKED).find { |k| Lane.by_process?(k, @settings.confidence) }
1258
+ "over pairs: #{workers} of each version are too few for an interval over them" + (least ? ", which takes #{least}" : "")
1259
+ end
1260
+
1261
+ # The most workers of each version #interval_over looks at, for how
1262
+ # many it takes (more would take long to check, and longer to run).
1263
+ PROCESSES_CHECKED = 20
1264
+
978
1265
  # Each version, prepared once for the run, before any of its workers
979
1266
  # start: its Ruby found (Config#ruby, or --ruby), and its
980
1267
  # Config#prepare(:version) blocks run, each given a Resource (named for
@@ -984,6 +1271,7 @@ module Duel
984
1271
  Side.unprepared(@config, @settings, @base_root).transform_values do |side|
985
1272
  side.ruby = Rubies.find(choice, side.root, side.name)
986
1273
  side.env = Rubies.environment(side.ruby)
1274
+ side.engine = Rubies.engine(side.ruby)
987
1275
  @config.prepares.fetch(:version).each_with_index do |block, i|
988
1276
  resource = Resource.new(File.join(@dir, "#{side.label}-#{i + 1}"), side.label, side.root, timeout: @settings.timeout,
989
1277
  cache: File.join(@config.state, "cache"), ruby: side.ruby, env: side.env)
@@ -994,6 +1282,47 @@ module Duel
994
1282
  end
995
1283
  end
996
1284
 
1285
+ # The memory Runner#fit leaves for everything but the workers and
1286
+ # their resources: the runner, the system, and other processes.
1287
+ RESERVE = 1 << 30
1288
+
1289
+ # How much more than what it took to start a pair of workers takes
1290
+ # while it measures (its heap grows, and so do its resources'), by
1291
+ # Runner#fit's reckoning.
1292
+ HEADROOM = 1.25
1293
+
1294
+ # Lowers Settings#jobs, and then Settings#processes, as far as
1295
+ # Settings#fit lets it, where their workers (and each one's
1296
+ # resources, a database server, say) wouldn't fit in the memory
1297
+ # available: more than that and the machine swaps, or a CI runner
1298
+ # stops answering and its job is lost. What one worker of each
1299
+ # version takes is found by starting them (a lane of them), and every
1300
+ # pair is taken to need that, HEADROOM over.
1301
+ def fit
1302
+ return if @settings.fit.empty?
1303
+
1304
+ available = Memory.available or return
1305
+ taken = Lane.open(@config, @settings, @sides, processes: 1) { available - (Memory.available || available) }
1306
+ return unless taken.positive?
1307
+
1308
+ pair = (taken * HEADROOM).ceil
1309
+ pairs = [(available - RESERVE) / pair, 1].max
1310
+ processes = @settings.fit.include?(:processes) ? [@settings.processes, pairs].min : @settings.processes
1311
+ # (More jobs than the settings' are never taken: only fewer are.)
1312
+ jobs = @settings.fit.include?(:jobs) ? [pairs / processes, 1].max : @settings.jobs
1313
+ fewer = [("#{Runner.count(jobs, "benchmark")} at a time (not #{@settings.jobs})" if jobs < @settings.jobs),
1314
+ ("#{Runner.count(processes, "worker")} of each version (not #{@settings.processes})" if processes < @settings.processes)].compact
1315
+ return if fewer.empty?
1316
+
1317
+ @out.puts "A worker of each version takes #{gigabytes(taken)} GB, with its resources, of #{gigabytes(available)} GB available, " \
1318
+ "so to fit: #{fewer.join(", and ")}"
1319
+ @settings.jobs = jobs
1320
+ @settings.processes = processes
1321
+ end
1322
+
1323
+ # +bytes+ in gigabytes, to a tenth.
1324
+ def gigabytes(bytes) = format("%.1f", bytes.fdiv(1 << 30))
1325
+
997
1326
  def with_lanes
998
1327
  # @type var lanes: Array[Lane]
999
1328
  lanes = []
@@ -1007,7 +1336,7 @@ module Duel
1007
1336
  # every benchmark, in fresh processes, and are confirmed if they're
1008
1337
  # still slower.
1009
1338
  def judge_across(results, names)
1010
- verdicts = Across.new(results, confidence: @settings.confidence).verdicts
1339
+ verdicts = across_of(results).verdicts
1011
1340
  slower = verdicts.select { |_, v| v.fetch("verdict") == "slower" }.keys
1012
1341
  return verdicts if slower.empty?
1013
1342
 
@@ -1020,11 +1349,16 @@ module Duel
1020
1349
  end
1021
1350
 
1022
1351
  @out.puts "\nMeasuring every benchmark again, in fresh processes: slower across the board in #{slower.join(", ")}?"
1023
- again = Across.new(with_lanes { |lanes| measure_all(lanes, names) }, confidence: @settings.confidence).verdicts
1352
+ again = across_of(with_lanes { |lanes| measure_all(lanes, names) }).verdicts
1024
1353
  slower.each { |metric| verdicts.fetch(metric)["confirmed"] = again.dig(metric, "verdict") == "slower" }
1025
1354
  verdicts
1026
1355
  end
1027
1356
 
1357
+ # Across the board, of +results+, judged over workers where the run
1358
+ # judged benchmarks over them (see Lane.by_process?).
1359
+ # mutant:disable -- whether it's over workers is only how fast: results judged over pairs have no workers' medians to find
1360
+ def across_of(results) = Across.new(results, confidence: @settings.confidence, by_process: Lane.by_process?(@settings.processes, @settings.confidence))
1361
+
1028
1362
  def open_lane = Lane.new(@config, @settings, @sides)
1029
1363
 
1030
1364
  def measure_all(lanes, names)
@@ -1145,7 +1479,7 @@ module Duel
1145
1479
  candidate = c.fetch("candidate").fdiv(n)
1146
1480
  # (No unit its name ends in: "retained_objects 2", not "2 objects".)
1147
1481
  named = c.fetch("plural")&.then { |plural| metric.end_with?(plural.tr(" ", "_")) }
1148
- [format("%s %.4g%s", metric, base, (format("→%.4g", candidate) unless base == candidate)),
1482
+ [format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
1149
1483
  (unit(candidate, c) unless named)].compact.join(" ")
1150
1484
  end
1151
1485
  memory = result.memory&.then { |m| ["peak_memory #{mb(m.fetch("baseline"))}→#{mb(m.fetch("candidate"))}"] }
@@ -1189,7 +1523,7 @@ module Duel
1189
1523
  def amount(value, n, measured)
1190
1524
  each = value.fdiv([n, 1].max)
1191
1525
  ns = Metric::TIME_UNITS[measured.fetch("unit")]
1192
- return [format("%.4g", each), unit(each, measured)].compact.join(" ") unless ns
1526
+ return [Runner.number(each), unit(each, measured)].compact.join(" ") unless ns
1193
1527
 
1194
1528
  time = each * ns
1195
1529
  shown, size = Metric::TIME_UNITS.reverse_each.find { |_, unit_ns| time >= unit_ns } || ["ns", 1]