duel 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +51 -27
  3. data/CHANGELOG.md +247 -1
  4. data/README.md +30 -14
  5. data/docs/benchmarks.md +28 -24
  6. data/docs/ci.md +328 -362
  7. data/docs/configuration.md +40 -37
  8. data/docs/how-it-works.md +23 -10
  9. data/docs/json.md +21 -25
  10. data/docs/plugins.md +40 -22
  11. data/docs/troubleshooting.md +3 -2
  12. data/lib/duel/across.rb +18 -18
  13. data/lib/duel/badge.rb +174 -0
  14. data/lib/duel/baseline.rb +38 -15
  15. data/lib/duel/cli.rb +175 -107
  16. data/lib/duel/comment.rb +80 -71
  17. data/lib/duel/config.rb +145 -63
  18. data/lib/duel/context.rb +29 -24
  19. data/lib/duel/forge.rb +87 -0
  20. data/lib/duel/init.rb +44 -87
  21. data/lib/duel/json_report.rb +34 -25
  22. data/lib/duel/lane.rb +107 -61
  23. data/lib/duel/lock.rb +1 -1
  24. data/lib/duel/mark_passed.rb +132 -0
  25. data/lib/duel/metric.rb +28 -27
  26. data/lib/duel/native.rb +7 -5
  27. data/lib/duel/platform.rb +4 -0
  28. data/lib/duel/process_sample.rb +9 -6
  29. data/lib/duel/rake_task.rb +4 -5
  30. data/lib/duel/resource.rb +10 -11
  31. data/lib/duel/result.rb +22 -14
  32. data/lib/duel/rubies.rb +6 -2
  33. data/lib/duel/runner.rb +27 -15
  34. data/lib/duel/spawn.rb +29 -4
  35. data/lib/duel/stats.rb +14 -12
  36. data/lib/duel/summary.rb +31 -29
  37. data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
  38. data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
  39. data/lib/duel/templates/duel.rb.erb +7 -7
  40. data/lib/duel/templates/perf-caller.yml.erb +22 -5
  41. data/lib/duel/templates/perf-comment.yml.erb +11 -11
  42. data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
  43. data/lib/duel/templates/perf-push.yml.erb +50 -0
  44. data/lib/duel/templates/perf.yml.erb +113 -335
  45. data/lib/duel/testing.rb +20 -25
  46. data/lib/duel/version.rb +1 -1
  47. data/lib/duel/worker.rb +16 -9
  48. data/lib/duel/worker_process.rb +6 -6
  49. data/lib/duel.rb +88 -33
  50. data/sig/duel/across.rbs +4 -0
  51. data/sig/duel/badge.rbs +51 -0
  52. data/sig/duel/baseline.rbs +7 -2
  53. data/sig/duel/cli.rbs +31 -20
  54. data/sig/duel/comment.rbs +14 -15
  55. data/sig/duel/config.rbs +71 -29
  56. data/sig/duel/context.rbs +4 -4
  57. data/sig/duel/forge.rbs +42 -0
  58. data/sig/duel/init.rbs +8 -26
  59. data/sig/duel/json_report.rbs +7 -6
  60. data/sig/duel/lane.rbs +29 -6
  61. data/sig/duel/mark_passed.rbs +39 -0
  62. data/sig/duel/metric.rbs +10 -10
  63. data/sig/duel/native.rbs +3 -0
  64. data/sig/duel/platform.rbs +1 -0
  65. data/sig/duel/process_sample.rbs +3 -0
  66. data/sig/duel/resource.rbs +3 -3
  67. data/sig/duel/result.rbs +12 -5
  68. data/sig/duel/rubies.rbs +3 -0
  69. data/sig/duel/runner.rbs +23 -16
  70. data/sig/duel/spawn.rbs +7 -0
  71. data/sig/duel/stats.rbs +4 -1
  72. data/sig/duel/summary.rbs +8 -0
  73. data/sig/duel/testing.rbs +2 -2
  74. data/sig/duel/worker.rbs +6 -5
  75. data/sig/duel/worker_process.rbs +3 -3
  76. data/sig/duel.rbs +50 -31
  77. metadata +9 -1
data/lib/duel/metric.rb CHANGED
@@ -9,7 +9,7 @@ module Duel
9
9
  # judged, in +measures+: {"name" => settings}, each of these, any left
10
10
  # out as by default:
11
11
  #
12
- # - +kind+: :ratio (noisy: judged on paired ratios with a confidence
12
+ # - +kind+: :timed (noisy: judged on paired ratios with a confidence
13
13
  # interval) or :counted (exact: any increase beyond what's allowed is
14
14
  # a regression, counted in separate samples).
15
15
  # - +unit+: what it's in, as reports show it: a time unit ("ns", "µs",
@@ -17,24 +17,24 @@ module Duel
17
17
  # shown after the number, or nil for none. A word comes with its
18
18
  # plural, [singular, plural] (%w[row rows]: 1 row, 2 rows), and a symbol ("B",
19
19
  # "KB") is one String, the same for any number.
20
- # - +floor+: for a ratio metric, the least a sample must measure (in
20
+ # - +floor+: for a timed metric, the least a sample must measure (in
21
21
  # its own unit) on either version for it to be judged. Below it, it's
22
22
  # mostly noise.
23
- # - +cpu_floor+: for a ratio metric measured in time (see +unit_ns+),
23
+ # - +cpu_floor+: for a timed metric measured in time (see +unit_ns+),
24
24
  # the least share of the sample's CPU time it must be on either
25
25
  # version to be judged (0.05: 5%): a floor that scales with the work.
26
- # - +limit+: for a ratio metric, its own ratio above which it's slower
26
+ # - +limit+: for a timed metric, its own ratio above which it's slower
27
27
  # (nil: the run's, 1.02 by default).
28
- # - +unit_ns+: for a ratio metric, nanoseconds per unit, for comparing
28
+ # - +unit_ns+: for a timed metric, nanoseconds per unit, for comparing
29
29
  # it with CPU time when sizing samples.
30
- # - +sizes_samples+: for a ratio metric, whether samples are made long
30
+ # - +size_samples+: for a timed metric, whether samples are made long
31
31
  # enough for it to settle, as they are for CPU time (false for one
32
32
  # that's noisy whatever the length, or not in step with the work).
33
- # - +across+: for a ratio metric, whether it's judged across the board
33
+ # - +across+: for a timed metric, whether it's judged across the board
34
34
  # too: false for one too noisy for so small a margin.
35
- # - +tolerance+: for a counted metric, the increase per iteration that
36
- # still isn't one (0: one more is).
37
- # - +allowance+: for a counted metric, the increase per sample that
35
+ # - +per_iteration+: for a counted metric, the increase per iteration
36
+ # that still isn't one (0: one more is).
37
+ # - +per_sample+: for a counted metric, the increase per sample that
38
38
  # still isn't one, whatever its length (noise that doesn't grow with
39
39
  # the iterations).
40
40
  #
@@ -65,7 +65,7 @@ module Duel
65
65
  # Every setting there is (see Metric), as #settings resolves them, and
66
66
  # running totals' calibrate and Config#measure's skip.
67
67
  # @api private
68
- SETTINGS = %i[kind unit floor cpu_floor limit unit_ns sizes_samples across tolerance allowance calibrate skip].freeze
68
+ SETTINGS = %i[kind unit floor cpu_floor limit unit_ns size_samples across per_iteration per_sample calibrate skip].freeze
69
69
 
70
70
  # A unit that's a word (or words), which needs its plural given: one
71
71
  # that isn't (a symbol: "B", "KB", "op/s") is the same for any number.
@@ -111,8 +111,8 @@ module Duel
111
111
  def check(name, settings)
112
112
  unknown = settings.keys - SETTINGS
113
113
  raise Error, "#{name}: no setting #{unknown.join(", ")} (settings: #{SETTINGS.join(", ")})" unless unknown.empty?
114
- unless %i[ratio counted].include?(settings.fetch(:kind, :counted))
115
- raise Error, "#{name}: kind is :ratio or :counted, not #{settings.fetch(:kind).inspect}"
114
+ unless %i[timed counted].include?(settings.fetch(:kind, :counted))
115
+ raise Error, "#{name}: kind is :timed or :counted, not #{settings.fetch(:kind).inspect}"
116
116
  end
117
117
  unit = settings.fetch(:unit, nil)
118
118
  if unit.instance_of?(String) && unit.match?(WORD) && !TIME_UNITS.key?(unit)
@@ -124,10 +124,10 @@ module Duel
124
124
  end
125
125
 
126
126
  # Whether +more+ (what a count grew by, over +n+ iterations) is
127
- # still within what's allowed: +tolerance+ an iteration and
128
- # +allowance+ a sample. Duel judges counts so, and so can a metric, to
127
+ # still within what's allowed: +per_iteration+ an iteration and
128
+ # +per_sample+ a sample. Duel judges counts so, and so can a metric, to
129
129
  # tell whether it's worth explaining one (see #within?).
130
- def within?(more, n, tolerance:, allowance:) = more <= allowance + (tolerance * n)
130
+ def within?(more, n, per_iteration:, per_sample:) = more <= per_sample + (per_iteration * n)
131
131
 
132
132
  # Each measure's settings, by name, every one of them filled in (as
133
133
  # declared, and the rest as by default): what a plugin's tests can
@@ -148,8 +148,8 @@ module Duel
148
148
  {kind: given.fetch(:kind, :counted), unit: unit, plural: plural || unit, floor: given.fetch(:floor, 0),
149
149
  cpu_floor: given.fetch(:cpu_floor, 0),
150
150
  limit: given.fetch(:limit, nil), unit_ns: given.fetch(:unit_ns) { TIME_UNITS.fetch(unit, 1) },
151
- sizes_samples: given.fetch(:sizes_samples, true), across: given.fetch(:across, true),
152
- tolerance: given.fetch(:tolerance, 0), allowance: given.fetch(:allowance, 0)}
151
+ size_samples: given.fetch(:size_samples, true), across: given.fetch(:across, true),
152
+ per_iteration: given.fetch(:per_iteration, 0), per_sample: given.fetch(:per_sample, 0)}
153
153
  end
154
154
 
155
155
  # Its measures as the runner learns them, with +overrides+ (see
@@ -183,15 +183,16 @@ module Duel
183
183
  # @api private
184
184
  def first_measure = @first_measure ||= settings.keys.first
185
185
 
186
- # The setting +key+ (:allowance, say) of +measure+: by default, of the
186
+ # The setting +key+ (:per_sample, say) of +measure+: by default, of the
187
187
  # one it measures (one that measures several names it).
188
188
  def setting(key, measure = only_measure) = settings.fetch(measure).fetch(key)
189
189
 
190
190
  # Whether +more+ (what +measure+'s count grew by, over +n+
191
- # iterations) is still within its tolerance and allowance, as Duel
191
+ # iterations) is still within what its per_iteration and per_sample
192
+ # allow, as Duel
192
193
  # judges it (see Metric.within?): whether it's worth explaining.
193
194
  def within?(more, n, measure = only_measure)
194
- Metric.within?(more, n, tolerance: setting(:tolerance, measure), allowance: setting(:allowance, measure))
195
+ Metric.within?(more, n, per_iteration: setting(:per_iteration, measure), per_sample: setting(:per_sample, measure))
195
196
  end
196
197
 
197
198
  # Sets up what it reads (a subscription, say), once in each worker,
@@ -249,7 +250,7 @@ module Duel
249
250
  # twice in a row, CALIBRATION times, and the least each counted total
250
251
  # grew by is taken from every sample's: unless a measure says
251
252
  # calibrate: false (its reading costs nothing, or a varying amount its
252
- # block takes away itself). A ratio one is left alone: its reading's
253
+ # block takes away itself). A timed one is left alone: its reading's
253
254
  # cost isn't steady enough to take away.
254
255
  #
255
256
  # Config#metric makes one, and returns it (to take .open, .close,
@@ -312,7 +313,7 @@ module Duel
312
313
  # around_iteration hooks, and what they add to it is taken away
313
314
  # instead (see Metric). Returns the metric.
314
315
  #
315
- # c.metric("postgres_queries") { queries }.costly
316
+ # c.metric("postgresql_queries") { queries }.costly
316
317
  def costly
317
318
  @costly = true
318
319
  self
@@ -513,7 +514,7 @@ module Duel
513
514
  # processes on the machine don't inflate. Always measured last, inside
514
515
  # every other metric.
515
516
  class CPU < Metric
516
- def self.measures = {"cpu" => {kind: :ratio, unit: "ns", floor: 300_000}}
517
+ def self.measures = {"cpu" => {kind: :timed, unit: "ns", floor: 300_000}}
517
518
 
518
519
  def start_sample(sample)
519
520
  @clock = sample.benchmark.cpu
@@ -534,7 +535,7 @@ module Duel
534
535
  # whole process's, if it can. It misses what instructions don't show,
535
536
  # like cache misses: CPU time is still judged.
536
537
  class Instructions < Metric
537
- def self.measures = {"instructions" => {kind: :ratio, floor: 100_000, sizes_samples: false, limit: 1.01}}
538
+ def self.measures = {"instructions" => {kind: :timed, floor: 100_000, size_samples: false, limit: 1.01}}
538
539
 
539
540
  def self.available? = !counter.nil?
540
541
 
@@ -566,7 +567,7 @@ module Duel
566
567
  # sample here and there some 31 bytes an iteration apart: so more by
567
568
  # 32 bytes an iteration, about a small object's, isn't more yet.
568
569
  class AllocatedBytes < Metric
569
- def self.measures = {"allocated_bytes" => {tolerance: 32, unit: %w[byte bytes]}}
570
+ def self.measures = {"allocated_bytes" => {per_iteration: 32, unit: %w[byte bytes]}}
570
571
 
571
572
  # mutant:disable -- mutant runs on CRuby, where this is always false
572
573
  def self.available? = Platform::JRUBY
@@ -583,7 +584,7 @@ module Duel
583
584
  # a few from run to run (ids of random lengths, say), so it's one
584
585
  # more per two iterations, not one more, that's a regression.
585
586
  class Allocations < Metric
586
- def self.measures = {"allocations" => {tolerance: 0.5}}
587
+ def self.measures = {"allocations" => {per_iteration: 0.5}}
587
588
 
588
589
  # mutant:disable -- mutant runs on CRuby, where this is always true
589
590
  def self.available? = Platform::CRUBY
data/lib/duel/native.rb CHANGED
@@ -167,11 +167,14 @@ module Duel
167
167
  STATUS = 64 # sizeof(MEMORYSTATUSEX)
168
168
  AVAILABLE = 16 # offsetof(ullAvailPhys)
169
169
 
170
+ # K32GetProcessMemoryInfo, a process's memory (raising
171
+ # Fiddle::DLError where there's no such function).
172
+ def self.process_memory_info = Native.function("K32GetProcessMemoryInfo", %i[VOIDP VOIDP INT], :INT, library: KERNEL32)
173
+
170
174
  def self.open(platform: Platform::OS, memory_info: nil)
171
175
  return unless platform.match?(Platform::WINDOWS_OS)
172
176
 
173
- memory_info ||= Native.function("K32GetProcessMemoryInfo", %i[VOIDP VOIDP INT], :INT, library: KERNEL32)
174
- windows = new(memory_info)
177
+ windows = new(memory_info || process_memory_info)
175
178
  windows if windows.resident
176
179
  rescue Fiddle::DLError # no such function
177
180
  nil
@@ -262,8 +265,7 @@ module Duel
262
265
  def initialize(handle, cycles: nil, memory_info: nil, close: nil)
263
266
  @handle = handle
264
267
  @cycles = cycles || Cycles.new(:process, handle: handle)
265
- @memory = Windows.new(memory_info || Native.function("K32GetProcessMemoryInfo", %i[VOIDP VOIDP INT], :INT, library: KERNEL32),
266
- handle: handle)
268
+ @memory = Windows.new(memory_info || Windows.process_memory_info, handle: handle)
267
269
  @close = close || Native.function("CloseHandle", %i[INTPTR_T], :INT, library: KERNEL32)
268
270
  end
269
271
 
@@ -329,7 +331,7 @@ module Duel
329
331
  # mutant:disable -- kept only so as not to open it again: it's the
330
332
  # same reader however often it's opened
331
333
  def self.opened_reader
332
- return @reader if defined?(@reader)
334
+ return @reader if instance_variable_defined?(:@reader)
333
335
 
334
336
  @reader = (MacOS.memory || Windows.open if FIDDLE)
335
337
  end
data/lib/duel/platform.rb CHANGED
@@ -19,6 +19,10 @@ module Duel
19
19
  WINDOWS_OS = /mingw|mswin/
20
20
  WINDOWS = OS.match?(WINDOWS_OS)
21
21
 
22
+ # What a program's file ends with: on Windows, .exe (see
23
+ # Duel.find_program).
24
+ EXE = RbConfig::CONFIG["EXEEXT"]
25
+
22
26
  CRUBY = RUBY_ENGINE == "ruby"
23
27
  JRUBY = RUBY_ENGINE == "jruby"
24
28
 
@@ -41,9 +41,14 @@ module Duel
41
41
  def self.fastest(samples)
42
42
  peaks = samples.map { |sample| sample.fetch("peak") }
43
43
  cpu = samples.map { |sample| sample.fetch("timed").fetch("cpu") }.min #: Integer
44
- {"timed" => {"cpu" => cpu}, "counted" => {}, "heap" => 0, "peak" => (peaks.min unless peaks.include?(nil))}
44
+ sampled(cpu, (peaks.min unless peaks.include?(nil)))
45
45
  end
46
46
 
47
+ # A process benchmark's sample, as a worker's is: its CPU time
48
+ # (nanoseconds) and its peak memory (bytes, or nil), counting nothing
49
+ # and leaving no heap.
50
+ def self.sampled(cpu, peak) = {"timed" => {"cpu" => cpu}, "counted" => {}, "heap" => 0, "peak" => peak}
51
+
47
52
  # Runs +command+ (as Process.spawn takes it) once, and returns its
48
53
  # sample ({"timed" => {"cpu" => nanoseconds}, "peak" => bytes or nil},
49
54
  # as a worker's has it). One that fails, or takes more than the
@@ -71,11 +76,9 @@ module Duel
71
76
 
72
77
  ended = JSON.parse(reply)
73
78
  failed = ended.fetch("failed")
74
- if failed
75
- tail = Spawn.tail(@log)
76
- raise Failed, "#{@label}: #{command.join(" ")} #{failed}#{"\n#{tail}" unless tail.empty?}"
77
- end
78
- {"timed" => {"cpu" => ended.fetch("cpu")}, "counted" => {}, "heap" => 0, "peak" => ended.fetch("peak")}
79
+ raise Failed, Spawn.with_tail("#{@label}: #{command.join(" ")} #{failed}", @log) if failed
80
+
81
+ ProcessSample.sampled(ended.fetch("cpu"), ended.fetch("peak"))
79
82
  end
80
83
 
81
84
  # Ends the Ruby process of its own, if it started: closing its input
@@ -3,7 +3,8 @@
3
3
  require "rake"
4
4
 
5
5
  module Duel
6
- # A rake task running `duel` with +args+:
6
+ # A rake task running `duel` with +args+, which exits as `duel` does:
7
+ # with its status, where it isn't 0 (see CLI).
7
8
  #
8
9
  # require "duel/rake_task"
9
10
  # Duel::RakeTask.new(:perf)
@@ -24,10 +25,8 @@ module Duel
24
25
  desc "Fail if this tree is slower than its baseline at any benchmark (duel)"
25
26
  task(name) do
26
27
  RakeTask.load_duel
27
- case CLI.new(args).run
28
- when 1 then abort "duel: slower than the baseline"
29
- when 2 then abort "duel: couldn't compare"
30
- end
28
+ status = CLI.new(args).run
29
+ exit(status) unless status.zero?
31
30
  end
32
31
  end
33
32
  end
data/lib/duel/resource.rb CHANGED
@@ -134,8 +134,9 @@ module Duel
134
134
 
135
135
  # A port of 127.0.0.1's that's free (one the system chose, let go of
136
136
  # again), for a server of its own to listen on, rather than on a Unix
137
- # socket in its directory (see #path).
138
- def port
137
+ # socket in its directory (see #path): another each call, so keep the
138
+ # one a server's given.
139
+ def free_port
139
140
  server = Control.tcp_server
140
141
  begin
141
142
  server.addr.fetch(1)
@@ -149,7 +150,7 @@ module Duel
149
150
  # copy): the first that asks for +key+ builds it, in a directory the
150
151
  # block is given, and the rest get it built. A key names what's built
151
152
  # (put a version in it), as a file name: tmp/duel/cache/+key+, which
152
- # `duel --clean` removes. A build that fails leaves nothing behind.
153
+ # `duel clean` removes. A build that fails leaves nothing behind.
153
154
  # A key that ends with a hash (and a version before it, if you like:
154
155
  # mysql-0.1.2-3f9c2a1b) replaces the builds of its name when the run
155
156
  # ends (see .evict): the one a run used is the one that's kept.
@@ -188,7 +189,7 @@ module Duel
188
189
  code, signal = start(command, log, options).wait
189
190
  return if code == 0
190
191
 
191
- failed("#{File.basename(log, ".log")} #{Spawn.why(code, signal)}", log: log)
192
+ fail!("#{File.basename(log, ".log")} #{Spawn.why(code, signal)}", log: log)
192
193
  end
193
194
 
194
195
  # Waits until the block is true: fails if a process spawned dies
@@ -201,9 +202,9 @@ module Duel
201
202
  # (It may have got ready, and ended, since the last look.)
202
203
  break if yield
203
204
 
204
- failed("#{ended.program} exited before it was ready", log: ended.log)
205
+ fail!("#{ended.program} exited before it was ready", log: ended.log)
205
206
  end
206
- failed("not ready within #{@timeout}s", log: @processes.last&.log) if monotonic > deadline
207
+ fail!("not ready within #{@timeout}s", log: @processes.last&.log) if monotonic > deadline
207
208
  sleep POLL
208
209
  end
209
210
  end
@@ -213,9 +214,8 @@ module Duel
213
214
 
214
215
  # Fails, saying it's this resource that did, with the end of +log+ (a
215
216
  # path) to say why.
216
- def failed(message, log: nil)
217
- tail = log ? Spawn.tail(log) : ""
218
- raise Failed, "#{@name}'s resource: #{message}#{"\n#{tail}" unless tail.empty?}"
217
+ def fail!(message, log: nil)
218
+ raise Failed, Spawn.with_tail("#{@name}'s resource: #{message}", log)
219
219
  end
220
220
 
221
221
  # Stops it: the at_stop blocks, last first, then each process.
@@ -263,8 +263,7 @@ module Duel
263
263
  # can't interrupt a wait on JRuby.)
264
264
  def start(command, log, options)
265
265
  # (Its own environment, if it's given one, over its version's.)
266
- none = {} #: Hash[String, String?]
267
- env, *program = command.first.is_a?(Hash) ? command : [none, *command]
266
+ env, program = Spawn.environment_and(command)
268
267
  @starter.logged(@env.merge(env), program, log: log, mode: "a", close_others: true, **options).detach
269
268
  end
270
269
 
data/lib/duel/result.rb CHANGED
@@ -45,11 +45,16 @@ module Duel
45
45
 
46
46
  # The measurements again that were too close to call (see #confirm).
47
47
  # @api private
48
- attr_reader :undecided
48
+ attr_reader :close_calls
49
49
 
50
50
  # @api private
51
51
  attr_writer :n, :seconds, :skipped, :error, :memory, :profile
52
52
 
53
+ # A counted measure's (+c+'s) count an iteration (of its "n", one at
54
+ # least): [baseline's, candidate's].
55
+ # @api private
56
+ def self.per_iteration(c) = [c.fetch("baseline").fdiv(c.fetch("n")), c.fetch("candidate").fdiv(c.fetch("n"))]
57
+
53
58
  # @api private
54
59
  def initialize(name)
55
60
  @name = name
@@ -57,7 +62,7 @@ module Duel
57
62
  @counted = {}
58
63
  @details = {}
59
64
  @confirmations = []
60
- @undecided = []
65
+ @close_calls = []
61
66
  @processes = {}
62
67
  end
63
68
 
@@ -73,14 +78,14 @@ module Duel
73
78
  end
74
79
 
75
80
  # The metrics on which the candidate did worse: a timed one slower, or
76
- # a counted one up by more than its tolerance per iteration and its
77
- # allowance per sample (counted by process, up by that much at least:
81
+ # a counted one up by more than its per_iteration an iteration and its
82
+ # per_sample a sample (counted by process, up by that much at least:
78
83
  # see Lane#count_by_process).
79
84
  # @api private
80
85
  def worse
81
86
  more = @counted.select do |_, c|
82
- !Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
83
- allowance: c.fetch("allowance"))
87
+ !Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), per_iteration: c.fetch("per_iteration"),
88
+ per_sample: c.fetch("per_sample"))
84
89
  end
85
90
  slower + more.keys + (more_memory? ? ["peak_memory"] : [])
86
91
  end
@@ -108,16 +113,16 @@ module Duel
108
113
  # Keeps +again+, a measurement of it again from scratch (see
109
114
  # Runner#confirm), as a confirmation, or sets it aside where it's too
110
115
  # close to call, up to Runner::CONFIRM times: every timing is still
111
- # undecided there (none slower, none the same or faster), and each
116
+ # unsettled there (none slower, none the same or faster), and each
112
117
  # measure it regressed on so far is one of them. One that can't tell
113
118
  # neither confirms it nor clears it, so it's measured again instead.
114
119
  # Returns whether +again+ was set aside.
115
120
  # @api private
116
121
  def confirm(again)
117
122
  timed = again.timed
118
- close = timed.all? { |_, t| t.fetch("verdict") == "undecided" } && regressed.all? { |metric| timed.key?(metric) } &&
119
- @undecided.size < Runner::CONFIRM
120
- (close ? @undecided : @confirmations) << again
123
+ close = timed.all? { |_, t| t.fetch("verdict") == "unsettled" } && regressed.all? { |metric| timed.key?(metric) } &&
124
+ @close_calls.size < Runner::CONFIRM
125
+ (close ? @close_calls : @confirmations) << again
121
126
  close
122
127
  end
123
128
 
@@ -137,8 +142,8 @@ module Duel
137
142
  def regressed
138
143
  return [] if @error || @skipped
139
144
 
140
- agreed = measurements.map(&:worse).reduce(:&)
141
- return agreed unless agreed.empty? && measurements.all?(&:slower_in_time?)
145
+ worse = agreed
146
+ return worse unless worse.empty? && measurements.all?(&:slower_in_time?)
142
147
 
143
148
  measurements.flat_map(&:slower).uniq
144
149
  end
@@ -148,18 +153,21 @@ module Duel
148
153
  # instructions in the next, say), slower in every one with none saying
149
154
  # it isn't: that takes one measurement more than one metric does.
150
155
  # @api private
151
- def mixed? = measurements.map(&:worse).reduce(:&).empty? && !regressed.empty?
156
+ def mixed? = agreed.empty? && !regressed.empty?
152
157
 
153
158
  # Whether it regressed on nothing, and didn't fail.
154
159
  def passed? = regressed.empty? && !@error
155
160
 
156
161
  # Whether a timed measure's verdict was too close to the limit to settle.
157
162
  # @api private
158
- def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "undecided" }
163
+ def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "unsettled" }
159
164
 
160
165
  private
161
166
 
162
167
  # The first measurement and its confirmations.
163
168
  def measurements = [self, *@confirmations]
169
+
170
+ # The metrics it was worse on in every measurement.
171
+ def agreed = measurements.map(&:worse).reduce(:&)
164
172
  end
165
173
  end
data/lib/duel/rubies.rb CHANGED
@@ -19,6 +19,10 @@ module Duel
19
19
  # directory.
20
20
  VERSION_FILES = %w[.ruby-version .tool-versions mise.toml .mise.toml].freeze
21
21
 
22
+ # What a Ruby's bin/ may name its executable: CRuby's, or else JRuby's,
23
+ # each on Windows with .exe.
24
+ PROGRAMS = %w[ruby ruby.exe jruby jruby.exe].freeze
25
+
22
26
  # Where chruby finds Rubies.
23
27
  CHRUBY = ["/opt/rubies", File.join(Dir.home, ".rubies")].freeze
24
28
 
@@ -194,8 +198,8 @@ module Duel
194
198
 
195
199
  MANAGERS.each_value do |manager|
196
200
  prefix = manager.call(version)
197
- # (JRuby's may have no bin/ruby.)
198
- ruby = prefix && %w[ruby jruby].map { |program| File.join(prefix, "bin", program) }.find { |path| File.executable?(path) }
201
+ # (JRuby's may have no bin/ruby, and on Windows, each is a .exe.)
202
+ ruby = prefix && PROGRAMS.map { |program| File.join(prefix, "bin", program) }.find { |path| File.executable?(path) }
199
203
  return ruby if ruby
200
204
  end
201
205
  raise Error, "the #{name}'s #{file} asks for Ruby #{version}, which isn't installed with rbenv, chruby, mise or " \
data/lib/duel/runner.rb CHANGED
@@ -21,12 +21,24 @@ module Duel
21
21
  JIT_WARM = {"jruby" => 60.0, "truffleruby" => 60.0}.fetch(RUBY_ENGINE, 0.0)
22
22
 
23
23
  # The most seconds each benchmark runs before it's measured where a
24
- # worker has YJIT on (an app may turn it on as it boots, as Rails
25
- # does), unless --warm says. A small library's methods are compiled
26
- # within the warm-up samples, but a big app's only after 120 calls
27
- # each, more than a slow benchmark's warm-up samples make.
24
+ # worker has one of CRuby's JITs on, YJIT or ZJIT (an app may turn one
25
+ # on as it boots, as Rails does YJIT), unless --warm says. Each compiles
26
+ # a method after 30 calls (YJIT, in a big app, after 120), so a small
27
+ # library's methods are compiled within the warm-up samples, but a big
28
+ # app's take more calls than a slow benchmark's warm-up samples make.
28
29
  # @api private
29
- YJIT_WARM = 10.0
30
+ CRUBY_JIT_WARM = 10.0
31
+
32
+ # CRuby's JITs, by the variable that turns each on in a worker, which
33
+ # Ruby takes as on where it's one of JIT_ON. A worker whose environment
34
+ # turns one on must run it (see Lane#check_jit).
35
+ # @api private
36
+ JIT_VARIABLES = {"yjit" => "RUBY_YJIT_ENABLE", "zjit" => "RUBY_ZJIT_ENABLE"}.freeze
37
+
38
+ # The values of a JIT's variable that turn it on, as Ruby reads them
39
+ # (exactly: TRUE or on doesn't).
40
+ # @api private
41
+ JIT_ON = %w[1 true yes].freeze
30
42
 
31
43
  # Worker processes of each version, by default: enough for an interval
32
44
  # over processes (see Lane#judge_processes), which takes 5 at 99%, and
@@ -62,7 +74,7 @@ module Duel
62
74
  # the memory available (see Runner#fit): the defaults, but not what
63
75
  # an option set.
64
76
  Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
65
- :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target, :time_limit, :accepted, :fit) do
77
+ :config_path, :timeout, :ruby, :pushed, :target, :time_limit, :accepted, :repository, :fit) do
66
78
  # @type self: singleton(Settings)
67
79
  def self.defaults = new(filter: //, jobs: 1, processes: PROCESSES, limit: 1.02, confidence: 0.99, warm: nil, force: false,
68
80
  timeout: TIMEOUT, time_limit: TIME_LIMIT, fit: [])
@@ -145,7 +157,7 @@ module Duel
145
157
  @base_root = base_root
146
158
  @out = out
147
159
  @prepared = []
148
- @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
160
+ @started = monotonic
149
161
  @lock = Mutex.new
150
162
  @unmeasured = [] #: Array[String]
151
163
  @unconfirmed = [] #: Array[String]
@@ -155,10 +167,10 @@ module Duel
155
167
  # Returns the Results.
156
168
  def run
157
169
  CycleClock.prepare
158
- started = @started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
170
+ @started = monotonic
159
171
  @dir = Lane.tmpdir
160
172
  begin
161
- compare_all(started)
173
+ compare_all
162
174
  ensure
163
175
  @prepared.reverse_each(&:stop)
164
176
  FileUtils.rm_rf(@dir)
@@ -191,7 +203,7 @@ module Duel
191
203
  # fails what still looks worse in all of them, so a run cut short
192
204
  # never passes a regression it would have failed. Each measurement it
193
205
  # started runs to its end.
194
- def out_of_time? = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started > @settings.time_limit
206
+ def out_of_time? = monotonic - @started > @settings.time_limit
195
207
 
196
208
  # The plugins the configuration used (Config#plugins), by name.
197
209
  def plugins = @config.plugins
@@ -203,8 +215,10 @@ module Duel
203
215
 
204
216
  private
205
217
 
218
+ def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
219
+
206
220
  # What #run does, with its versions prepared in its directory.
207
- def compare_all(started)
221
+ def compare_all
208
222
  @sides = prepare
209
223
  fit
210
224
  names, results = with_lanes do |lanes|
@@ -221,7 +235,7 @@ module Duel
221
235
  confirm(results.select { |r| r.worse.any? })
222
236
  @across = judge_across(results, names)
223
237
  profile(results.select { |r| (r.regressed & r.timed.keys).any? && !@benchmarks.fetch(r.name).fetch("process") })
224
- report(results, @seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
238
+ report(results, @seconds = monotonic - @started)
225
239
  results
226
240
  end
227
241
 
@@ -465,9 +479,7 @@ module Duel
465
479
  end
466
480
  # (An iteration's, as the explanation's are.)
467
481
  counted = result.counted.map do |metric, c|
468
- n = c.fetch("n") # (at least one)
469
- base = c.fetch("baseline").fdiv(n)
470
- candidate = c.fetch("candidate").fdiv(n)
482
+ base, candidate = Result.per_iteration(c)
471
483
  # (No unit its name ends in: "retained_objects 2", not "2 objects".)
472
484
  named = c.fetch("plural")&.then { |plural| metric.end_with?(plural.tr(" ", "_")) }
473
485
  [format("%s %s%s", metric, Runner.number(base), ("→#{Runner.number(candidate)}" unless base == candidate)),
data/lib/duel/spawn.rb CHANGED
@@ -100,8 +100,20 @@ module Duel
100
100
  nil
101
101
  end
102
102
 
103
- # Waits up to +seconds+ for it to end, then kills it (see #kill).
104
- def stop(seconds) = (kill unless waiter.join(seconds))
103
+ # Waits up to +seconds+ for it to end, then kills it (see #kill). One
104
+ # in a group of its own that ended leaves none of its group running:
105
+ # what it started, orphaned, would go on (a command it ran when it
106
+ # was killed, say), so what's left of the group is killed. (Not on
107
+ # Windows, where a process that's ended has no tree left to find.)
108
+ def stop(seconds)
109
+ if waiter.join(seconds)
110
+ Process.kill("KILL", -@pid) if @group && !@windows
111
+ else
112
+ kill
113
+ end
114
+ rescue *GONE
115
+ nil
116
+ end
105
117
  end
106
118
 
107
119
  # Starts a process with Process.spawn, given its redirects and pipes.
@@ -210,6 +222,13 @@ module Duel
210
222
  # LOG_LINES lines, without CRLF's CR (a Windows program's).
211
223
  def self.tail(path) = File.readlines(path, chomp: true).last(LOG_LINES).map { |line| "#{line}\n" }.join
212
224
 
225
+ # +message+, then the end of the log at +path+ (see .tail), where
226
+ # there's a log with anything in it.
227
+ def self.with_tail(message, path)
228
+ tail = path ? tail(path) : ""
229
+ tail.empty? ? message : "#{message}\n#{tail}"
230
+ end
231
+
213
232
  # The log at +path+, opened with +mode+, for a process's output: here,
214
233
  # so it's there for a failure to show, and binary, so what's copied to
215
234
  # it is as the process wrote it (on Windows, CRuby's text mode would
@@ -251,8 +270,7 @@ module Duel
251
270
  def self.open3(name, *args, jruby_on_windows: Platform::JRUBY_ON_WINDOWS, **options, &)
252
271
  return Open3.public_send(name, *args, **options, &) unless jruby_on_windows
253
272
 
254
- none = {} #: Hash[String, String?]
255
- env, *command = args.first.is_a?(Hash) ? args : [none, *args]
273
+ env, command = environment_and(args)
256
274
  current = ENV.to_h
257
275
  under = options[:unsetenv_others] ? current.keys.product([nil]).to_h : current #: Hash[String, String?]
258
276
  # (Each given by ENV's name for it, whatever its case: Windows's
@@ -262,6 +280,13 @@ module Duel
262
280
  Open3.public_send(name, under.merge(given), *program(command), **options, &)
263
281
  end
264
282
 
283
+ # +command+ as Process.spawn takes it, split: its environment (none,
284
+ # if it doesn't start with one), and the rest.
285
+ def self.environment_and(command)
286
+ none = {} #: Hash[String, String?]
287
+ command.first.is_a?(Hash) ? [command.first, command.drop(1)] : [none, command]
288
+ end
289
+
265
290
  # +command+ as Windows runs it: a lone String that a shell runs (with
266
291
  # a space in it, say) as cmd.exe's, and otherwise with its program
267
292
  # found (by name, in +path+'s directories), as the program it is, or