duel 0.1.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +2 -2
  3. data/CHANGELOG.md +203 -1
  4. data/README.md +41 -1
  5. data/docs/ci.md +170 -76
  6. data/docs/configuration.md +38 -2
  7. data/docs/how-it-works.md +18 -9
  8. data/docs/plugins.md +23 -0
  9. data/lib/duel/across.rb +1 -6
  10. data/lib/duel/baseline.rb +8 -7
  11. data/lib/duel/cli.rb +96 -28
  12. data/lib/duel/comment.rb +296 -0
  13. data/lib/duel/config.rb +124 -4
  14. data/lib/duel/context.rb +33 -34
  15. data/lib/duel/control.rb +52 -0
  16. data/lib/duel/init.rb +255 -20
  17. data/lib/duel/lane.rb +734 -0
  18. data/lib/duel/lock.rb +34 -4
  19. data/lib/duel/memory.rb +14 -5
  20. data/lib/duel/metric.rb +97 -26
  21. data/lib/duel/native.rb +199 -32
  22. data/lib/duel/platform.rb +31 -0
  23. data/lib/duel/process_sample.rb +74 -49
  24. data/lib/duel/resource.rb +71 -42
  25. data/lib/duel/result.rb +165 -0
  26. data/lib/duel/rubies.rb +21 -3
  27. data/lib/duel/runner.rb +33 -1044
  28. data/lib/duel/spawn.rb +327 -0
  29. data/lib/duel/summary.rb +21 -15
  30. data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
  31. data/lib/duel/templates/perf-caller.yml.erb +43 -0
  32. data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
  33. data/lib/duel/templates/perf-comment.yml.erb +61 -44
  34. data/lib/duel/templates/perf.yml.erb +143 -7
  35. data/lib/duel/testing.rb +3 -0
  36. data/lib/duel/version.rb +1 -1
  37. data/lib/duel/worker.rb +13 -4
  38. data/lib/duel/worker_process.rb +142 -0
  39. data/lib/duel.rb +35 -3
  40. data/sig/duel/cli.rbs +19 -0
  41. data/sig/duel/comment.rbs +97 -0
  42. data/sig/duel/config.rbs +57 -2
  43. data/sig/duel/context.rbs +4 -31
  44. data/sig/duel/control.rbs +27 -0
  45. data/sig/duel/init.rbs +95 -1
  46. data/sig/duel/lane.rbs +202 -0
  47. data/sig/duel/lock.rbs +8 -1
  48. data/sig/duel/memory.rbs +1 -1
  49. data/sig/duel/metric.rbs +53 -10
  50. data/sig/duel/native.rbs +109 -9
  51. data/sig/duel/platform.rbs +13 -0
  52. data/sig/duel/process_sample.rbs +7 -9
  53. data/sig/duel/resource.rbs +28 -9
  54. data/sig/duel/result.rbs +164 -0
  55. data/sig/duel/rubies.rbs +4 -0
  56. data/sig/duel/runner.rbs +6 -460
  57. data/sig/duel/spawn.rbs +111 -0
  58. data/sig/duel/summary.rbs +6 -3
  59. data/sig/duel/worker_process.rbs +90 -0
  60. data/sig/duel.rbs +17 -3
  61. metadata +121 -105
@@ -2,6 +2,9 @@
2
2
 
3
3
  require "json"
4
4
  require_relative "native"
5
+ require_relative "platform"
6
+ require_relative "metric"
7
+ require_relative "spawn"
5
8
 
6
9
  module Duel
7
10
  # The runs of a process benchmark (see DSL#duel_process) for one worker:
@@ -15,14 +18,17 @@ module Duel
15
18
  # memory where it can tell, and how it ended.
16
19
  # @api private
17
20
  class ProcessSample
18
- # The lines of its output a failure shows.
19
- LOG_LINES = 10
21
+ # Seconds the Ruby process of its own may take to end once its input
22
+ # is closed, before it's killed, with what it started (see #stop).
23
+ STOP_WAIT = 10
20
24
 
21
25
  # Runs for the worker +label+, in +chdir+ with +env+ (over the
22
26
  # runner's own), each process's output to +log+. A run that takes
23
- # more than +timeout+ seconds fails.
24
- def initialize(label, env:, chdir:, log:, timeout:)
27
+ # more than +timeout+ seconds fails. (+starter+: how its Ruby process
28
+ # starts, Spawn::Directly or Spawn::ThroughOpen3.)
29
+ def initialize(label, env:, chdir:, log:, timeout:, starter: Spawn::STARTER)
25
30
  @label = label
31
+ @starter = starter
26
32
  @env = env
27
33
  @chdir = chdir
28
34
  @log = log
@@ -44,13 +50,20 @@ module Duel
44
50
  # timeout (it's killed, with what it started, and the Ruby process
45
51
  # that ran it), fails.
46
52
  def run(command)
47
- start unless @pid
53
+ start unless @child
48
54
  @requests.puts JSON.generate(command)
49
- unless @replies.wait_readable(@timeout)
55
+ # (Read on a thread of its own, waited for until the timeout: on JRuby
56
+ # on Windows, IO.select takes Open3's pipe for ready before it is,
57
+ # https://github.com/jruby/jruby/issues/5765, and wait_readable
58
+ # never does, on JRuby 10.0.)
59
+ replies = @replies #: IO
60
+ reading = Thread.new { replies.gets }
61
+ reading.report_on_exception = false
62
+ unless reading.join(@timeout)
50
63
  kill
51
64
  raise Failed, "#{@label}: #{command.join(" ")} didn't end within #{@timeout}s (killed)"
52
65
  end
53
- reply = @replies.gets
66
+ reply = reading.value
54
67
  unless reply
55
68
  stop
56
69
  raise Failed, "#{@label}: the process running #{command.join(" ")} exited"
@@ -59,30 +72,33 @@ module Duel
59
72
  ended = JSON.parse(reply)
60
73
  failed = ended.fetch("failed")
61
74
  if failed
62
- tail = File.readlines(@log).last(LOG_LINES).join
75
+ tail = Spawn.tail(@log)
63
76
  raise Failed, "#{@label}: #{command.join(" ")} #{failed}#{"\n#{tail}" unless tail.empty?}"
64
77
  end
65
78
  {"timed" => {"cpu" => ended.fetch("cpu")}, "counted" => {}, "heap" => 0, "peak" => ended.fetch("peak")}
66
79
  end
67
80
 
68
81
  # Ends the Ruby process of its own, if it started: closing its input
69
- # does.
70
- def stop
71
- pid = @pid
72
- return unless pid
82
+ # does. One still running +wait+ seconds later is killed, with what it
83
+ # started: on Windows, a process started meanwhile (another worker)
84
+ # can have inherited the input's other end, which keeps it open.
85
+ def stop(wait: STOP_WAIT)
86
+ child = @child
87
+ return unless child
73
88
 
74
89
  @requests.close
75
- Process.wait(pid)
90
+ child.stop(wait)
76
91
  @replies.close
77
- @pid = nil
92
+ @child = nil
78
93
  end
79
94
 
80
- # Runs a process for each line of +input+ (a command, in JSON), with
81
- # what +spec+ says ({"env", "chdir", "log"}), and reports each (see
82
- # .once) as a line of +output+, at once, until the input ends: run in
83
- # the Ruby process of its own.
84
- def self.main(spec, input: $stdin, output: $stdout)
85
- spec = JSON.parse(spec)
95
+ # Runs a process for each line of +input+ (a command, in JSON) after
96
+ # its first, with what that says ({"env", "chdir", "log"}), and
97
+ # reports each (see .once) as a line of +output+, at once, until the
98
+ # input ends: run in the Ruby process of its own.
99
+ def self.main(input: $stdin, output: $stdout)
100
+ first = input.gets or return
101
+ spec = JSON.parse(first)
86
102
  output.sync = true
87
103
  while (line = input.gets)
88
104
  output.puts JSON.generate(once(JSON.parse(line), spec.fetch("env"), spec.fetch("chdir"), spec.fetch("log")))
@@ -91,12 +107,32 @@ module Duel
91
107
 
92
108
  # Runs +command+ once, in +chdir+ with +env+ alone, its output to
93
109
  # +log+, and returns what it took: its CPU time, its peak memory (or
94
- # nil), and how it failed (or nil).
95
- def self.once(command, env, chdir, log)
110
+ # nil), and how it failed (or nil). On Windows, whose children's CPU
111
+ # time Process.times doesn't have, what its own handle on the process
112
+ # says (see Native::Windows::Spawned), its cycles in nanoseconds (see
113
+ # CycleClock). A command that can't start (there's no such program,
114
+ # say, or on Windows, a shell command with no COMSPEC in +env+ to
115
+ # find cmd.exe by) failed, and took nothing.
116
+ def self.once(command, env, chdir, log, windows: Platform::WINDOWS, starter: Spawn::STARTER)
96
117
  before = cpu
97
- code, signal, peak = Native::Child.wait(Process.spawn(env, *command, chdir: chdir, in: File::NULL, out: [log, "w"],
98
- err: [:child, :out], unsetenv_others: true, close_others: true))
99
- {"cpu" => cpu - before, "peak" => peak, "failed" => (why(code, signal) unless code == 0)}
118
+ begin
119
+ child = starter.logged(env, command, log: log, mode: "w", chdir: chdir, unsetenv_others: true, close_others: true)
120
+ rescue SystemCallError => e
121
+ return {"cpu" => 0, "peak" => nil, "failed" => "couldn't start: #{e}"}
122
+ end
123
+ spawned = Native::Windows::Spawned.open(child.pid) if windows
124
+ begin
125
+ code, signal, peak = child.wait
126
+ if spawned
127
+ taken = CycleClock.ns(spawned.cycles)
128
+ peak = spawned.peak
129
+ else
130
+ taken = cpu - before
131
+ end
132
+ {"cpu" => taken, "peak" => peak, "failed" => (Spawn.why(code, signal) unless code == 0)} #: ended
133
+ ensure
134
+ spawned&.close
135
+ end
100
136
  end
101
137
 
102
138
  # The CPU time of the children waited for so far, in nanoseconds.
@@ -105,40 +141,29 @@ module Duel
105
141
  ((times.cutime + times.cstime) * 1_000_000_000).round
106
142
  end
107
143
 
108
- # How a process that failed ended: its exit status +code+, or the
109
- # +signal+ that killed it.
110
- def self.why(code, signal)
111
- return "exited with #{code}" if code
112
-
113
- killed = signal #: Integer
114
- "was killed by SIG#{Signal.signame(killed)}"
115
- end
116
-
117
144
  private
118
145
 
119
- # Starts the Ruby process of its own, given the environment, the
120
- # directory and the log, and connected by a pipe each way (not a
121
- # socket: JRuby's spawn can't give one).
146
+ # Starts the Ruby process of its own, connected by a pipe each way
147
+ # (not a socket: JRuby's spawn can't give one), and gives it the
148
+ # environment, the directory and the log, as the first line of its
149
+ # input. (Not an argument: JRuby's spawn on Windows splits one at its
150
+ # spaces, and the environment can be longer than Windows lets a
151
+ # command be.)
122
152
  def start
123
- spec = JSON.generate({"env" => {**ENV, **@env}, "chdir" => @chdir, "log" => @log})
124
- requests, @requests = IO.pipe
125
- @replies, replies = IO.pipe
126
153
  # (Out of the runner's bundle: it needs nothing of it.)
127
- @pid = Process.spawn({"RUBYOPT" => nil, "RUBYLIB" => nil}, RbConfig.ruby,
128
- *WorkerProcess.load_path(RbConfig.ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/process_sample", "-e",
129
- "Duel::ProcessSample.main(*ARGV)", spec, in: requests, out: replies, pgroup: true, close_others: true)
130
- requests.close
131
- replies.close
154
+ @child, @requests, @replies = @starter.piped({"RUBYOPT" => nil, "RUBYLIB" => nil},
155
+ [RbConfig.ruby, *WorkerProcess.load_path(RbConfig.ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/process_sample", "-e",
156
+ "Duel::ProcessSample.main"], group: true, close_others: true)
157
+ @requests.puts JSON.generate({"env" => {**ENV, **@env}, "chdir" => @chdir, "log" => @log})
132
158
  end
133
159
 
134
160
  # Kills the Ruby process of its own, and what it started.
135
161
  def kill
136
- pid = @pid #: Integer
137
- Process.kill("KILL", -pid)
138
- Process.wait(pid)
162
+ child = @child #: Spawn::Child
163
+ child.kill
139
164
  @requests.close
140
165
  @replies.close
141
- @pid = nil
166
+ @child = nil
142
167
  end
143
168
  end
144
169
  end
data/lib/duel/resource.rb CHANGED
@@ -2,6 +2,8 @@
2
2
 
3
3
  require "fileutils"
4
4
  require "rbconfig"
5
+ require_relative "control"
6
+ require_relative "spawn"
5
7
 
6
8
  module Duel
7
9
  # What a prepare block (see Config#prepare) gets: a worker's, or a
@@ -29,10 +31,6 @@ module Duel
29
31
  # @api private
30
32
  STOP_WAIT = 5
31
33
 
32
- # The lines of its log a failure shows.
33
- # @api private
34
- LOG_LINES = 10
35
-
36
34
  # What a cache key may be.
37
35
  # @api private
38
36
  KEY = /\A[\w.-]+\z/
@@ -71,6 +69,26 @@ module Duel
71
69
  nil
72
70
  end
73
71
 
72
+ # A process #spawn started: its program (what its log is named for),
73
+ # its log, the signal that stops it, and its Spawn::Child.
74
+ # @api private
75
+ Spawned = Data.define(:program, :log, :stop_signal, :child)
76
+
77
+ # Runs each of +blocks+ (Config#prepare's), in order, each given a
78
+ # Resource of its own (+dir+-1, +dir+-2...) for +name+ in +root+, on
79
+ # +ruby+, which it gives the block +kept+ (to stop it later). Returns
80
+ # +env+ with what each block returned (see .environment). Each
81
+ # Resource's own env is +env+: a worker's, its version's alone, but a
82
+ # version's (+so_far+), with what the blocks before it returned too.
83
+ # @api private
84
+ def self.prepare(blocks, dir, name, root, env:, ruby:, timeout:, cache:, so_far: false)
85
+ blocks.each_with_index.reduce(env) do |set, (block, i)|
86
+ resource = new("#{dir}-#{i + 1}", name, root, timeout:, cache:, ruby:, env: so_far ? set : env)
87
+ yield resource
88
+ environment(name, set, block.call(resource))
89
+ end
90
+ end
91
+
74
92
  # The worker it's for ("base-1", say), or the version ("base",
75
93
  # for Config#prepare(:version)), the directory it has of its own, the
76
94
  # tree of the version that worker runs, and the Ruby it runs on (see
@@ -93,8 +111,10 @@ module Duel
93
111
  end
94
112
  end
95
113
 
114
+ # (+starter+: how its processes start, Spawn::Directly or
115
+ # Spawn::ThroughOpen3.)
96
116
  # @api private
97
- def initialize(dir, name, root, timeout:, cache:, ruby: RbConfig.ruby, env: {})
117
+ def initialize(dir, name, root, timeout:, cache:, ruby: RbConfig.ruby, env: {}, starter: Spawn::STARTER)
98
118
  @name = name
99
119
  @dir = dir
100
120
  @root = root
@@ -102,7 +122,8 @@ module Duel
102
122
  @env = env
103
123
  @timeout = timeout
104
124
  @cache = cache
105
- @processes = {}
125
+ @starter = starter
126
+ @processes = []
106
127
  @stops = []
107
128
  FileUtils.mkdir_p(dir)
108
129
  end
@@ -111,6 +132,18 @@ module Duel
111
132
  # bytes on macOS. Duel's directories leave room.)
112
133
  def path(file) = File.join(@dir, file)
113
134
 
135
+ # A port of 127.0.0.1's that's free (one the system chose, let go of
136
+ # again), for a server of its own to listen on, rather than on a Unix
137
+ # socket in its directory (see #path).
138
+ def port
139
+ server = Control.tcp_server
140
+ begin
141
+ server.addr.fetch(1)
142
+ ensure
143
+ server.close
144
+ end
145
+ end
146
+
114
147
  # A directory built once, by the block, for every worker of this run
115
148
  # and later ones (a database's initial files, say, for each worker to
116
149
  # copy): the first that asks for +key+ builds it, in a directory the
@@ -134,15 +167,17 @@ module Duel
134
167
  # Starts a process (as Process.spawn takes it), its output added to a
135
168
  # log in the directory (+log+.log: by default, its program's name),
136
169
  # and stopped with the resource, by +stop_signal+ (then KILL, five
137
- # seconds later). Returns its pid. Its input is
170
+ # seconds later). On Windows, which can't send another process a
171
+ # signal, it's killed: ask it to stop in an #at_stop block, which
172
+ # runs first. Returns its pid. Its input is
138
173
  # nothing, and it gets no other file descriptors, unless +options+
139
174
  # say otherwise (in:, close_others:). It runs in its version's
140
175
  # environment (on its Ruby: see Config#ruby), under any it's given.
141
176
  def spawn(*command, log: nil, stop_signal: "TERM", **options)
142
177
  log = log_path(command, log)
143
- pid = start(command, log, options)
144
- @processes[pid] = [File.basename(log, ".log"), log, stop_signal]
145
- pid
178
+ child = start(command, log, options)
179
+ @processes << Spawned.new(program: File.basename(log, ".log"), log:, stop_signal:, child:)
180
+ child.pid
146
181
  end
147
182
 
148
183
  # Runs a command (setting something up, say) to its end, its output
@@ -150,12 +185,10 @@ module Duel
150
185
  # succeeds.
151
186
  def run(*command, log: nil, **options)
152
187
  log = log_path(command, log)
153
- _, status = Process.wait2(start(command, log, options))
154
- return if status.success?
188
+ code, signal = start(command, log, options).wait
189
+ return if code == 0
155
190
 
156
- code = status.exitstatus
157
- signal = status.termsig #: Integer
158
- failed("#{File.basename(log, ".log")} #{code ? "exited with #{code}" : "was killed by SIG#{Signal.signame(signal)}"}", log: log)
191
+ failed("#{File.basename(log, ".log")} #{Spawn.why(code, signal)}", log: log)
159
192
  end
160
193
 
161
194
  # Waits until the block is true: fails if a process spawned dies
@@ -163,15 +196,14 @@ module Duel
163
196
  def wait_until
164
197
  deadline = monotonic + @timeout
165
198
  until yield
166
- ended = @processes.find { |pid, _| Process.wait(pid, Process::WNOHANG) }
199
+ ended = @processes.find { |process| !process.child.alive? }
167
200
  if ended
168
201
  # (It may have got ready, and ended, since the last look.)
169
202
  break if yield
170
203
 
171
- program, log = ended.last
172
- failed("#{program} exited before it was ready", log: log)
204
+ failed("#{ended.program} exited before it was ready", log: ended.log)
173
205
  end
174
- failed("not ready within #{@timeout}s", log: @processes.values.last&.fetch(1)) if monotonic > deadline
206
+ failed("not ready within #{@timeout}s", log: @processes.last&.log) if monotonic > deadline
175
207
  sleep POLL
176
208
  end
177
209
  end
@@ -182,7 +214,7 @@ module Duel
182
214
  # Fails, saying it's this resource that did, with the end of +log+ (a
183
215
  # path) to say why.
184
216
  def failed(message, log: nil)
185
- tail = log ? File.readlines(log).last(LOG_LINES).join : ""
217
+ tail = log ? Spawn.tail(log) : ""
186
218
  raise Failed, "#{@name}'s resource: #{message}#{"\n#{tail}" unless tail.empty?}"
187
219
  end
188
220
 
@@ -190,7 +222,10 @@ module Duel
190
222
  # @api private
191
223
  def stop
192
224
  @stops.reverse_each(&:call)
193
- @processes.each { |pid, (_, _, signal)| end_process(pid, signal) }
225
+ @processes.each do |process|
226
+ process.child.signal(process.stop_signal)
227
+ process.child.stop(STOP_WAIT)
228
+ end
194
229
  end
195
230
 
196
231
  private
@@ -211,32 +246,26 @@ module Duel
211
246
  end
212
247
  end
213
248
 
214
- # The log a command's output goes to: +name+.log, or its program's.
215
- def log_path(command, name) = path("#{name || File.basename(command.grep(String).fetch(0).split.fetch(0))}.log")
249
+ # The log a command's output goes to: +name+.log, or its program's:
250
+ # a command line's first word (a String alone, which a shell runs), or
251
+ # else the program given its arguments, whole, as its path may have a
252
+ # space in it (C:/Program Files/PostgreSQL/17/bin/initdb, say).
253
+ def log_path(command, name)
254
+ words = command.grep(String)
255
+ program = words.fetch(0)
256
+ program = program.split.fetch(0) if words.size == 1
257
+ path("#{name || File.basename(program)}.log")
258
+ end
216
259
 
260
+ # Starts +command+, its output added to +log+, and returns its
261
+ # Spawn::Child, waited for on a thread from the start (see
262
+ # Child#detach), so it's reaped as it ends. (On a thread: a Timeout
263
+ # can't interrupt a wait on JRuby.)
217
264
  def start(command, log, options)
218
265
  # (Its own environment, if it's given one, over its version's.)
219
266
  none = {} #: Hash[String, String?]
220
267
  env, *program = command.first.is_a?(Hash) ? command : [none, *command]
221
- out = File.new(log, "a") # (here, so it's there for a failure to show)
222
- begin
223
- Process.spawn(@env.merge(env), *program, in: File::NULL, out: out, err: out, close_others: true, **options)
224
- ensure
225
- out.close
226
- end
227
- end
228
-
229
- # (Waited for on a thread of its own: a Timeout can't interrupt a wait
230
- # on JRuby.)
231
- def end_process(pid, signal)
232
- Process.kill(signal, pid)
233
- waiter = Process.detach(pid)
234
- return if waiter.join(STOP_WAIT)
235
-
236
- Process.kill("KILL", pid)
237
- waiter.join
238
- rescue Errno::ESRCH # gone already
239
- nil
268
+ @starter.logged(@env.merge(env), program, log: log, mode: "a", close_others: true, **options).detach
240
269
  end
241
270
 
242
271
  def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
@@ -0,0 +1,165 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../duel"
4
+
5
+ module Duel
6
+ # What one comparison of a benchmark found (Testing::Comparison#results
7
+ # has them): its #name, whether it #passed?, the measures it #regressed
8
+ # on, each #timed measure's verdict and each #counted one's counts,
9
+ # what metrics said of it (#details), its peak #memory, its iterations
10
+ # a sample (#n), the #seconds it took, why it was #skipped or its
11
+ # #error, and its #confirmations (the measurements that confirmed it
12
+ # was worse, not those too close to call: see #confirm). Its writers,
13
+ # and the rest, are @api private.
14
+ class Result
15
+ # Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
16
+ # bytes at least, before it's worse (it's measured as a whole, with the
17
+ # garbage collector working, so it isn't exact).
18
+ # @api private
19
+ MEMORY_RATIO = 1.1
20
+
21
+ # @api private
22
+ MEMORY_FLOOR = 1 << 20
23
+
24
+ # What a sample says when no metric said anything (and what a measure
25
+ # it said nothing of did).
26
+ # @api private
27
+ NOTHING_SAID = {} #: Hash[String, String]
28
+
29
+ # The verdicts that say a timing isn't slower.
30
+ # @api private
31
+ CLEARED = %w[same faster].freeze
32
+
33
+ attr_reader :name, :timed, :counted, :details, :confirmations, :n, :seconds, :skipped, :error, :memory
34
+
35
+ # Where a regression's time went (see Profile).
36
+ # @api private
37
+ attr_reader :profile
38
+
39
+ # Each worker's median of each measure judged by process (see
40
+ # Lane#judge_processes): {metric => {"lane" => its lane's number,
41
+ # "baseline" => [each baseline worker's], "candidate" => [...]}},
42
+ # for across the board.
43
+ # @api private
44
+ attr_reader :processes
45
+
46
+ # The measurements again that were too close to call (see #confirm).
47
+ # @api private
48
+ attr_reader :undecided
49
+
50
+ # @api private
51
+ attr_writer :n, :seconds, :skipped, :error, :memory, :profile
52
+
53
+ # @api private
54
+ def initialize(name)
55
+ @name = name
56
+ @timed = {}
57
+ @counted = {}
58
+ @details = {}
59
+ @confirmations = []
60
+ @undecided = []
61
+ @processes = {}
62
+ end
63
+
64
+ # Keeps what metrics said of a pair of samples (Metric#details): each
65
+ # measure's latest, by version (only +measure+'s, if given).
66
+ # @api private
67
+ def note(base, candidate, measure = nil)
68
+ {"baseline" => base, "candidate" => candidate}.each do |side, sample|
69
+ said = sample["details"] || NOTHING_SAID
70
+ said = said.slice(measure) if measure
71
+ said.each { |name, text| (@details[name] ||= {}).store(side, text) }
72
+ end
73
+ end
74
+
75
+ # The metrics on which the candidate did worse: a timed one slower, or
76
+ # a counted one up by more than its tolerance per iteration and its
77
+ # allowance per sample (counted by process, up by that much at least:
78
+ # see Lane#count_by_process).
79
+ # @api private
80
+ def worse
81
+ more = @counted.select do |_, c|
82
+ !Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
83
+ allowance: c.fetch("allowance"))
84
+ end
85
+ slower + more.keys + (more_memory? ? ["peak_memory"] : [])
86
+ end
87
+
88
+ # The timed metrics on which the candidate was slower.
89
+ # @api private
90
+ def slower = @timed.filter_map { |metric, t| metric if t.fetch("verdict") == "slower" }
91
+
92
+ # Whether its timings say slower: one is, and none says it isn't
93
+ # (same, or faster).
94
+ # @api private
95
+ def slower_in_time? = slower.any? && @timed.none? { |_, t| CLEARED.include?(t.fetch("verdict")) }
96
+
97
+ # Whether it's to be measured again (see Runner#confirm): it looks
98
+ # worse in every measurement so far, and there are fewer confirmations
99
+ # than it takes to fail (Runner::CONFIRM, or one more where it's
100
+ # #mixed?).
101
+ # @api private
102
+ def confirming?
103
+ return false if regressed.empty?
104
+
105
+ @confirmations.size < (mixed? ? Runner::CONFIRM + 1 : Runner::CONFIRM)
106
+ end
107
+
108
+ # Keeps +again+, a measurement of it again from scratch (see
109
+ # Runner#confirm), as a confirmation, or sets it aside where it's too
110
+ # close to call, up to Runner::CONFIRM times: every timing is still
111
+ # undecided there (none slower, none the same or faster), and each
112
+ # measure it regressed on so far is one of them. One that can't tell
113
+ # neither confirms it nor clears it, so it's measured again instead.
114
+ # Returns whether +again+ was set aside.
115
+ # @api private
116
+ def confirm(again)
117
+ timed = again.timed
118
+ close = timed.all? { |_, t| t.fetch("verdict") == "undecided" } && regressed.all? { |metric| timed.key?(metric) } &&
119
+ @undecided.size < Runner::CONFIRM
120
+ (close ? @undecided : @confirmations) << again
121
+ close
122
+ end
123
+
124
+ # Whether its peak memory grew by too much (see MEMORY_RATIO).
125
+ # @api private
126
+ def more_memory?
127
+ memory = @memory
128
+ return false unless memory
129
+
130
+ base = memory.fetch("baseline")
131
+ memory.fetch("candidate") - base > [MEMORY_FLOOR, base * (MEMORY_RATIO - 1)].max
132
+ end
133
+
134
+ # Worse on a metric in the first measurement and in every confirmation.
135
+ # Or, where no one metric is, slower in time in all of them (see
136
+ # #mixed?): the timed metrics slower in any.
137
+ def regressed
138
+ return [] if @error || @skipped
139
+
140
+ agreed = measurements.map(&:worse).reduce(:&)
141
+ return agreed unless agreed.empty? && measurements.all?(&:slower_in_time?)
142
+
143
+ measurements.flat_map(&:slower).uniq
144
+ end
145
+
146
+ # Whether it regressed in time on one timed metric in some
147
+ # measurements and on another in the rest (CPU time in one,
148
+ # instructions in the next, say), slower in every one with none saying
149
+ # it isn't: that takes one measurement more than one metric does.
150
+ # @api private
151
+ def mixed? = measurements.map(&:worse).reduce(:&).empty? && !regressed.empty?
152
+
153
+ # Whether it regressed on nothing, and didn't fail.
154
+ def passed? = regressed.empty? && !@error
155
+
156
+ # Whether a timed measure's verdict was too close to the limit to settle.
157
+ # @api private
158
+ def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "undecided" }
159
+
160
+ private
161
+
162
+ # The first measurement and its confirmations.
163
+ def measurements = [self, *@confirmations]
164
+ end
165
+ end
data/lib/duel/rubies.rb CHANGED
@@ -3,6 +3,8 @@
3
3
  require "open3"
4
4
  require "rbconfig"
5
5
  require_relative "../duel"
6
+ require_relative "platform"
7
+ require_relative "spawn"
6
8
 
7
9
  module Duel
8
10
  # The Ruby each version runs on (Config#ruby): by default the runner's.
@@ -70,8 +72,9 @@ module Duel
70
72
  def engine(ruby)
71
73
  return RUBY_ENGINE if runner?(ruby)
72
74
 
73
- out = IO.popen(environment(ruby).merge("RUBYOPT" => nil), [ruby, "--disable-gems", "-e", "print RUBY_ENGINE"], err: File::NULL, &:read)
74
- status = $? #: Process::Status
75
+ # (Its errors read, not shown: JRuby's IO.popen on Windows ignores
76
+ # err:, https://github.com/jruby/jruby/issues/10081.)
77
+ out, _, status = Spawn.open3(:capture3, environment(ruby).merge("RUBYOPT" => nil), ruby, "--disable-gems", "-e", "print RUBY_ENGINE")
75
78
  status.success? ? out : "ruby"
76
79
  rescue SystemCallError
77
80
  "ruby"
@@ -92,6 +95,21 @@ module Duel
92
95
  # Whether +ruby+ is the runner's own.
93
96
  def runner?(ruby) = File.realpath(ruby) == File.realpath(RbConfig.ruby)
94
97
 
98
+ # The command that starts +ruby+ (whose engine is +engine+), with
99
+ # +env+: +ruby+ itself, but on JRuby on Windows, whose jruby.exe
100
+ # starts the JVM in a process of its own (javaw.exe), the JVM's
101
+ # command, as `jruby -Xcommand` says it is: so the process Duel
102
+ # measures, waits for, and kills is the one running Ruby.
103
+ def launch(ruby, engine, env, windows: Platform::WINDOWS)
104
+ return [ruby] unless windows && engine == "jruby"
105
+
106
+ out, status = Spawn.open3(:capture2, env, ruby, "-Xcommand", "-e", "1")
107
+ args = out.lines(chomp: true)
108
+ (status.success? && args.last(2) == %w[-e 1]) ? args.first(args.size - 2) : [ruby]
109
+ rescue SystemCallError
110
+ [ruby]
111
+ end
112
+
95
113
  # What a version's processes on +ruby+ get in their environment:
96
114
  # nothing on the runner's Ruby. On another, its directory first on
97
115
  # +path+ (for what they start by name, `bundle` say), and none of the
@@ -154,7 +172,7 @@ module Duel
154
172
 
155
173
  # What a command prints, or nil if it fails (or isn't there).
156
174
  def command(*command)
157
- out, _, status = Open3.capture3(*command)
175
+ out, _, status = Spawn.open3(:capture3, *command)
158
176
  out.strip if status.success?
159
177
  rescue SystemCallError
160
178
  nil