duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/lib/duel/process_sample.rb
CHANGED
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "json"
|
|
4
4
|
require_relative "native"
|
|
5
|
+
require_relative "platform"
|
|
6
|
+
require_relative "metric"
|
|
7
|
+
require_relative "spawn"
|
|
5
8
|
|
|
6
9
|
module Duel
|
|
7
10
|
# The runs of a process benchmark (see DSL#duel_process) for one worker:
|
|
@@ -15,14 +18,17 @@ module Duel
|
|
|
15
18
|
# memory where it can tell, and how it ended.
|
|
16
19
|
# @api private
|
|
17
20
|
class ProcessSample
|
|
18
|
-
#
|
|
19
|
-
|
|
21
|
+
# Seconds the Ruby process of its own may take to end once its input
|
|
22
|
+
# is closed, before it's killed, with what it started (see #stop).
|
|
23
|
+
STOP_WAIT = 10
|
|
20
24
|
|
|
21
25
|
# Runs for the worker +label+, in +chdir+ with +env+ (over the
|
|
22
26
|
# runner's own), each process's output to +log+. A run that takes
|
|
23
|
-
# more than +timeout+ seconds fails.
|
|
24
|
-
|
|
27
|
+
# more than +timeout+ seconds fails. (+starter+: how its Ruby process
|
|
28
|
+
# starts, Spawn::Directly or Spawn::ThroughOpen3.)
|
|
29
|
+
def initialize(label, env:, chdir:, log:, timeout:, starter: Spawn::STARTER)
|
|
25
30
|
@label = label
|
|
31
|
+
@starter = starter
|
|
26
32
|
@env = env
|
|
27
33
|
@chdir = chdir
|
|
28
34
|
@log = log
|
|
@@ -44,13 +50,20 @@ module Duel
|
|
|
44
50
|
# timeout (it's killed, with what it started, and the Ruby process
|
|
45
51
|
# that ran it), fails.
|
|
46
52
|
def run(command)
|
|
47
|
-
start unless @
|
|
53
|
+
start unless @child
|
|
48
54
|
@requests.puts JSON.generate(command)
|
|
49
|
-
|
|
55
|
+
# (Read on a thread of its own, waited for until the timeout: on JRuby
|
|
56
|
+
# on Windows, IO.select takes Open3's pipe for ready before it is,
|
|
57
|
+
# https://github.com/jruby/jruby/issues/5765, and wait_readable
|
|
58
|
+
# never does, on JRuby 10.0.)
|
|
59
|
+
replies = @replies #: IO
|
|
60
|
+
reading = Thread.new { replies.gets }
|
|
61
|
+
reading.report_on_exception = false
|
|
62
|
+
unless reading.join(@timeout)
|
|
50
63
|
kill
|
|
51
64
|
raise Failed, "#{@label}: #{command.join(" ")} didn't end within #{@timeout}s (killed)"
|
|
52
65
|
end
|
|
53
|
-
reply =
|
|
66
|
+
reply = reading.value
|
|
54
67
|
unless reply
|
|
55
68
|
stop
|
|
56
69
|
raise Failed, "#{@label}: the process running #{command.join(" ")} exited"
|
|
@@ -59,30 +72,33 @@ module Duel
|
|
|
59
72
|
ended = JSON.parse(reply)
|
|
60
73
|
failed = ended.fetch("failed")
|
|
61
74
|
if failed
|
|
62
|
-
tail =
|
|
75
|
+
tail = Spawn.tail(@log)
|
|
63
76
|
raise Failed, "#{@label}: #{command.join(" ")} #{failed}#{"\n#{tail}" unless tail.empty?}"
|
|
64
77
|
end
|
|
65
78
|
{"timed" => {"cpu" => ended.fetch("cpu")}, "counted" => {}, "heap" => 0, "peak" => ended.fetch("peak")}
|
|
66
79
|
end
|
|
67
80
|
|
|
68
81
|
# Ends the Ruby process of its own, if it started: closing its input
|
|
69
|
-
# does.
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
82
|
+
# does. One still running +wait+ seconds later is killed, with what it
|
|
83
|
+
# started: on Windows, a process started meanwhile (another worker)
|
|
84
|
+
# can have inherited the input's other end, which keeps it open.
|
|
85
|
+
def stop(wait: STOP_WAIT)
|
|
86
|
+
child = @child
|
|
87
|
+
return unless child
|
|
73
88
|
|
|
74
89
|
@requests.close
|
|
75
|
-
|
|
90
|
+
child.stop(wait)
|
|
76
91
|
@replies.close
|
|
77
|
-
@
|
|
92
|
+
@child = nil
|
|
78
93
|
end
|
|
79
94
|
|
|
80
|
-
# Runs a process for each line of +input+ (a command, in JSON)
|
|
81
|
-
# what
|
|
82
|
-
# .once) as a line of +output+, at once, until the
|
|
83
|
-
# the Ruby process of its own.
|
|
84
|
-
def self.main(
|
|
85
|
-
|
|
95
|
+
# Runs a process for each line of +input+ (a command, in JSON) after
|
|
96
|
+
# its first, with what that says ({"env", "chdir", "log"}), and
|
|
97
|
+
# reports each (see .once) as a line of +output+, at once, until the
|
|
98
|
+
# input ends: run in the Ruby process of its own.
|
|
99
|
+
def self.main(input: $stdin, output: $stdout)
|
|
100
|
+
first = input.gets or return
|
|
101
|
+
spec = JSON.parse(first)
|
|
86
102
|
output.sync = true
|
|
87
103
|
while (line = input.gets)
|
|
88
104
|
output.puts JSON.generate(once(JSON.parse(line), spec.fetch("env"), spec.fetch("chdir"), spec.fetch("log")))
|
|
@@ -91,12 +107,32 @@ module Duel
|
|
|
91
107
|
|
|
92
108
|
# Runs +command+ once, in +chdir+ with +env+ alone, its output to
|
|
93
109
|
# +log+, and returns what it took: its CPU time, its peak memory (or
|
|
94
|
-
# nil), and how it failed (or nil).
|
|
95
|
-
|
|
110
|
+
# nil), and how it failed (or nil). On Windows, whose children's CPU
|
|
111
|
+
# time Process.times doesn't have, what its own handle on the process
|
|
112
|
+
# says (see Native::Windows::Spawned), its cycles in nanoseconds (see
|
|
113
|
+
# CycleClock). A command that can't start (there's no such program,
|
|
114
|
+
# say, or on Windows, a shell command with no COMSPEC in +env+ to
|
|
115
|
+
# find cmd.exe by) failed, and took nothing.
|
|
116
|
+
def self.once(command, env, chdir, log, windows: Platform::WINDOWS, starter: Spawn::STARTER)
|
|
96
117
|
before = cpu
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
118
|
+
begin
|
|
119
|
+
child = starter.logged(env, command, log: log, mode: "w", chdir: chdir, unsetenv_others: true, close_others: true)
|
|
120
|
+
rescue SystemCallError => e
|
|
121
|
+
return {"cpu" => 0, "peak" => nil, "failed" => "couldn't start: #{e}"}
|
|
122
|
+
end
|
|
123
|
+
spawned = Native::Windows::Spawned.open(child.pid) if windows
|
|
124
|
+
begin
|
|
125
|
+
code, signal, peak = child.wait
|
|
126
|
+
if spawned
|
|
127
|
+
taken = CycleClock.ns(spawned.cycles)
|
|
128
|
+
peak = spawned.peak
|
|
129
|
+
else
|
|
130
|
+
taken = cpu - before
|
|
131
|
+
end
|
|
132
|
+
{"cpu" => taken, "peak" => peak, "failed" => (Spawn.why(code, signal) unless code == 0)} #: ended
|
|
133
|
+
ensure
|
|
134
|
+
spawned&.close
|
|
135
|
+
end
|
|
100
136
|
end
|
|
101
137
|
|
|
102
138
|
# The CPU time of the children waited for so far, in nanoseconds.
|
|
@@ -105,40 +141,29 @@ module Duel
|
|
|
105
141
|
((times.cutime + times.cstime) * 1_000_000_000).round
|
|
106
142
|
end
|
|
107
143
|
|
|
108
|
-
# How a process that failed ended: its exit status +code+, or the
|
|
109
|
-
# +signal+ that killed it.
|
|
110
|
-
def self.why(code, signal)
|
|
111
|
-
return "exited with #{code}" if code
|
|
112
|
-
|
|
113
|
-
killed = signal #: Integer
|
|
114
|
-
"was killed by SIG#{Signal.signame(killed)}"
|
|
115
|
-
end
|
|
116
|
-
|
|
117
144
|
private
|
|
118
145
|
|
|
119
|
-
# Starts the Ruby process of its own,
|
|
120
|
-
#
|
|
121
|
-
#
|
|
146
|
+
# Starts the Ruby process of its own, connected by a pipe each way
|
|
147
|
+
# (not a socket: JRuby's spawn can't give one), and gives it the
|
|
148
|
+
# environment, the directory and the log, as the first line of its
|
|
149
|
+
# input. (Not an argument: JRuby's spawn on Windows splits one at its
|
|
150
|
+
# spaces, and the environment can be longer than Windows lets a
|
|
151
|
+
# command be.)
|
|
122
152
|
def start
|
|
123
|
-
spec = JSON.generate({"env" => {**ENV, **@env}, "chdir" => @chdir, "log" => @log})
|
|
124
|
-
requests, @requests = IO.pipe
|
|
125
|
-
@replies, replies = IO.pipe
|
|
126
153
|
# (Out of the runner's bundle: it needs nothing of it.)
|
|
127
|
-
@
|
|
128
|
-
*WorkerProcess.load_path(RbConfig.ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/process_sample", "-e",
|
|
129
|
-
|
|
130
|
-
requests.
|
|
131
|
-
replies.close
|
|
154
|
+
@child, @requests, @replies = @starter.piped({"RUBYOPT" => nil, "RUBYLIB" => nil},
|
|
155
|
+
[RbConfig.ruby, *WorkerProcess.load_path(RbConfig.ruby).flat_map { |lib| ["-I", lib] }, "-r", "duel/process_sample", "-e",
|
|
156
|
+
"Duel::ProcessSample.main"], group: true, close_others: true)
|
|
157
|
+
@requests.puts JSON.generate({"env" => {**ENV, **@env}, "chdir" => @chdir, "log" => @log})
|
|
132
158
|
end
|
|
133
159
|
|
|
134
160
|
# Kills the Ruby process of its own, and what it started.
|
|
135
161
|
def kill
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
Process.wait(pid)
|
|
162
|
+
child = @child #: Spawn::Child
|
|
163
|
+
child.kill
|
|
139
164
|
@requests.close
|
|
140
165
|
@replies.close
|
|
141
|
-
@
|
|
166
|
+
@child = nil
|
|
142
167
|
end
|
|
143
168
|
end
|
|
144
169
|
end
|
data/lib/duel/resource.rb
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
require "fileutils"
|
|
4
4
|
require "rbconfig"
|
|
5
|
+
require_relative "control"
|
|
6
|
+
require_relative "spawn"
|
|
5
7
|
|
|
6
8
|
module Duel
|
|
7
9
|
# What a prepare block (see Config#prepare) gets: a worker's, or a
|
|
@@ -29,10 +31,6 @@ module Duel
|
|
|
29
31
|
# @api private
|
|
30
32
|
STOP_WAIT = 5
|
|
31
33
|
|
|
32
|
-
# The lines of its log a failure shows.
|
|
33
|
-
# @api private
|
|
34
|
-
LOG_LINES = 10
|
|
35
|
-
|
|
36
34
|
# What a cache key may be.
|
|
37
35
|
# @api private
|
|
38
36
|
KEY = /\A[\w.-]+\z/
|
|
@@ -71,6 +69,26 @@ module Duel
|
|
|
71
69
|
nil
|
|
72
70
|
end
|
|
73
71
|
|
|
72
|
+
# A process #spawn started: its program (what its log is named for),
|
|
73
|
+
# its log, the signal that stops it, and its Spawn::Child.
|
|
74
|
+
# @api private
|
|
75
|
+
Spawned = Data.define(:program, :log, :stop_signal, :child)
|
|
76
|
+
|
|
77
|
+
# Runs each of +blocks+ (Config#prepare's), in order, each given a
|
|
78
|
+
# Resource of its own (+dir+-1, +dir+-2...) for +name+ in +root+, on
|
|
79
|
+
# +ruby+, which it gives the block +kept+ (to stop it later). Returns
|
|
80
|
+
# +env+ with what each block returned (see .environment). Each
|
|
81
|
+
# Resource's own env is +env+: a worker's, its version's alone, but a
|
|
82
|
+
# version's (+so_far+), with what the blocks before it returned too.
|
|
83
|
+
# @api private
|
|
84
|
+
def self.prepare(blocks, dir, name, root, env:, ruby:, timeout:, cache:, so_far: false)
|
|
85
|
+
blocks.each_with_index.reduce(env) do |set, (block, i)|
|
|
86
|
+
resource = new("#{dir}-#{i + 1}", name, root, timeout:, cache:, ruby:, env: so_far ? set : env)
|
|
87
|
+
yield resource
|
|
88
|
+
environment(name, set, block.call(resource))
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
74
92
|
# The worker it's for ("base-1", say), or the version ("base",
|
|
75
93
|
# for Config#prepare(:version)), the directory it has of its own, the
|
|
76
94
|
# tree of the version that worker runs, and the Ruby it runs on (see
|
|
@@ -93,8 +111,10 @@ module Duel
|
|
|
93
111
|
end
|
|
94
112
|
end
|
|
95
113
|
|
|
114
|
+
# (+starter+: how its processes start, Spawn::Directly or
|
|
115
|
+
# Spawn::ThroughOpen3.)
|
|
96
116
|
# @api private
|
|
97
|
-
def initialize(dir, name, root, timeout:, cache:, ruby: RbConfig.ruby, env: {})
|
|
117
|
+
def initialize(dir, name, root, timeout:, cache:, ruby: RbConfig.ruby, env: {}, starter: Spawn::STARTER)
|
|
98
118
|
@name = name
|
|
99
119
|
@dir = dir
|
|
100
120
|
@root = root
|
|
@@ -102,7 +122,8 @@ module Duel
|
|
|
102
122
|
@env = env
|
|
103
123
|
@timeout = timeout
|
|
104
124
|
@cache = cache
|
|
105
|
-
@
|
|
125
|
+
@starter = starter
|
|
126
|
+
@processes = []
|
|
106
127
|
@stops = []
|
|
107
128
|
FileUtils.mkdir_p(dir)
|
|
108
129
|
end
|
|
@@ -111,6 +132,18 @@ module Duel
|
|
|
111
132
|
# bytes on macOS. Duel's directories leave room.)
|
|
112
133
|
def path(file) = File.join(@dir, file)
|
|
113
134
|
|
|
135
|
+
# A port of 127.0.0.1's that's free (one the system chose, let go of
|
|
136
|
+
# again), for a server of its own to listen on, rather than on a Unix
|
|
137
|
+
# socket in its directory (see #path).
|
|
138
|
+
def port
|
|
139
|
+
server = Control.tcp_server
|
|
140
|
+
begin
|
|
141
|
+
server.addr.fetch(1)
|
|
142
|
+
ensure
|
|
143
|
+
server.close
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
114
147
|
# A directory built once, by the block, for every worker of this run
|
|
115
148
|
# and later ones (a database's initial files, say, for each worker to
|
|
116
149
|
# copy): the first that asks for +key+ builds it, in a directory the
|
|
@@ -134,15 +167,17 @@ module Duel
|
|
|
134
167
|
# Starts a process (as Process.spawn takes it), its output added to a
|
|
135
168
|
# log in the directory (+log+.log: by default, its program's name),
|
|
136
169
|
# and stopped with the resource, by +stop_signal+ (then KILL, five
|
|
137
|
-
# seconds later).
|
|
170
|
+
# seconds later). On Windows, which can't send another process a
|
|
171
|
+
# signal, it's killed: ask it to stop in an #at_stop block, which
|
|
172
|
+
# runs first. Returns its pid. Its input is
|
|
138
173
|
# nothing, and it gets no other file descriptors, unless +options+
|
|
139
174
|
# say otherwise (in:, close_others:). It runs in its version's
|
|
140
175
|
# environment (on its Ruby: see Config#ruby), under any it's given.
|
|
141
176
|
def spawn(*command, log: nil, stop_signal: "TERM", **options)
|
|
142
177
|
log = log_path(command, log)
|
|
143
|
-
|
|
144
|
-
@processes
|
|
145
|
-
pid
|
|
178
|
+
child = start(command, log, options)
|
|
179
|
+
@processes << Spawned.new(program: File.basename(log, ".log"), log:, stop_signal:, child:)
|
|
180
|
+
child.pid
|
|
146
181
|
end
|
|
147
182
|
|
|
148
183
|
# Runs a command (setting something up, say) to its end, its output
|
|
@@ -150,12 +185,10 @@ module Duel
|
|
|
150
185
|
# succeeds.
|
|
151
186
|
def run(*command, log: nil, **options)
|
|
152
187
|
log = log_path(command, log)
|
|
153
|
-
|
|
154
|
-
return if
|
|
188
|
+
code, signal = start(command, log, options).wait
|
|
189
|
+
return if code == 0
|
|
155
190
|
|
|
156
|
-
code
|
|
157
|
-
signal = status.termsig #: Integer
|
|
158
|
-
failed("#{File.basename(log, ".log")} #{code ? "exited with #{code}" : "was killed by SIG#{Signal.signame(signal)}"}", log: log)
|
|
191
|
+
failed("#{File.basename(log, ".log")} #{Spawn.why(code, signal)}", log: log)
|
|
159
192
|
end
|
|
160
193
|
|
|
161
194
|
# Waits until the block is true: fails if a process spawned dies
|
|
@@ -163,15 +196,14 @@ module Duel
|
|
|
163
196
|
def wait_until
|
|
164
197
|
deadline = monotonic + @timeout
|
|
165
198
|
until yield
|
|
166
|
-
ended = @processes.find { |
|
|
199
|
+
ended = @processes.find { |process| !process.child.alive? }
|
|
167
200
|
if ended
|
|
168
201
|
# (It may have got ready, and ended, since the last look.)
|
|
169
202
|
break if yield
|
|
170
203
|
|
|
171
|
-
program, log
|
|
172
|
-
failed("#{program} exited before it was ready", log: log)
|
|
204
|
+
failed("#{ended.program} exited before it was ready", log: ended.log)
|
|
173
205
|
end
|
|
174
|
-
failed("not ready within #{@timeout}s", log: @processes.
|
|
206
|
+
failed("not ready within #{@timeout}s", log: @processes.last&.log) if monotonic > deadline
|
|
175
207
|
sleep POLL
|
|
176
208
|
end
|
|
177
209
|
end
|
|
@@ -182,7 +214,7 @@ module Duel
|
|
|
182
214
|
# Fails, saying it's this resource that did, with the end of +log+ (a
|
|
183
215
|
# path) to say why.
|
|
184
216
|
def failed(message, log: nil)
|
|
185
|
-
tail = log ?
|
|
217
|
+
tail = log ? Spawn.tail(log) : ""
|
|
186
218
|
raise Failed, "#{@name}'s resource: #{message}#{"\n#{tail}" unless tail.empty?}"
|
|
187
219
|
end
|
|
188
220
|
|
|
@@ -190,7 +222,10 @@ module Duel
|
|
|
190
222
|
# @api private
|
|
191
223
|
def stop
|
|
192
224
|
@stops.reverse_each(&:call)
|
|
193
|
-
@processes.each
|
|
225
|
+
@processes.each do |process|
|
|
226
|
+
process.child.signal(process.stop_signal)
|
|
227
|
+
process.child.stop(STOP_WAIT)
|
|
228
|
+
end
|
|
194
229
|
end
|
|
195
230
|
|
|
196
231
|
private
|
|
@@ -211,32 +246,26 @@ module Duel
|
|
|
211
246
|
end
|
|
212
247
|
end
|
|
213
248
|
|
|
214
|
-
# The log a command's output goes to: +name+.log, or its program's
|
|
215
|
-
|
|
249
|
+
# The log a command's output goes to: +name+.log, or its program's:
|
|
250
|
+
# a command line's first word (a String alone, which a shell runs), or
|
|
251
|
+
# else the program given its arguments, whole, as its path may have a
|
|
252
|
+
# space in it (C:/Program Files/PostgreSQL/17/bin/initdb, say).
|
|
253
|
+
def log_path(command, name)
|
|
254
|
+
words = command.grep(String)
|
|
255
|
+
program = words.fetch(0)
|
|
256
|
+
program = program.split.fetch(0) if words.size == 1
|
|
257
|
+
path("#{name || File.basename(program)}.log")
|
|
258
|
+
end
|
|
216
259
|
|
|
260
|
+
# Starts +command+, its output added to +log+, and returns its
|
|
261
|
+
# Spawn::Child, waited for on a thread from the start (see
|
|
262
|
+
# Child#detach), so it's reaped as it ends. (On a thread: a Timeout
|
|
263
|
+
# can't interrupt a wait on JRuby.)
|
|
217
264
|
def start(command, log, options)
|
|
218
265
|
# (Its own environment, if it's given one, over its version's.)
|
|
219
266
|
none = {} #: Hash[String, String?]
|
|
220
267
|
env, *program = command.first.is_a?(Hash) ? command : [none, *command]
|
|
221
|
-
|
|
222
|
-
begin
|
|
223
|
-
Process.spawn(@env.merge(env), *program, in: File::NULL, out: out, err: out, close_others: true, **options)
|
|
224
|
-
ensure
|
|
225
|
-
out.close
|
|
226
|
-
end
|
|
227
|
-
end
|
|
228
|
-
|
|
229
|
-
# (Waited for on a thread of its own: a Timeout can't interrupt a wait
|
|
230
|
-
# on JRuby.)
|
|
231
|
-
def end_process(pid, signal)
|
|
232
|
-
Process.kill(signal, pid)
|
|
233
|
-
waiter = Process.detach(pid)
|
|
234
|
-
return if waiter.join(STOP_WAIT)
|
|
235
|
-
|
|
236
|
-
Process.kill("KILL", pid)
|
|
237
|
-
waiter.join
|
|
238
|
-
rescue Errno::ESRCH # gone already
|
|
239
|
-
nil
|
|
268
|
+
@starter.logged(@env.merge(env), program, log: log, mode: "a", close_others: true, **options).detach
|
|
240
269
|
end
|
|
241
270
|
|
|
242
271
|
def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
data/lib/duel/result.rb
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../duel"
|
|
4
|
+
|
|
5
|
+
module Duel
|
|
6
|
+
# What one comparison of a benchmark found (Testing::Comparison#results
|
|
7
|
+
# has them): its #name, whether it #passed?, the measures it #regressed
|
|
8
|
+
# on, each #timed measure's verdict and each #counted one's counts,
|
|
9
|
+
# what metrics said of it (#details), its peak #memory, its iterations
|
|
10
|
+
# a sample (#n), the #seconds it took, why it was #skipped or its
|
|
11
|
+
# #error, and its #confirmations (the measurements that confirmed it
|
|
12
|
+
# was worse, not those too close to call: see #confirm). Its writers,
|
|
13
|
+
# and the rest, are @api private.
|
|
14
|
+
class Result
|
|
15
|
+
# Peak memory grows by more than MEMORY_RATIO, and by MEMORY_FLOOR
|
|
16
|
+
# bytes at least, before it's worse (it's measured as a whole, with the
|
|
17
|
+
# garbage collector working, so it isn't exact).
|
|
18
|
+
# @api private
|
|
19
|
+
MEMORY_RATIO = 1.1
|
|
20
|
+
|
|
21
|
+
# @api private
|
|
22
|
+
MEMORY_FLOOR = 1 << 20
|
|
23
|
+
|
|
24
|
+
# What a sample says when no metric said anything (and what a measure
|
|
25
|
+
# it said nothing of did).
|
|
26
|
+
# @api private
|
|
27
|
+
NOTHING_SAID = {} #: Hash[String, String]
|
|
28
|
+
|
|
29
|
+
# The verdicts that say a timing isn't slower.
|
|
30
|
+
# @api private
|
|
31
|
+
CLEARED = %w[same faster].freeze
|
|
32
|
+
|
|
33
|
+
attr_reader :name, :timed, :counted, :details, :confirmations, :n, :seconds, :skipped, :error, :memory
|
|
34
|
+
|
|
35
|
+
# Where a regression's time went (see Profile).
|
|
36
|
+
# @api private
|
|
37
|
+
attr_reader :profile
|
|
38
|
+
|
|
39
|
+
# Each worker's median of each measure judged by process (see
|
|
40
|
+
# Lane#judge_processes): {metric => {"lane" => its lane's number,
|
|
41
|
+
# "baseline" => [each baseline worker's], "candidate" => [...]}},
|
|
42
|
+
# for across the board.
|
|
43
|
+
# @api private
|
|
44
|
+
attr_reader :processes
|
|
45
|
+
|
|
46
|
+
# The measurements again that were too close to call (see #confirm).
|
|
47
|
+
# @api private
|
|
48
|
+
attr_reader :undecided
|
|
49
|
+
|
|
50
|
+
# @api private
|
|
51
|
+
attr_writer :n, :seconds, :skipped, :error, :memory, :profile
|
|
52
|
+
|
|
53
|
+
# @api private
|
|
54
|
+
def initialize(name)
|
|
55
|
+
@name = name
|
|
56
|
+
@timed = {}
|
|
57
|
+
@counted = {}
|
|
58
|
+
@details = {}
|
|
59
|
+
@confirmations = []
|
|
60
|
+
@undecided = []
|
|
61
|
+
@processes = {}
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Keeps what metrics said of a pair of samples (Metric#details): each
|
|
65
|
+
# measure's latest, by version (only +measure+'s, if given).
|
|
66
|
+
# @api private
|
|
67
|
+
def note(base, candidate, measure = nil)
|
|
68
|
+
{"baseline" => base, "candidate" => candidate}.each do |side, sample|
|
|
69
|
+
said = sample["details"] || NOTHING_SAID
|
|
70
|
+
said = said.slice(measure) if measure
|
|
71
|
+
said.each { |name, text| (@details[name] ||= {}).store(side, text) }
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# The metrics on which the candidate did worse: a timed one slower, or
|
|
76
|
+
# a counted one up by more than its tolerance per iteration and its
|
|
77
|
+
# allowance per sample (counted by process, up by that much at least:
|
|
78
|
+
# see Lane#count_by_process).
|
|
79
|
+
# @api private
|
|
80
|
+
def worse
|
|
81
|
+
more = @counted.select do |_, c|
|
|
82
|
+
!Metric.within?(c["more"] || (c.fetch("candidate") - c.fetch("baseline")), c.fetch("n"), tolerance: c.fetch("tolerance"),
|
|
83
|
+
allowance: c.fetch("allowance"))
|
|
84
|
+
end
|
|
85
|
+
slower + more.keys + (more_memory? ? ["peak_memory"] : [])
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# The timed metrics on which the candidate was slower.
|
|
89
|
+
# @api private
|
|
90
|
+
def slower = @timed.filter_map { |metric, t| metric if t.fetch("verdict") == "slower" }
|
|
91
|
+
|
|
92
|
+
# Whether its timings say slower: one is, and none says it isn't
|
|
93
|
+
# (same, or faster).
|
|
94
|
+
# @api private
|
|
95
|
+
def slower_in_time? = slower.any? && @timed.none? { |_, t| CLEARED.include?(t.fetch("verdict")) }
|
|
96
|
+
|
|
97
|
+
# Whether it's to be measured again (see Runner#confirm): it looks
|
|
98
|
+
# worse in every measurement so far, and there are fewer confirmations
|
|
99
|
+
# than it takes to fail (Runner::CONFIRM, or one more where it's
|
|
100
|
+
# #mixed?).
|
|
101
|
+
# @api private
|
|
102
|
+
def confirming?
|
|
103
|
+
return false if regressed.empty?
|
|
104
|
+
|
|
105
|
+
@confirmations.size < (mixed? ? Runner::CONFIRM + 1 : Runner::CONFIRM)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Keeps +again+, a measurement of it again from scratch (see
|
|
109
|
+
# Runner#confirm), as a confirmation, or sets it aside where it's too
|
|
110
|
+
# close to call, up to Runner::CONFIRM times: every timing is still
|
|
111
|
+
# undecided there (none slower, none the same or faster), and each
|
|
112
|
+
# measure it regressed on so far is one of them. One that can't tell
|
|
113
|
+
# neither confirms it nor clears it, so it's measured again instead.
|
|
114
|
+
# Returns whether +again+ was set aside.
|
|
115
|
+
# @api private
|
|
116
|
+
def confirm(again)
|
|
117
|
+
timed = again.timed
|
|
118
|
+
close = timed.all? { |_, t| t.fetch("verdict") == "undecided" } && regressed.all? { |metric| timed.key?(metric) } &&
|
|
119
|
+
@undecided.size < Runner::CONFIRM
|
|
120
|
+
(close ? @undecided : @confirmations) << again
|
|
121
|
+
close
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# Whether its peak memory grew by too much (see MEMORY_RATIO).
|
|
125
|
+
# @api private
|
|
126
|
+
def more_memory?
|
|
127
|
+
memory = @memory
|
|
128
|
+
return false unless memory
|
|
129
|
+
|
|
130
|
+
base = memory.fetch("baseline")
|
|
131
|
+
memory.fetch("candidate") - base > [MEMORY_FLOOR, base * (MEMORY_RATIO - 1)].max
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Worse on a metric in the first measurement and in every confirmation.
|
|
135
|
+
# Or, where no one metric is, slower in time in all of them (see
|
|
136
|
+
# #mixed?): the timed metrics slower in any.
|
|
137
|
+
def regressed
|
|
138
|
+
return [] if @error || @skipped
|
|
139
|
+
|
|
140
|
+
agreed = measurements.map(&:worse).reduce(:&)
|
|
141
|
+
return agreed unless agreed.empty? && measurements.all?(&:slower_in_time?)
|
|
142
|
+
|
|
143
|
+
measurements.flat_map(&:slower).uniq
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Whether it regressed in time on one timed metric in some
|
|
147
|
+
# measurements and on another in the rest (CPU time in one,
|
|
148
|
+
# instructions in the next, say), slower in every one with none saying
|
|
149
|
+
# it isn't: that takes one measurement more than one metric does.
|
|
150
|
+
# @api private
|
|
151
|
+
def mixed? = measurements.map(&:worse).reduce(:&).empty? && !regressed.empty?
|
|
152
|
+
|
|
153
|
+
# Whether it regressed on nothing, and didn't fail.
|
|
154
|
+
def passed? = regressed.empty? && !@error
|
|
155
|
+
|
|
156
|
+
# Whether a timed measure's verdict was too close to the limit to settle.
|
|
157
|
+
# @api private
|
|
158
|
+
def unsettled? = @timed.any? { |_, t| t.fetch("verdict") == "undecided" }
|
|
159
|
+
|
|
160
|
+
private
|
|
161
|
+
|
|
162
|
+
# The first measurement and its confirmations.
|
|
163
|
+
def measurements = [self, *@confirmations]
|
|
164
|
+
end
|
|
165
|
+
end
|
data/lib/duel/rubies.rb
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require "open3"
|
|
4
4
|
require "rbconfig"
|
|
5
5
|
require_relative "../duel"
|
|
6
|
+
require_relative "platform"
|
|
7
|
+
require_relative "spawn"
|
|
6
8
|
|
|
7
9
|
module Duel
|
|
8
10
|
# The Ruby each version runs on (Config#ruby): by default the runner's.
|
|
@@ -70,8 +72,9 @@ module Duel
|
|
|
70
72
|
def engine(ruby)
|
|
71
73
|
return RUBY_ENGINE if runner?(ruby)
|
|
72
74
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
+
# (Its errors read, not shown: JRuby's IO.popen on Windows ignores
|
|
76
|
+
# err:, https://github.com/jruby/jruby/issues/10081.)
|
|
77
|
+
out, _, status = Spawn.open3(:capture3, environment(ruby).merge("RUBYOPT" => nil), ruby, "--disable-gems", "-e", "print RUBY_ENGINE")
|
|
75
78
|
status.success? ? out : "ruby"
|
|
76
79
|
rescue SystemCallError
|
|
77
80
|
"ruby"
|
|
@@ -92,6 +95,21 @@ module Duel
|
|
|
92
95
|
# Whether +ruby+ is the runner's own.
|
|
93
96
|
def runner?(ruby) = File.realpath(ruby) == File.realpath(RbConfig.ruby)
|
|
94
97
|
|
|
98
|
+
# The command that starts +ruby+ (whose engine is +engine+), with
|
|
99
|
+
# +env+: +ruby+ itself, but on JRuby on Windows, whose jruby.exe
|
|
100
|
+
# starts the JVM in a process of its own (javaw.exe), the JVM's
|
|
101
|
+
# command, as `jruby -Xcommand` says it is: so the process Duel
|
|
102
|
+
# measures, waits for, and kills is the one running Ruby.
|
|
103
|
+
def launch(ruby, engine, env, windows: Platform::WINDOWS)
|
|
104
|
+
return [ruby] unless windows && engine == "jruby"
|
|
105
|
+
|
|
106
|
+
out, status = Spawn.open3(:capture2, env, ruby, "-Xcommand", "-e", "1")
|
|
107
|
+
args = out.lines(chomp: true)
|
|
108
|
+
(status.success? && args.last(2) == %w[-e 1]) ? args.first(args.size - 2) : [ruby]
|
|
109
|
+
rescue SystemCallError
|
|
110
|
+
[ruby]
|
|
111
|
+
end
|
|
112
|
+
|
|
95
113
|
# What a version's processes on +ruby+ get in their environment:
|
|
96
114
|
# nothing on the runner's Ruby. On another, its directory first on
|
|
97
115
|
# +path+ (for what they start by name, `bundle` say), and none of the
|
|
@@ -154,7 +172,7 @@ module Duel
|
|
|
154
172
|
|
|
155
173
|
# What a command prints, or nil if it fails (or isn't there).
|
|
156
174
|
def command(*command)
|
|
157
|
-
out, _, status =
|
|
175
|
+
out, _, status = Spawn.open3(:capture3, *command)
|
|
158
176
|
out.strip if status.success?
|
|
159
177
|
rescue SystemCallError
|
|
160
178
|
nil
|