duel 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +2 -2
- data/CHANGELOG.md +203 -1
- data/README.md +41 -1
- data/docs/ci.md +170 -76
- data/docs/configuration.md +38 -2
- data/docs/how-it-works.md +18 -9
- data/docs/plugins.md +23 -0
- data/lib/duel/across.rb +1 -6
- data/lib/duel/baseline.rb +8 -7
- data/lib/duel/cli.rb +96 -28
- data/lib/duel/comment.rb +296 -0
- data/lib/duel/config.rb +124 -4
- data/lib/duel/context.rb +33 -34
- data/lib/duel/control.rb +52 -0
- data/lib/duel/init.rb +255 -20
- data/lib/duel/lane.rb +734 -0
- data/lib/duel/lock.rb +34 -4
- data/lib/duel/memory.rb +14 -5
- data/lib/duel/metric.rb +97 -26
- data/lib/duel/native.rb +199 -32
- data/lib/duel/platform.rb +31 -0
- data/lib/duel/process_sample.rb +74 -49
- data/lib/duel/resource.rb +71 -42
- data/lib/duel/result.rb +165 -0
- data/lib/duel/rubies.rb +21 -3
- data/lib/duel/runner.rb +33 -1044
- data/lib/duel/spawn.rb +327 -0
- data/lib/duel/summary.rb +21 -15
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +8 -19
- data/lib/duel/templates/perf-caller.yml.erb +43 -0
- data/lib/duel/templates/perf-comment-caller.yml.erb +42 -0
- data/lib/duel/templates/perf-comment.yml.erb +61 -44
- data/lib/duel/templates/perf.yml.erb +143 -7
- data/lib/duel/testing.rb +3 -0
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +13 -4
- data/lib/duel/worker_process.rb +142 -0
- data/lib/duel.rb +35 -3
- data/sig/duel/cli.rbs +19 -0
- data/sig/duel/comment.rbs +97 -0
- data/sig/duel/config.rbs +57 -2
- data/sig/duel/context.rbs +4 -31
- data/sig/duel/control.rbs +27 -0
- data/sig/duel/init.rbs +95 -1
- data/sig/duel/lane.rbs +202 -0
- data/sig/duel/lock.rbs +8 -1
- data/sig/duel/memory.rbs +1 -1
- data/sig/duel/metric.rbs +53 -10
- data/sig/duel/native.rbs +109 -9
- data/sig/duel/platform.rbs +13 -0
- data/sig/duel/process_sample.rbs +7 -9
- data/sig/duel/resource.rbs +28 -9
- data/sig/duel/result.rbs +164 -0
- data/sig/duel/rubies.rbs +4 -0
- data/sig/duel/runner.rbs +6 -460
- data/sig/duel/spawn.rbs +111 -0
- data/sig/duel/summary.rbs +6 -3
- data/sig/duel/worker_process.rbs +90 -0
- data/sig/duel.rbs +17 -3
- metadata +121 -105
data/lib/duel/spawn.rb
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "open3"
|
|
4
|
+
require_relative "native"
|
|
5
|
+
require_relative "platform"
|
|
6
|
+
|
|
7
|
+
module Duel
|
|
8
|
+
# Starts a process whose output goes to a log, or whose input and
|
|
9
|
+
# output are pipes, and gives the Child to wait for it and stop it by:
|
|
10
|
+
# with Process.spawn's redirects (Directly), or where spawn can't
|
|
11
|
+
# redirect or wait (JRuby on Windows), through Open3 (ThroughOpen3),
|
|
12
|
+
# the output copied to the log on a thread of its own. STARTER is the
|
|
13
|
+
# one that works here. (JRuby's Open3 there does what it's asked, but
|
|
14
|
+
# with the JVM's own environment, not ENV, and only a program that's an
|
|
15
|
+
# .exe found by name: see .open3.)
|
|
16
|
+
# @api private
|
|
17
|
+
module Spawn
|
|
18
|
+
# Whether Process.spawn redirects a process's input and output: not on
|
|
19
|
+
# JRuby on Windows, whose spawn writes to neither a file nor a pipe
|
|
20
|
+
# it's given (https://github.com/jruby/jruby/issues/3116).
|
|
21
|
+
REDIRECTS = !Platform::JRUBY_ON_WINDOWS
|
|
22
|
+
|
|
23
|
+
# Whether Process.wait waits for a process spawned: not on JRuby on
|
|
24
|
+
# Windows, where it returns at once, and so does Process.detach's
|
|
25
|
+
# thread (https://github.com/jruby/jruby/issues/7515).
|
|
26
|
+
WAITS = !Platform::JRUBY_ON_WINDOWS
|
|
27
|
+
|
|
28
|
+
# A lone String, as Process.spawn takes one, that a shell runs.
|
|
29
|
+
SHELL = /[\s"'*?<>|&;$`()]/
|
|
30
|
+
|
|
31
|
+
# Seconds the copy of a process's output to its log may go on once
|
|
32
|
+
# the process has ended: no longer, where a process it started (a
|
|
33
|
+
# server's daemon, say) keeps the output open.
|
|
34
|
+
COPY_WAIT = 5
|
|
35
|
+
|
|
36
|
+
# The most of a process's output copied to its log at a time.
|
|
37
|
+
CHUNK = 65_536
|
|
38
|
+
|
|
39
|
+
# The lines of a process's log a failure shows (see .tail).
|
|
40
|
+
LOG_LINES = 10
|
|
41
|
+
|
|
42
|
+
# A process started, to wait for and stop.
|
|
43
|
+
class Child
|
|
44
|
+
# What Process.kill raises for a process that's gone: ESRCH, or on
|
|
45
|
+
# JRuby on Windows, EPERM.
|
|
46
|
+
GONE = [Errno::ESRCH, Errno::EPERM].freeze
|
|
47
|
+
|
|
48
|
+
attr_reader :pid
|
|
49
|
+
|
|
50
|
+
# +pid+'s, waited for by +waiter+, a thread whose value is how it
|
|
51
|
+
# ended (a Process::Status), or for one spawned, by Process.detach's,
|
|
52
|
+
# once it's asked for (see #waiter and #detach). +group+: whether it
|
|
53
|
+
# was started in a group of its own (see Spawn.group), which #kill
|
|
54
|
+
# kills with it. (+windows+: whether it's on Windows, where it can
|
|
55
|
+
# only be killed: see #signal.)
|
|
56
|
+
def initialize(pid, waiter = nil, group: false, windows: Platform::WINDOWS)
|
|
57
|
+
@pid = pid
|
|
58
|
+
@waiter = waiter
|
|
59
|
+
@group = group
|
|
60
|
+
@windows = windows
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# The thread whose value is how it ended (a Process::Status). For one
|
|
64
|
+
# spawned, Process.detach's, made once it's asked for: until then,
|
|
65
|
+
# #wait waits for it itself.
|
|
66
|
+
def waiter = @waiter ||= Process.detach(@pid)
|
|
67
|
+
|
|
68
|
+
# Waits for it on a thread from now on (see #waiter), so that it's
|
|
69
|
+
# reaped as it ends: returns itself.
|
|
70
|
+
def detach = tap { waiter }
|
|
71
|
+
|
|
72
|
+
def alive? = waiter.alive?
|
|
73
|
+
|
|
74
|
+
# Waits for it, and returns how it ended, as Native::Child.wait says:
|
|
75
|
+
# its exit status (or nil), the signal that killed it (or nil), and
|
|
76
|
+
# its peak memory, where it was spawned and nothing else waits for it
|
|
77
|
+
# (see #waiter), and that can be told (or nil).
|
|
78
|
+
def wait
|
|
79
|
+
waiter = @waiter
|
|
80
|
+
waiter ? Native::Child.ended(waiter.value) : Native::Child.wait(@pid)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Sends it +signal+. On Windows, which can send another process no
|
|
84
|
+
# signal but KILL, kills it, with what it started (taskkill /T): a
|
|
85
|
+
# process it started would go on otherwise (JRuby's JVM, started by
|
|
86
|
+
# its jruby.exe, say). One that's gone is let be.
|
|
87
|
+
def signal(signal)
|
|
88
|
+
@windows ? Spawn.kill_tree(@pid, windows: true) : Process.kill(signal, @pid)
|
|
89
|
+
rescue *GONE
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Kills it, with what it started where that can be told (its group,
|
|
94
|
+
# or on Windows, its tree: see Spawn.kill_tree), and waits for it to
|
|
95
|
+
# end. One that's gone is let be.
|
|
96
|
+
def kill
|
|
97
|
+
(@group || @windows) ? Spawn.kill_tree(@pid, windows: @windows) : Process.kill("KILL", @pid)
|
|
98
|
+
waiter.join
|
|
99
|
+
rescue *GONE
|
|
100
|
+
nil
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Waits up to +seconds+ for it to end, then kills it (see #kill).
|
|
104
|
+
def stop(seconds) = (kill unless waiter.join(seconds))
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Starts a process with Process.spawn, given its redirects and pipes.
|
|
108
|
+
module Directly
|
|
109
|
+
# Starts +command+ (as Process.spawn takes it, an environment first),
|
|
110
|
+
# its input nothing and its output and errors in +log+, opened with
|
|
111
|
+
# +mode+ ("a" or "w"), with +options+ (Process.spawn's: chdir:, say).
|
|
112
|
+
# Returns its Child, not yet waited for, so that Child#wait can wait
|
|
113
|
+
# for it as any child, with wait4 (or Child#detach, on a thread).
|
|
114
|
+
def self.logged(env, command, log:, mode:, **options)
|
|
115
|
+
out = Spawn.log(log, mode)
|
|
116
|
+
# (At its end: on Windows, a child given a file opened to append
|
|
117
|
+
# writes from where the file's at, which is its start.)
|
|
118
|
+
out.seek(0, IO::SEEK_END)
|
|
119
|
+
begin
|
|
120
|
+
Child.new(Process.spawn(env, *command, in: File::NULL, out: out, err: out, **options))
|
|
121
|
+
ensure
|
|
122
|
+
out.close
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Starts +command+ (as Process.spawn takes it, an environment first)
|
|
127
|
+
# with a pipe each way, and +options+, in a group of its own if
|
|
128
|
+
# +group+ (see Spawn.group). Returns its Child, its input and its
|
|
129
|
+
# output.
|
|
130
|
+
def self.piped(env, command, group: false, **options)
|
|
131
|
+
options = options.merge(Spawn.group) if group
|
|
132
|
+
input, requests = IO.pipe
|
|
133
|
+
replies, output = IO.pipe
|
|
134
|
+
pid = begin
|
|
135
|
+
Process.spawn(env, *command, in: input, out: output, **options)
|
|
136
|
+
ensure
|
|
137
|
+
input.close
|
|
138
|
+
output.close
|
|
139
|
+
end
|
|
140
|
+
[Child.new(pid, Process.detach(pid), group: group), requests, replies]
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# Starts +command+ (as Process.spawn takes it, an environment first),
|
|
144
|
+
# its input nothing, and its output and errors this process's, with
|
|
145
|
+
# +options+. Returns its Child.
|
|
146
|
+
def self.started(env, command, **options)
|
|
147
|
+
pid = Process.spawn(env, *command, in: File::NULL, **options)
|
|
148
|
+
Child.new(pid, Process.detach(pid))
|
|
149
|
+
end
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Starts a process through Open3, as Directly does, where
|
|
153
|
+
# Process.spawn can't redirect or wait: what it says is copied to its
|
|
154
|
+
# log (or to this process's output) on a thread of its own, and its
|
|
155
|
+
# Child's thread (Child#waiter) is Open3's, its value once what it
|
|
156
|
+
# said is copied (see Spawn.ended).
|
|
157
|
+
module ThroughOpen3
|
|
158
|
+
def self.logged(env, command, log:, mode:, **options)
|
|
159
|
+
out = Spawn.log(log, mode)
|
|
160
|
+
input, output, waiter = begin
|
|
161
|
+
Spawn.open3(:popen2e, env, *command, **options)
|
|
162
|
+
rescue SystemCallError
|
|
163
|
+
out.close
|
|
164
|
+
raise
|
|
165
|
+
end
|
|
166
|
+
input.close
|
|
167
|
+
Child.new(waiter.pid, Spawn.ended(waiter, Spawn.copy(output, out), output))
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def self.piped(env, command, group: false, **options)
|
|
171
|
+
options = options.merge(Spawn.group) if group
|
|
172
|
+
input, output, waiter = Spawn.open3(:popen2, env, *command, **options)
|
|
173
|
+
[Child.new(waiter.pid, waiter, group: group), input, output]
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# (Its output and errors copied to this one's, so what it said as it
|
|
177
|
+
# failed is there to see.)
|
|
178
|
+
def self.started(env, command, **options)
|
|
179
|
+
input, output, errors, waiter = Spawn.open3(:popen3, env, *command, **options)
|
|
180
|
+
input.close
|
|
181
|
+
copies = [Spawn.copy(output, $stdout, close: false), Spawn.copy(errors, $stderr, close: false)]
|
|
182
|
+
Child.new(waiter.pid, Spawn.ended(waiter, Thread.new { copies.each(&:join) }, output, errors))
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
# How a process starts here: Directly, but where Process.spawn can't
|
|
187
|
+
# redirect or wait, ThroughOpen3.
|
|
188
|
+
# simplecov:disable ThroughOpen3 is JRuby on Windows's (its test run
|
|
189
|
+
# covers it).
|
|
190
|
+
STARTER = (REDIRECTS && WAITS) ? Directly : ThroughOpen3
|
|
191
|
+
# simplecov:enable
|
|
192
|
+
|
|
193
|
+
# How a process that failed ended: its exit status +code+, or the
|
|
194
|
+
# +signal+ that killed it (as Child#wait says).
|
|
195
|
+
def self.why(code, signal)
|
|
196
|
+
return "exited with #{code}" if code
|
|
197
|
+
|
|
198
|
+
killed = signal #: Integer
|
|
199
|
+
name = begin
|
|
200
|
+
Signal.signame(killed)
|
|
201
|
+
# (JRuby's, for a number it has no name for, where CRuby's is nil,
|
|
202
|
+
# and on Windows for KILL's: https://github.com/jruby/jruby/issues/10080.)
|
|
203
|
+
rescue ArgumentError
|
|
204
|
+
nil
|
|
205
|
+
end
|
|
206
|
+
name ? "was killed by SIG#{name}" : "was killed by signal #{killed}"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# The end of the log at +path+, for a failure to show: its last
|
|
210
|
+
# LOG_LINES lines, without CRLF's CR (a Windows program's).
|
|
211
|
+
def self.tail(path) = File.readlines(path, chomp: true).last(LOG_LINES).map { |line| "#{line}\n" }.join
|
|
212
|
+
|
|
213
|
+
# The log at +path+, opened with +mode+, for a process's output: here,
|
|
214
|
+
# so it's there for a failure to show, and binary, so what's copied to
|
|
215
|
+
# it is as the process wrote it (on Windows, CRuby's text mode would
|
|
216
|
+
# make its CRLF CRCRLF).
|
|
217
|
+
def self.log(path, mode) = File.new(path, mode, binmode: true)
|
|
218
|
+
|
|
219
|
+
# Process.spawn's options (and Open3's) that start a process in a
|
|
220
|
+
# group of its own, for .kill_tree to kill it with what it starts: a
|
|
221
|
+
# process group, or on Windows, which has none, a console's (and
|
|
222
|
+
# Ctrl-C at the runner's doesn't reach it, as a process group's
|
|
223
|
+
# doesn't).
|
|
224
|
+
def self.group(windows: Platform::WINDOWS) = windows ? {new_pgroup: true} : {pgroup: true}
|
|
225
|
+
|
|
226
|
+
# Kills +pid+ and what it started: its process group (see .group), or
|
|
227
|
+
# on Windows, where no group can be killed, its tree of children
|
|
228
|
+
# (taskkill /T).
|
|
229
|
+
def self.kill_tree(pid, windows: Platform::WINDOWS)
|
|
230
|
+
return Process.kill("KILL", -pid) unless windows
|
|
231
|
+
|
|
232
|
+
# (Its output read, not shown, through Open3: JRuby's system on
|
|
233
|
+
# Windows ignores out: and err:,
|
|
234
|
+
# https://github.com/jruby/jruby/issues/10081. Where there's no
|
|
235
|
+
# taskkill, the process alone is killed.)
|
|
236
|
+
begin
|
|
237
|
+
open3(:capture2e, "taskkill", "/T", "/F", "/PID", pid.to_s).last.success?
|
|
238
|
+
rescue Errno::ENOENT
|
|
239
|
+
Process.kill("KILL", pid)
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Open3's +name+d method (capture3, popen2e...), given +args+ (an
|
|
244
|
+
# environment first, or none) and +options+. On JRuby on Windows
|
|
245
|
+
# (+jruby_on_windows+), whose Open3 gives a process the JVM's own
|
|
246
|
+
# environment (https://github.com/jruby/jruby/issues/8680) and ignores
|
|
247
|
+
# unsetenv_others, with ENV under the environment given (or with
|
|
248
|
+
# unsetenv_others, every one of ENV's variables unset but those
|
|
249
|
+
# given), each by ENV's name for it, and the program found, and its
|
|
250
|
+
# arguments quoted, as Windows takes them (see .program).
|
|
251
|
+
def self.open3(name, *args, jruby_on_windows: Platform::JRUBY_ON_WINDOWS, **options, &)
|
|
252
|
+
return Open3.public_send(name, *args, **options, &) unless jruby_on_windows
|
|
253
|
+
|
|
254
|
+
none = {} #: Hash[String, String?]
|
|
255
|
+
env, *command = args.first.is_a?(Hash) ? args : [none, *args]
|
|
256
|
+
current = ENV.to_h
|
|
257
|
+
under = options[:unsetenv_others] ? current.keys.product([nil]).to_h : current #: Hash[String, String?]
|
|
258
|
+
# (Each given by ENV's name for it, whatever its case: Windows's
|
|
259
|
+
# names are one, but JRuby's PATH given doesn't replace ENV's Path,
|
|
260
|
+
# https://github.com/jruby/jruby/issues/8951.)
|
|
261
|
+
given = env.transform_keys { |var| current.each_key.find { |name| name.casecmp?(var) } || var }
|
|
262
|
+
Open3.public_send(name, under.merge(given), *program(command), **options, &)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# +command+ as Windows runs it: a lone String that a shell runs (with
|
|
266
|
+
# a space in it, say) as cmd.exe's, and otherwise with its program
|
|
267
|
+
# found (by name, in +path+'s directories), as the program it is, or
|
|
268
|
+
# with each of +exts+ added where it has no extension (a .cmd file's,
|
|
269
|
+
# say), and its arguments quoted (see .quote). Errno::ENOENT if
|
|
270
|
+
# there's no such program.
|
|
271
|
+
def self.program(command, path: ENV.fetch("PATH", ""), exts: ENV.fetch("PATHEXT", ".EXE").split(";"))
|
|
272
|
+
program = command.fetch(0)
|
|
273
|
+
args = command.drop(1)
|
|
274
|
+
return ["cmd.exe", "/c", program] if args.empty? && program.match?(SHELL)
|
|
275
|
+
|
|
276
|
+
places = program.match?(%r{[/\\]}) ? [program] : path.split(File::PATH_SEPARATOR).map { |dir| File.join(dir, program) }
|
|
277
|
+
found = places.flat_map { |place| File.extname(place).empty? ? exts.map { |ext| place + ext } : place }.find { |file| File.file?(file) }
|
|
278
|
+
raise Errno::ENOENT, program unless found
|
|
279
|
+
|
|
280
|
+
[found, *args.map { |arg| quote(arg) }]
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# +arg+ as a Windows program's command line has it, where it's empty or
|
|
284
|
+
# has a space or a quote in it: in quotes, a quote in it after a
|
|
285
|
+
# backslash, and the backslashes before a quote, or before its end,
|
|
286
|
+
# doubled. (Java quotes one with a space alone, and passes its quotes
|
|
287
|
+
# on as they are, which the program takes out:
|
|
288
|
+
# https://github.com/jruby/jruby/issues/8945.)
|
|
289
|
+
def self.quote(arg)
|
|
290
|
+
return arg unless arg.empty? || arg.match?(/[\s"]/)
|
|
291
|
+
|
|
292
|
+
%("#{arg.gsub(/(\\*)"/) { "#{$1}#{$1}\\\"" }.sub(/(\\+)\z/) { "#{$1}#{$1}" }}")
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# A thread copying +output+ to +out+, until it ends (or is closed),
|
|
296
|
+
# then closing both (but +out+, unless +close+: this process's
|
|
297
|
+
# $stdout, say).
|
|
298
|
+
def self.copy(output, out, close: true)
|
|
299
|
+
Thread.new do
|
|
300
|
+
# (A read at a time, not IO.copy_stream, which on Linux closing
|
|
301
|
+
# the pipe under it doesn't stop, and each in the log at once, for
|
|
302
|
+
# what waits on it to see.)
|
|
303
|
+
loop do
|
|
304
|
+
out.write(output.readpartial(CHUNK))
|
|
305
|
+
out.flush
|
|
306
|
+
end
|
|
307
|
+
# (At its end, EOFError, or closed under it, COPY_WAIT seconds after
|
|
308
|
+
# its process ended: IOError, or EBADF.)
|
|
309
|
+
rescue IOError, Errno::EBADF
|
|
310
|
+
nil
|
|
311
|
+
ensure
|
|
312
|
+
output.close
|
|
313
|
+
out.close if close
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# A thread whose value is +waiter+'s, once +copy+ is done copying
|
|
318
|
+
# +outputs+, or COPY_WAIT seconds later, when they're closed under it.
|
|
319
|
+
def self.ended(waiter, copy, *outputs)
|
|
320
|
+
Thread.new do
|
|
321
|
+
status = waiter.value
|
|
322
|
+
outputs.each(&:close) unless copy.join(COPY_WAIT)
|
|
323
|
+
status
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
end
|
data/lib/duel/summary.rb
CHANGED
|
@@ -27,9 +27,11 @@ module Duel
|
|
|
27
27
|
FAILED = %w[regressed error].freeze
|
|
28
28
|
|
|
29
29
|
# What the heading starts with: a run that passed, one that failed (or
|
|
30
|
-
# couldn't compare),
|
|
30
|
+
# couldn't compare), one that failed but was accepted (see
|
|
31
|
+
# Config#accept_label), and one with nothing compared.
|
|
31
32
|
PASSED_MARK = "🟢"
|
|
32
33
|
FAILED_MARK = "🔻"
|
|
34
|
+
ACCEPTED_MARK = "🟡"
|
|
33
35
|
NEUTRAL_MARK = "⬜"
|
|
34
36
|
|
|
35
37
|
# What an unsettled timing means.
|
|
@@ -91,9 +93,11 @@ module Duel
|
|
|
91
93
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
92
94
|
# +candidate+ (each a description, in Markdown, whose commit stands
|
|
93
95
|
# bare, so GitHub links it) as +settings+ say, with a +note+ (in
|
|
94
|
-
# Markdown) under the heading, if there's one
|
|
95
|
-
|
|
96
|
+
# Markdown) under the heading, if there's one, and whether what failed
|
|
97
|
+
# was +accepted+ (see Config#accept_label).
|
|
98
|
+
def initialize(runner, results, settings:, baseline:, candidate:, note: nil, accepted: nil)
|
|
96
99
|
@note = note
|
|
100
|
+
@accepted = accepted
|
|
97
101
|
@runner = runner
|
|
98
102
|
@results = results
|
|
99
103
|
@settings = settings
|
|
@@ -120,14 +124,16 @@ module Duel
|
|
|
120
124
|
end
|
|
121
125
|
|
|
122
126
|
# The verdict: what failed (benchmarks, or a metric across the board),
|
|
123
|
-
# or that every benchmark compared
|
|
127
|
+
# and whether that was accepted, or that every benchmark compared
|
|
128
|
+
# passed, and those skipped.
|
|
124
129
|
def heading
|
|
125
130
|
skipped = @results.count(&:skipped)
|
|
126
131
|
compared = @results.size > skipped
|
|
127
132
|
slower = @runner.regressed_across
|
|
128
133
|
said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
|
|
129
134
|
("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
|
|
130
|
-
|
|
135
|
+
said << "accepted" if @accepted && said.any?
|
|
136
|
+
mark = if said.any? then @accepted ? ACCEPTED_MARK : FAILED_MARK
|
|
131
137
|
elsif compared then PASSED_MARK
|
|
132
138
|
else NEUTRAL_MARK
|
|
133
139
|
end
|
|
@@ -151,10 +157,7 @@ module Duel
|
|
|
151
157
|
def failures
|
|
152
158
|
return if @failed.empty?
|
|
153
159
|
|
|
154
|
-
|
|
155
|
-
header << "Most extra CPU in" unless @failed.all? { |r| top(r).empty? }
|
|
156
|
-
rows = @failed.map { |r| row([Summary.code(cell(r.name)), failure(r), top(r)].first(header.size)) }
|
|
157
|
-
([row(header), "|#{"---|" * header.size}"] + rows).join("\n")
|
|
160
|
+
grid(["Benchmark", "Failure", "Most extra CPU in"], @failed.map { |r| [Summary.code(cell(r.name)), failure(r), top(r)] })
|
|
158
161
|
end
|
|
159
162
|
|
|
160
163
|
# What got worse, or the error's first line.
|
|
@@ -172,7 +175,7 @@ module Duel
|
|
|
172
175
|
else
|
|
173
176
|
# (Peak memory, which grew too much.)
|
|
174
177
|
memory = result.memory #: Hash[String, Integer]
|
|
175
|
-
"#{metric} #{mb(memory.fetch("baseline"))} → #{mb(memory.fetch("candidate"))}"
|
|
178
|
+
"#{metric} #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
|
|
176
179
|
end
|
|
177
180
|
end
|
|
178
181
|
worse.join(result.mixed? ? " or " : ", ")
|
|
@@ -219,8 +222,13 @@ module Duel
|
|
|
219
222
|
shown = Summary.verdict(r)
|
|
220
223
|
[Summary.code(cell(r.name)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r)]
|
|
221
224
|
end
|
|
222
|
-
|
|
223
|
-
|
|
225
|
+
grid(["Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"], rows)
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
# A table of +rows+ under +header+: without its last column, where
|
|
229
|
+
# none of the rows has anything in it.
|
|
230
|
+
def grid(header, rows)
|
|
231
|
+
columns = (rows.any? { |cells| !cells.last.empty? }) ? header.size : header.size - 1
|
|
224
232
|
([row(header.first(columns)), "|#{"---|" * columns}"] + rows.map { |cells| row(cells.first(columns)) }).join("\n")
|
|
225
233
|
end
|
|
226
234
|
|
|
@@ -251,13 +259,11 @@ module Duel
|
|
|
251
259
|
end
|
|
252
260
|
if result.more_memory?
|
|
253
261
|
memory = result.memory #: Hash[String, Integer]
|
|
254
|
-
counted << "peak_memory #{mb(memory.fetch("baseline"))} → #{mb(memory.fetch("candidate"))}"
|
|
262
|
+
counted << "peak_memory #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
|
|
255
263
|
end
|
|
256
264
|
counted.join("<br>")
|
|
257
265
|
end
|
|
258
266
|
|
|
259
|
-
def mb(bytes) = format("%.1fMB", bytes / 1_048_576.0)
|
|
260
|
-
|
|
261
267
|
# The first +explained+ regressions, explained as the terminal report
|
|
262
268
|
# explains them (and the rest named).
|
|
263
269
|
def regressions(explained)
|
|
@@ -64,9 +64,11 @@
|
|
|
64
64
|
# The report, as a new comment on the merge request, when Duel says to:
|
|
65
65
|
# always (post), or only when its last comment's status differs from this
|
|
66
66
|
# run's (change), so the latest comment is never stale. It never edits or
|
|
67
|
-
# deletes a comment.
|
|
67
|
+
# deletes a comment. It runs `duel comment`, from RubyGems (none of the
|
|
68
|
+
# merge request's code), so a fix to how it comments comes with a release
|
|
69
|
+
# of Duel.
|
|
68
70
|
<%= gitlab_job %>:comment:
|
|
69
|
-
image:
|
|
71
|
+
image: ruby:slim
|
|
70
72
|
stage: .post
|
|
71
73
|
needs: [<%= gitlab_job %>]
|
|
72
74
|
rules:
|
|
@@ -75,23 +77,10 @@
|
|
|
75
77
|
variables:
|
|
76
78
|
GIT_STRATEGY: none # it runs none of the code
|
|
77
79
|
script:
|
|
78
|
-
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
[ -f "$report" ] || exit 0
|
|
83
|
-
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
84
|
-
token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
|
|
85
|
-
marker="<!-- <%= marker %>"
|
|
86
|
-
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
87
|
-
# The first line of the last comment Duel posted here: its marker,
|
|
88
|
-
# with its status.
|
|
89
|
-
last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
|
|
90
|
-
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
|
|
91
|
-
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
92
|
-
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
|
|
93
|
-
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
|
|
94
|
-
fi
|
|
80
|
+
- gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
|
|
81
|
+
# (The merge request's head: in a merged results pipeline, not the
|
|
82
|
+
# commit that merges it.)
|
|
83
|
+
- duel comment --gitlab --report <%= working_directory || "." %> --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker '<%= marker.gsub("'") { "'\\''" } %>'
|
|
95
84
|
|
|
96
85
|
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
97
86
|
# for a merge request: only where a regression in a push passes (see
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
name: Perf<%= " (#{working_directory})" if working_directory %>
|
|
2
|
+
|
|
3
|
+
# Duel's performance regression guards, run by Duel's reusable workflow
|
|
4
|
+
# (see Duel's docs/ci.md), which this one calls with what .duel.rb sets
|
|
5
|
+
# that it needs: it compares a pull request with the branch it targets,
|
|
6
|
+
# a merge queue's group with the queue's base, a push with what was there
|
|
7
|
+
# before (or as c.push_base in .duel.rb says, the last commit Duel
|
|
8
|
+
# passed), and on a schedule, the last release (slow creep).
|
|
9
|
+
on:
|
|
10
|
+
pull_request:
|
|
11
|
+
merge_group: # (a merge queue's)
|
|
12
|
+
push:
|
|
13
|
+
branches: [main] # (and any other branch to guard, a stable one, say)
|
|
14
|
+
schedule:
|
|
15
|
+
- cron: "0 4 * * *"
|
|
16
|
+
workflow_dispatch:
|
|
17
|
+
|
|
18
|
+
# A pull request's newer push cancels its run, if that's still going, as
|
|
19
|
+
# only its newest commit's result counts. Every other run (a push's, say)
|
|
20
|
+
# runs to the end, so that each push is compared.
|
|
21
|
+
concurrency:
|
|
22
|
+
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
|
|
23
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
24
|
+
|
|
25
|
+
permissions: {}
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
duel:
|
|
29
|
+
uses: <%= REUSABLE %>/duel.yml@v<%= Duel::VERSION %>
|
|
30
|
+
# (Its Duel job, which runs the branch's code, only reads.)
|
|
31
|
+
permissions:
|
|
32
|
+
<% if writes? -%>
|
|
33
|
+
contents: write # to mark a push passed, comment on it, or keep the badge
|
|
34
|
+
<% else -%>
|
|
35
|
+
contents: read
|
|
36
|
+
<% end -%>
|
|
37
|
+
pull-requests: read # for the pull request's labels (see c.accept_label)
|
|
38
|
+
# (And with setup:, shell commands that set up whatever else your
|
|
39
|
+
# tests need: packages, services.)
|
|
40
|
+
with:
|
|
41
|
+
<% perf_inputs.each do |name, value| -%>
|
|
42
|
+
<%= name %>: <%= value %>
|
|
43
|
+
<% end -%>
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
name: Perf comment<%= " (#{working_directory})" if working_directory %>
|
|
2
|
+
|
|
3
|
+
# Duel's report, as a new comment on the pull request, when Duel says
|
|
4
|
+
# to, by Duel's reusable workflow (see Duel's docs/ci.md), which this
|
|
5
|
+
# one calls once the Perf workflow has run, with a token that can write,
|
|
6
|
+
# so a pull request from a fork gets a comment too. It checks nothing
|
|
7
|
+
# out and runs none of the pull request's code, only `duel comment`.
|
|
8
|
+
<% if accept_label -%>
|
|
9
|
+
#
|
|
10
|
+
# And when the label that accepts a regression (c.accept_label in
|
|
11
|
+
# .duel.rb) is added to a pull request, or taken off, it runs the Duel
|
|
12
|
+
# job again.
|
|
13
|
+
<% end -%>
|
|
14
|
+
#
|
|
15
|
+
# (The workflow must be on the default branch before it runs at all.)
|
|
16
|
+
on:
|
|
17
|
+
workflow_run:
|
|
18
|
+
workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
|
|
19
|
+
types: [completed]
|
|
20
|
+
<% if accept_label -%>
|
|
21
|
+
pull_request_target:
|
|
22
|
+
types: [labeled, unlabeled]
|
|
23
|
+
<% end -%>
|
|
24
|
+
|
|
25
|
+
permissions: {}
|
|
26
|
+
|
|
27
|
+
jobs:
|
|
28
|
+
comment:
|
|
29
|
+
uses: <%= REUSABLE %>/duel-comment.yml@v<%= Duel::VERSION %>
|
|
30
|
+
permissions:
|
|
31
|
+
<% if accept_label -%>
|
|
32
|
+
actions: write # to get the report from the Perf run, and run its Duel job again
|
|
33
|
+
<% else -%>
|
|
34
|
+
actions: read # to get the report from the Perf run
|
|
35
|
+
<% end -%>
|
|
36
|
+
pull-requests: write # to comment on the pull request with the report
|
|
37
|
+
<% unless comment_inputs.empty? -%>
|
|
38
|
+
with:
|
|
39
|
+
<% comment_inputs.each do |name, value| -%>
|
|
40
|
+
<%= name %>: <%= value %>
|
|
41
|
+
<% end -%>
|
|
42
|
+
<% end -%>
|