duel 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +20 -16
- data/CHANGELOG.md +148 -1
- data/README.md +59 -26
- data/docs/ci.md +612 -54
- data/docs/configuration.md +146 -4
- data/docs/how-it-works.md +8 -3
- data/docs/json.md +5 -2
- data/docs/troubleshooting.md +6 -5
- data/lib/duel/baseline.rb +61 -7
- data/lib/duel/cli.rb +259 -48
- data/lib/duel/config.rb +108 -3
- data/lib/duel/init.rb +49 -7
- data/lib/duel/json_report.rb +11 -9
- data/lib/duel/runner.rb +1 -1
- data/lib/duel/summary.rb +148 -74
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +148 -0
- data/lib/duel/templates/duel.rb.erb +20 -2
- data/lib/duel/templates/perf-comment.yml.erb +59 -0
- data/lib/duel/templates/perf.yml.erb +174 -37
- data/lib/duel/version.rb +1 -1
- data/sig/duel/baseline.rbs +30 -2
- data/sig/duel/cli.rbs +88 -5
- data/sig/duel/config.rbs +52 -1
- data/sig/duel/init.rbs +19 -0
- data/sig/duel/json_report.rbs +7 -5
- data/sig/duel/runner.rbs +18 -3
- data/sig/duel/summary.rbs +50 -18
- metadata +3 -1
data/lib/duel/cli.rb
CHANGED
|
@@ -16,9 +16,12 @@ module Duel
|
|
|
16
16
|
# rake task or a CI expression can't pass as an option can be set in
|
|
17
17
|
# the environment (DUEL_BASE, DUEL_FILTER, DUEL_FORCE), and options win.
|
|
18
18
|
# On GitHub Actions (GITHUB_STEP_SUMMARY), it adds its report to the
|
|
19
|
-
# job's summary, in Markdown (see Summary)
|
|
20
|
-
#
|
|
21
|
-
#
|
|
19
|
+
# job's summary, in Markdown (see Summary), and tells the step's
|
|
20
|
+
# outputs (GITHUB_OUTPUT) what to do about the comment on a pull
|
|
21
|
+
# request (see .comment), and for a push, whether to mark it passed
|
|
22
|
+
# (see Config#push_base). --markdown writes the Markdown to a file, and
|
|
23
|
+
# --json the report as JSON (see JSONReport), whatever the run's
|
|
24
|
+
# outcome, but not for a command line that isn't valid.
|
|
22
25
|
# @api private
|
|
23
26
|
class CLI
|
|
24
27
|
USAGE = "Usage: duel [options]\n duel init [options]\n\n" \
|
|
@@ -27,6 +30,32 @@ module Duel
|
|
|
27
30
|
# Where GitHub Actions has a step add to its job's summary.
|
|
28
31
|
STEP_SUMMARY = "GITHUB_STEP_SUMMARY"
|
|
29
32
|
|
|
33
|
+
# Where GitHub Actions has a step set its outputs.
|
|
34
|
+
STEP_OUTPUT = "GITHUB_OUTPUT"
|
|
35
|
+
|
|
36
|
+
# Where else to write them, as variables (DUEL_COMMENT, DUEL_STATUS,
|
|
37
|
+
# DUEL_PASSED): on GitLab CI, a dotenv report, for the jobs after.
|
|
38
|
+
OUTPUT = "DUEL_OUTPUT"
|
|
39
|
+
|
|
40
|
+
# The statuses (as the JSON report has them) that the workflow
|
|
41
|
+
# comments on, by when it does (Config#comment, but :never, which
|
|
42
|
+
# comments on none).
|
|
43
|
+
COMMENTED = {always: %w[passed failed nothing_to_compare error], results: %w[passed failed error], failures: %w[failed error]}.freeze #: Hash[Config::comment, Array[String]]
|
|
44
|
+
|
|
45
|
+
# A run's status (as the JSON report has it) as the comment's marker
|
|
46
|
+
# has it: passed, failed (or couldn't compare), or neutral.
|
|
47
|
+
MARKED = {"passed" => "passed", "failed" => "failed", "error" => "failed", "nothing_to_compare" => "neutral"}.freeze
|
|
48
|
+
|
|
49
|
+
# The GitHub Actions events of a pull request, as GITHUB_EVENT_NAME
|
|
50
|
+
# has them: a merge queue's too (see Config#fail_on). (A GitLab merge
|
|
51
|
+
# request's is pull_request: see #event.)
|
|
52
|
+
PULL_REQUESTS = %w[pull_request pull_request_target merge_group].freeze
|
|
53
|
+
|
|
54
|
+
# The exit status of a run that was slower outside a merge request on
|
|
55
|
+
# GitLab CI, where only a merge request fails (Config#fail_on): the
|
|
56
|
+
# job that `duel init --gitlab` writes lets it fail, as a warning.
|
|
57
|
+
WARNED = 3
|
|
58
|
+
|
|
30
59
|
# The variables that set options, to their values, and the one that
|
|
31
60
|
# sets a flag by being set at all.
|
|
32
61
|
ENV_OPTIONS = {"DUEL_BASE" => "--base", "DUEL_FILTER" => "--filter"}.freeze
|
|
@@ -40,16 +69,17 @@ module Duel
|
|
|
40
69
|
end
|
|
41
70
|
|
|
42
71
|
# Runs, and returns the exit status: 0 not slower (or nothing to compare),
|
|
43
|
-
# 1 slower (or a benchmark failed), 2 couldn't compare
|
|
72
|
+
# 1 slower (or a benchmark failed), 2 couldn't compare, 3 slower but
|
|
73
|
+
# warned (see WARNED).
|
|
44
74
|
def run
|
|
45
75
|
return Init.new(@argv.drop(1), out: @out, err: @err).run if @argv.first == "init"
|
|
46
76
|
|
|
47
77
|
settings = @settings = catch(:version) { parse } or return 0
|
|
48
|
-
config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
78
|
+
config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
49
79
|
Lock.hold(File.join(config.root, "tmp", "duel")) { compare(config, settings) }
|
|
50
80
|
rescue Error => e
|
|
51
81
|
@err.puts "duel: #{e}"
|
|
52
|
-
unfinished("couldn't compare", "error",
|
|
82
|
+
unfinished(Summary::FAILED_MARK, "couldn't compare", "error", e.to_s, 2)
|
|
53
83
|
rescue OptionParser::ParseError => e
|
|
54
84
|
@err.puts "duel: #{e}"
|
|
55
85
|
2
|
|
@@ -63,11 +93,43 @@ module Duel
|
|
|
63
93
|
raise OptionParser::InvalidArgument, "#{source} (#{e})"
|
|
64
94
|
end
|
|
65
95
|
|
|
96
|
+
# Whether the workflow comments on a pull request, when it comments as
|
|
97
|
+
# +mode+ says (Config#comment) and the run's +status+ is the JSON
|
|
98
|
+
# report's: "post" a new comment, "change", posting one only if it
|
|
99
|
+
# commented on the pull request before and its last comment's status
|
|
100
|
+
# (see MARKED) isn't this run's, or "none". It never edits a comment,
|
|
101
|
+
# so the pull request keeps every one.
|
|
102
|
+
def self.comment(mode, status)
|
|
103
|
+
return "none" if mode == :never
|
|
104
|
+
|
|
105
|
+
COMMENTED.fetch(mode).include?(status) ? "post" : "change"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# The mode --comment names (see Config#comment), or an invalid
|
|
109
|
+
# argument.
|
|
110
|
+
def self.comment_mode(name) = named(name, Config::COMMENTS)
|
|
111
|
+
|
|
112
|
+
# The runs --fail-on names (see Config#fail_on), or an invalid
|
|
113
|
+
# argument.
|
|
114
|
+
def self.fail_on(name) = named(name, Config::FAIL_ON)
|
|
115
|
+
|
|
116
|
+
# Which of +values+ (symbols) an option's argument names, or an
|
|
117
|
+
# invalid argument.
|
|
118
|
+
def self.named(name, values)
|
|
119
|
+
values.find { |value| value.to_s == name } or
|
|
120
|
+
raise OptionParser::InvalidArgument, "#{name} (give #{values.join(", ")})"
|
|
121
|
+
end
|
|
122
|
+
|
|
66
123
|
# The settings, from the environment and then the options.
|
|
67
124
|
def parse
|
|
68
125
|
settings = Settings.defaults
|
|
69
126
|
settings.jobs = [Etc.nprocessors / 2, 1].max
|
|
70
127
|
parser(settings).parse!(env_options + @argv)
|
|
128
|
+
branch = variable("GITHUB_REF_NAME", "CI_COMMIT_BRANCH")
|
|
129
|
+
base = variable("DUEL_BASE")
|
|
130
|
+
settings.pushed = branch if pushed? && !base.nil? && settings.base == base
|
|
131
|
+
target = variable("GITHUB_BASE_REF", "CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
|
|
132
|
+
settings.target = "origin/#{target}" if target && pull_request?
|
|
71
133
|
settings.config_path ||= (".duel.rb" if File.exist?(".duel.rb"))
|
|
72
134
|
settings.config_path &&= File.expand_path(settings.config_path)
|
|
73
135
|
settings
|
|
@@ -83,9 +145,15 @@ module Duel
|
|
|
83
145
|
@out.puts "duel: removed tmp/duel and its baseline's worktree"
|
|
84
146
|
return 0
|
|
85
147
|
end
|
|
86
|
-
|
|
148
|
+
if settings.base_dir.nil? && baseline.ref.nil?
|
|
149
|
+
return nothing("no commit to compare against", "no commit before this push to compare against") if settings.pushed
|
|
150
|
+
|
|
151
|
+
return nothing("no release to compare against", "no release tag (#{config.release_tags}) before HEAD to compare against")
|
|
152
|
+
end
|
|
87
153
|
if !settings.force && baseline.unchanged?
|
|
88
|
-
|
|
154
|
+
@passed = marks_passed?(config, settings)
|
|
155
|
+
return nothing("nothing to compare",
|
|
156
|
+
"#{baseline.watched.join(", ")} unchanged since the baseline, so there's nothing to compare (--force to anyway)")
|
|
89
157
|
end
|
|
90
158
|
|
|
91
159
|
candidate = settings.candidate_dir || config.root
|
|
@@ -94,44 +162,176 @@ module Duel
|
|
|
94
162
|
uneven(base_root, candidate)
|
|
95
163
|
runner = Runner.new(config, settings, base_root, out: @out)
|
|
96
164
|
results = runner.run
|
|
97
|
-
|
|
165
|
+
passed = runner.passed?(results)
|
|
166
|
+
status = passed ? 0 : slower(config, settings)
|
|
167
|
+
@passed = (passed || accepted?(config)) && marks_passed?(config, settings)
|
|
98
168
|
publish(
|
|
169
|
+
passed ? "passed" : "failed",
|
|
99
170
|
-> {
|
|
100
171
|
Summary.new(runner, results, settings: settings, baseline: described_baseline(config, settings, baseline),
|
|
101
|
-
candidate: described_candidate(config, settings)).markdown
|
|
172
|
+
candidate: described_candidate(config, settings), note: @note).markdown
|
|
102
173
|
},
|
|
103
|
-
-> {
|
|
104
|
-
JSONReport.new(runner, results, settings: settings, exit: status,
|
|
105
|
-
candidate: candidate_json(config, settings)).to_h
|
|
174
|
+
->(comment) {
|
|
175
|
+
JSONReport.new(runner, results, settings: settings, exit: status, comment: comment,
|
|
176
|
+
baseline: baseline_json(config, settings, baseline, base_root), candidate: candidate_json(config, settings)).to_h
|
|
106
177
|
}
|
|
107
178
|
)
|
|
108
179
|
status
|
|
109
180
|
end
|
|
110
181
|
|
|
111
|
-
#
|
|
112
|
-
|
|
182
|
+
# The exit status of a run that was slower (or whose benchmark
|
|
183
|
+
# failed): 1, or 0 where the pull request's label accepts it
|
|
184
|
+
# (Config#accept_label), or on GitHub Actions outside a pull request,
|
|
185
|
+
# where c.fail_on (or --fail-on) says only a pull request fails, with
|
|
186
|
+
# a warning that says so, and a note in the report.
|
|
187
|
+
def slower(config, settings)
|
|
188
|
+
if accepted?(config)
|
|
189
|
+
# (Set: see #accepted?.)
|
|
190
|
+
label = config.accept_label #: String
|
|
191
|
+
return passing("accepted with the #{label} label", "Accepted with the #{Summary.code(label)} label.")
|
|
192
|
+
end
|
|
193
|
+
return 1 unless pull_requests_only?(config, settings)
|
|
194
|
+
return 1 unless outside_pull_request?
|
|
195
|
+
|
|
196
|
+
# (Outside a pull request on CI, so there's one: see #event.)
|
|
197
|
+
run = event #: String
|
|
198
|
+
passing("only a pull request fails (c.fail_on), so this #{run} passes",
|
|
199
|
+
"Only a pull request fails (#{Summary.code("c.fail_on")}), so this #{run} passes.", gitlab? ? WARNED : 0)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Passes a run that was slower (or on GitLab CI, warns: +exit+), as
|
|
203
|
+
# +why+ says, in a warning (on GitHub Actions, an annotation), and as
|
|
204
|
+
# +note+ says in the report.
|
|
205
|
+
def passing(why, note, exit = 0)
|
|
206
|
+
@out.puts "#{gitlab? ? "duel: " : "::warning::Duel: "}slower, but #{why}"
|
|
207
|
+
@note = note
|
|
208
|
+
exit
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Whether the pull request's labels have the one that accepts a
|
|
212
|
+
# regression (Config#accept_label).
|
|
213
|
+
def accepted?(config)
|
|
214
|
+
label = config.accept_label
|
|
215
|
+
!label.nil? && labels.include?(label)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The pull request's labels: DUEL_LABELS (a JSON array of their names)
|
|
219
|
+
# on GitHub Actions, or a GitLab merge request's
|
|
220
|
+
# (CI_MERGE_REQUEST_LABELS, by commas), or none, where neither is
|
|
221
|
+
# set, or DUEL_LABELS isn't a JSON array.
|
|
222
|
+
def labels
|
|
223
|
+
set = variables
|
|
224
|
+
return set.fetch("CI_MERGE_REQUEST_LABELS").split(",") if !set.key?("DUEL_LABELS") && set.key?("CI_MERGE_REQUEST_LABELS")
|
|
225
|
+
return [] unless set.key?("DUEL_LABELS")
|
|
226
|
+
|
|
227
|
+
names = JSON.parse(set.fetch("DUEL_LABELS"))
|
|
228
|
+
names.instance_of?(Array) ? names : []
|
|
229
|
+
rescue JSON::ParserError
|
|
230
|
+
[]
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Whether the workflow marks a push that passed (or had nothing to
|
|
234
|
+
# compare) as the last commit Duel passed: where Config#push_base
|
|
235
|
+
# says to compare a push with that commit.
|
|
236
|
+
def marks_passed?(config, settings) = !settings.pushed.nil? && config.push_base == :last_passed
|
|
237
|
+
|
|
238
|
+
# Whether a regression fails only a pull request (--fail-on, or else
|
|
239
|
+
# Config#fail_on).
|
|
240
|
+
def pull_requests_only?(config, settings) = (settings.fail_on || config.fail_on) == :pull_requests
|
|
241
|
+
|
|
242
|
+
# What a run on CI is for, as GitHub Actions names it
|
|
243
|
+
# (GITHUB_EVENT_NAME): push, pull_request, schedule, say. On GitLab
|
|
244
|
+
# CI, its pipeline's source (CI_PIPELINE_SOURCE), as GitHub would
|
|
245
|
+
# name it: a merge request's is pull_request. Outside CI, nil.
|
|
246
|
+
def event
|
|
247
|
+
github = variable("GITHUB_EVENT_NAME")
|
|
248
|
+
return github if github
|
|
249
|
+
return unless gitlab?
|
|
250
|
+
|
|
251
|
+
source = variable("CI_PIPELINE_SOURCE")
|
|
252
|
+
(source == "merge_request_event") ? "pull_request" : source
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Whether this is a run on GitLab CI.
|
|
256
|
+
def gitlab? = variables.key?("GITLAB_CI")
|
|
257
|
+
|
|
258
|
+
# Whether this is a run on CI for something other than a pull request
|
|
259
|
+
# (PULL_REQUESTS): a push, say.
|
|
260
|
+
def outside_pull_request?
|
|
261
|
+
run = event
|
|
262
|
+
!run.nil? && !PULL_REQUESTS.include?(run)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Whether this is a run on CI for a pull request (or a merge queue:
|
|
266
|
+
# see PULL_REQUESTS).
|
|
267
|
+
def pull_request? = PULL_REQUESTS.include?(event)
|
|
268
|
+
|
|
269
|
+
# Whether the baseline is a merge queue's base: on GitHub Actions for
|
|
270
|
+
# a merge queue, as DUEL_BASE gives it.
|
|
271
|
+
def queued?(settings) = event == "merge_group" && settings.base == variable("DUEL_BASE")
|
|
272
|
+
|
|
273
|
+
# Whether this is a run on CI for a push.
|
|
274
|
+
def pushed? = event == "push"
|
|
275
|
+
|
|
276
|
+
# The first of the variables +names+ that's set, or nil.
|
|
277
|
+
def variable(*names)
|
|
278
|
+
set = variables
|
|
279
|
+
name = names.find { |n| set.key?(n) }
|
|
280
|
+
set.fetch(name) if name
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# Says there's nothing to compare (+title+, in Markdown), and +why+,
|
|
284
|
+
# and passes.
|
|
285
|
+
def nothing(title, why)
|
|
113
286
|
@out.puts "duel: #{why}"
|
|
114
|
-
unfinished(
|
|
287
|
+
unfinished(Summary::NEUTRAL_MARK, title, "nothing_to_compare", why, 0)
|
|
115
288
|
end
|
|
116
289
|
|
|
117
|
-
# Reports a run that didn't compare (+title+
|
|
118
|
-
# and +status+ in JSON)
|
|
119
|
-
def unfinished(title, status,
|
|
120
|
-
# (Parsed, so set
|
|
290
|
+
# Reports a run that didn't compare (+mark+ and +title+ say so in
|
|
291
|
+
# Markdown, and +status+ in JSON), and +why+, and returns +exit+.
|
|
292
|
+
def unfinished(mark, title, status, why, exit)
|
|
293
|
+
# (Parsed, so set: see #run.)
|
|
121
294
|
settings = @settings #: Settings
|
|
122
|
-
publish(-> { Summary.note(title,
|
|
295
|
+
publish(status, -> { Summary.note(mark, title, why) }, ->(comment) { JSONReport.note(status, exit, "duel: #{why}", settings, comment) })
|
|
123
296
|
exit
|
|
124
297
|
end
|
|
125
298
|
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
#
|
|
130
|
-
|
|
299
|
+
# What the workflow does about its comment on a pull request, or on
|
|
300
|
+
# a pushed commit (see .comment), for a run whose +status+ is the
|
|
301
|
+
# JSON report's: as --comment says, or else the configuration, or the
|
|
302
|
+
# default where the configuration didn't load. It comments on a push
|
|
303
|
+
# only where a regression in one passes (Config#fail_on), so the
|
|
304
|
+
# comment says what the check doesn't.
|
|
305
|
+
def commenting(status)
|
|
306
|
+
# (Parsed, so set: see #run.)
|
|
307
|
+
settings = @settings #: Settings
|
|
308
|
+
config = @config || Config.new
|
|
309
|
+
return "none" if pushed? && !pull_requests_only?(config, settings)
|
|
310
|
+
|
|
311
|
+
CLI.comment(settings.comment || config.comment, status)
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
# Writes the reports of a run whose +status+ is the JSON report's
|
|
315
|
+
# where they're asked for: the JSON +json+ makes, given what the
|
|
316
|
+
# workflow does about its comment (see #commenting), to the file
|
|
317
|
+
# --json names, and the Markdown +markdown+ makes to the file
|
|
318
|
+
# --markdown names, and added to the job's summary on GitHub Actions
|
|
319
|
+
# (STEP_SUMMARY). Each is only made if it's written. On GitHub
|
|
320
|
+
# Actions, it also sets the step's outputs (STEP_OUTPUT) for the jobs
|
|
321
|
+
# that comment: +comment+, and +status+ as the comment's marker has
|
|
322
|
+
# it (MARKED), and for the job that marks a push passed, "passed"
|
|
323
|
+
# (see #marks_passed?). Where DUEL_OUTPUT names a file (OUTPUT), it
|
|
324
|
+
# writes them there too, as variables.
|
|
325
|
+
def publish(status, markdown, json)
|
|
326
|
+
comment = commenting(status)
|
|
327
|
+
set = variables
|
|
328
|
+
outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed=true\n" if @passed}"
|
|
329
|
+
File.write(set.fetch(STEP_OUTPUT), outputs, mode: "a") if set.key?(STEP_OUTPUT)
|
|
330
|
+
File.write(set.fetch(OUTPUT), outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if set.key?(OUTPUT)
|
|
131
331
|
path = @json
|
|
132
|
-
write(path, "#{JSON.pretty_generate(json.call)}\n") if path
|
|
332
|
+
write(path, "#{JSON.pretty_generate(json.call(comment))}\n") if path
|
|
133
333
|
path = @markdown
|
|
134
|
-
summary = (
|
|
334
|
+
summary = (set.fetch(STEP_SUMMARY) if set.key?(STEP_SUMMARY))
|
|
135
335
|
return unless path || summary
|
|
136
336
|
|
|
137
337
|
text = markdown.call
|
|
@@ -145,31 +345,42 @@ module Duel
|
|
|
145
345
|
File.write(path, text)
|
|
146
346
|
end
|
|
147
347
|
|
|
148
|
-
# The baseline, in Markdown: its directory, or its commit,
|
|
149
|
-
#
|
|
348
|
+
# The baseline, in Markdown: its directory, or its commit, bare (so
|
|
349
|
+
# GitHub links it) and followed by a space, never punctuation, and
|
|
350
|
+
# what chose it: the main branch, the push (on GitHub Actions, the
|
|
351
|
+
# commit before it, from DUEL_BASE, or the last commit Duel passed),
|
|
352
|
+
# the merge queue (its base, from DUEL_BASE), the last release, or
|
|
353
|
+
# --base.
|
|
150
354
|
def described_baseline(config, settings, baseline)
|
|
151
355
|
dir = settings.base_dir
|
|
152
356
|
return Summary.code(dir) if dir
|
|
153
357
|
|
|
154
358
|
# (There's one: #compare checked.)
|
|
155
359
|
ref = baseline.ref #: String
|
|
156
|
-
commit =
|
|
157
|
-
case settings.base
|
|
158
|
-
when nil then "
|
|
159
|
-
when Baseline::LAST_RELEASE then "#{Summary.code(ref)}
|
|
160
|
-
else
|
|
360
|
+
commit = git(config.root, "rev-parse", "--short", "#{ref}^{commit}").first.chomp
|
|
361
|
+
chosen = case settings.base
|
|
362
|
+
when nil then "where HEAD left #{Summary.code(baseline.main_branch)}"
|
|
363
|
+
when Baseline::LAST_RELEASE then "the last release, #{Summary.code(ref)}"
|
|
364
|
+
else
|
|
365
|
+
if settings.pushed then (ref == baseline.passed) ? "the last commit Duel passed" : "the commit before the push"
|
|
366
|
+
elsif queued?(settings) then "the merge queue's base"
|
|
367
|
+
elsif !ref.match?(/\A\h+\z/) then Summary.code(ref)
|
|
368
|
+
else "from --base"
|
|
369
|
+
end
|
|
161
370
|
end
|
|
371
|
+
"#{commit} (#{chosen})"
|
|
162
372
|
end
|
|
163
373
|
|
|
164
|
-
# The candidate, in Markdown: its directory, or its commit,
|
|
165
|
-
#
|
|
374
|
+
# The candidate, in Markdown: its directory, or its commit, bare (see
|
|
375
|
+
# #described_baseline), and whether what's watched (Baseline#watched)
|
|
376
|
+
# has changes not committed.
|
|
166
377
|
def described_candidate(config, settings)
|
|
167
378
|
dir = settings.candidate_dir || config.root
|
|
168
379
|
head, found = git(dir, "rev-parse", "--short", "HEAD")
|
|
169
380
|
return Summary.code(dir) unless found.success?
|
|
170
381
|
|
|
171
|
-
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *config.
|
|
172
|
-
"#{
|
|
382
|
+
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *Baseline.new(config, settings).watched)
|
|
383
|
+
"#{head.chomp}#{" (with changes not committed)" unless clean.success?}"
|
|
173
384
|
end
|
|
174
385
|
|
|
175
386
|
# The baseline, for JSON: the ref that named it (the main branch,
|
|
@@ -181,23 +392,19 @@ module Duel
|
|
|
181
392
|
|
|
182
393
|
# (There's one: #compare checked.)
|
|
183
394
|
ref = baseline.ref #: String
|
|
184
|
-
named =
|
|
185
|
-
when nil then baseline.main_branch
|
|
186
|
-
when Baseline::LAST_RELEASE then ref
|
|
187
|
-
else settings.base
|
|
188
|
-
end
|
|
395
|
+
named = settings.base ? ref : baseline.main_branch
|
|
189
396
|
described.merge("ref" => named, "commit" => git(config.root, "rev-parse", "#{ref}^{commit}").first.chomp)
|
|
190
397
|
end
|
|
191
398
|
|
|
192
|
-
# The candidate, for JSON: its commit, and whether what
|
|
193
|
-
#
|
|
194
|
-
# its tree.
|
|
399
|
+
# The candidate, for JSON: its commit, and whether what's watched
|
|
400
|
+
# (Baseline#watched) has changes not committed (each nil outside a
|
|
401
|
+
# repository), and its tree.
|
|
195
402
|
def candidate_json(config, settings)
|
|
196
403
|
dir = File.expand_path(settings.candidate_dir || config.root)
|
|
197
404
|
head, found = git(dir, "rev-parse", "HEAD")
|
|
198
405
|
return {"commit" => nil, "uncommitted_changes" => nil, "root" => dir} unless found.success?
|
|
199
406
|
|
|
200
|
-
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *config.
|
|
407
|
+
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *Baseline.new(config, settings).watched)
|
|
201
408
|
{"commit" => head.chomp, "uncommitted_changes" => !clean.success?, "root" => dir}
|
|
202
409
|
end
|
|
203
410
|
|
|
@@ -253,6 +460,10 @@ module Duel
|
|
|
253
460
|
o.on("--markdown PATH", "Also write the report to PATH in Markdown, for a pull request comment (on GitHub",
|
|
254
461
|
"Actions, it's added to the job's summary too)") { |v| @markdown = v }
|
|
255
462
|
o.on("--json PATH", "Also write the report to PATH as JSON, for a program to read") { |v| @json = v }
|
|
463
|
+
o.on("--comment MODE", "When the workflow comments on a pull request: always, results (passed or failed),",
|
|
464
|
+
"failures, or never (default: c.comment, or failures)") { |v| settings.comment = CLI.comment_mode(v) }
|
|
465
|
+
o.on("--fail-on RUNS", "Which runs a regression fails: every_run, or pull_requests (on GitHub Actions, another",
|
|
466
|
+
"warns and passes) (default: c.fail_on, or every_run)") { |v| settings.fail_on = CLI.fail_on(v) }
|
|
256
467
|
o.on("--clean", "Remove tmp/duel, and the baseline's git worktree, and stop") { @clean = true }
|
|
257
468
|
o.on("-v", "--version", "Print the version") do
|
|
258
469
|
@out.puts "duel #{VERSION}"
|
data/lib/duel/config.rb
CHANGED
|
@@ -11,7 +11,8 @@ module Duel
|
|
|
11
11
|
# - a project with ext/ is built with `bundle exec rake compile` when
|
|
12
12
|
# a baseline is checked out
|
|
13
13
|
# - a change is compared with where it left the branch origin/HEAD
|
|
14
|
-
# points to (origin/main, say), when lib/ or ext/ changed
|
|
14
|
+
# points to (origin/main, say), when lib/ or ext/ changed, or what
|
|
15
|
+
# decides how they're measured (see #watch)
|
|
15
16
|
# - releases are tagged v*
|
|
16
17
|
#
|
|
17
18
|
# .duel.rb, at the root, changes what doesn't fit:
|
|
@@ -33,8 +34,11 @@ module Duel
|
|
|
33
34
|
# A glob, from the root, of the files defining benchmarks.
|
|
34
35
|
attr_accessor :benchmarks
|
|
35
36
|
|
|
36
|
-
# Paths, from the root, whose changes a comparison is
|
|
37
|
-
# of them changed since the baseline,
|
|
37
|
+
# Paths, from the root, of the code whose changes a comparison is
|
|
38
|
+
# about: when none of them changed since the baseline, nor what
|
|
39
|
+
# decides how it's measured (the benchmarks' directory, the
|
|
40
|
+
# configuration's file, and with each version on its own Ruby, the
|
|
41
|
+
# version files at the root), there's nothing to compare.
|
|
38
42
|
attr_accessor :watch
|
|
39
43
|
|
|
40
44
|
# The library to require from each version, as `require` takes it,
|
|
@@ -62,6 +66,104 @@ module Duel
|
|
|
62
66
|
# c.ruby = ->(root) { File.join(Dir.home, ".rubies", File.read(File.join(root, ".ruby-version")).strip, "bin", "ruby") }
|
|
63
67
|
attr_accessor :ruby
|
|
64
68
|
|
|
69
|
+
# When the workflow that `duel init` writes comments on a pull request
|
|
70
|
+
# with the report (`duel --comment` overrides it):
|
|
71
|
+
#
|
|
72
|
+
# - :always, on every run
|
|
73
|
+
# - :results, when a run passed or failed, but not with nothing
|
|
74
|
+
# compared
|
|
75
|
+
# - :failures (the default), only when a run failed or couldn't
|
|
76
|
+
# compare
|
|
77
|
+
# - :never
|
|
78
|
+
#
|
|
79
|
+
# It never edits a comment: each is a new one, so the pull request
|
|
80
|
+
# keeps them all. Once it has commented, a run that it wouldn't
|
|
81
|
+
# comment on still does, but :never's, when its status differs from
|
|
82
|
+
# the last comment's, so the last comment is never stale: after a
|
|
83
|
+
# failure, the first run that passes says so. Duel tells the workflow
|
|
84
|
+
# which to do (see CLI.comment).
|
|
85
|
+
#
|
|
86
|
+
# c.comment = :results
|
|
87
|
+
attr_reader :comment
|
|
88
|
+
|
|
89
|
+
# The values #comment takes.
|
|
90
|
+
COMMENTS = %i[always results failures never].freeze #: Array[comment]
|
|
91
|
+
|
|
92
|
+
def comment=(mode)
|
|
93
|
+
raise Error, "c.comment = #{mode.inspect}: give #{COMMENTS.map(&:inspect).join(", ")}" unless COMMENTS.include?(mode)
|
|
94
|
+
|
|
95
|
+
@comment = mode
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Which runs a regression fails (`duel --fail-on` overrides it):
|
|
99
|
+
#
|
|
100
|
+
# - :every_run (the default)
|
|
101
|
+
# - :pull_requests, only a pull request's (or a merge queue's), and a
|
|
102
|
+
# run outside GitHub Actions. On GitHub Actions, a regression in
|
|
103
|
+
# any other run (a push, the schedule, run by hand) passes, with a
|
|
104
|
+
# warning, and the workflow that `duel init` writes comments on a
|
|
105
|
+
# pushed commit with the report, as #comment says, which notifies
|
|
106
|
+
# its author.
|
|
107
|
+
#
|
|
108
|
+
# A run that couldn't compare fails either way.
|
|
109
|
+
#
|
|
110
|
+
# c.fail_on = :pull_requests
|
|
111
|
+
attr_reader :fail_on
|
|
112
|
+
|
|
113
|
+
# The values #fail_on takes.
|
|
114
|
+
FAIL_ON = %i[every_run pull_requests].freeze #: Array[fail_on]
|
|
115
|
+
|
|
116
|
+
def fail_on=(runs)
|
|
117
|
+
raise Error, "c.fail_on = #{runs.inspect}: give #{FAIL_ON.map(&:inspect).join(", ")}" unless FAIL_ON.include?(runs)
|
|
118
|
+
|
|
119
|
+
@fail_on = runs
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# The name of a label that accepts a regression in a pull request
|
|
123
|
+
# (or nil, the default, for none): on GitHub Actions, the workflow
|
|
124
|
+
# that `duel init` writes passes Duel the pull request's labels
|
|
125
|
+
# (DUEL_LABELS), and with this one, a regression passes, with a
|
|
126
|
+
# warning, and the report says it was accepted. For a push, it's the
|
|
127
|
+
# labels of the pull request that merged it, so where #push_base is
|
|
128
|
+
# :last_passed, the push is marked passed too. The labels are read
|
|
129
|
+
# when the workflow runs, so after adding the label, run it again.
|
|
130
|
+
#
|
|
131
|
+
# c.accept_label = "duel: accept"
|
|
132
|
+
attr_reader :accept_label
|
|
133
|
+
|
|
134
|
+
def accept_label=(label)
|
|
135
|
+
raise Error, "c.accept_label = #{label.inspect}: give a label's name, or nil" unless label.nil? || label.instance_of?(String)
|
|
136
|
+
|
|
137
|
+
@accept_label = label
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# What the workflow that `duel init` writes compares a push with:
|
|
141
|
+
#
|
|
142
|
+
# - :before (the default), the commit before the push
|
|
143
|
+
# - :last_passed, the last commit Duel passed, which the workflow
|
|
144
|
+
# marks with a branch, duel/passed/main for main (see
|
|
145
|
+
# Baseline#passed_branch),
|
|
146
|
+
# when a push passes, or has nothing to compare. Slowdowns too
|
|
147
|
+
# small for any one push to fail then add up until they fail one,
|
|
148
|
+
# and a push the workflow didn't compare (its run cancelled, say)
|
|
149
|
+
# is compared with the next. After a regression, every push is
|
|
150
|
+
# compared with the commit before it, until one is no slower, or
|
|
151
|
+
# the branch is moved to accept the regression. Without the branch,
|
|
152
|
+
# or where HEAD doesn't have its commit (after a force push, say),
|
|
153
|
+
# a push is compared with the commit before it.
|
|
154
|
+
#
|
|
155
|
+
# c.push_base = :last_passed
|
|
156
|
+
attr_reader :push_base
|
|
157
|
+
|
|
158
|
+
# The values #push_base takes.
|
|
159
|
+
PUSH_BASES = %i[before last_passed].freeze #: Array[push_base]
|
|
160
|
+
|
|
161
|
+
def push_base=(base)
|
|
162
|
+
raise Error, "c.push_base = #{base.inspect}: give #{PUSH_BASES.map(&:inspect).join(", ")}" unless PUSH_BASES.include?(base)
|
|
163
|
+
|
|
164
|
+
@push_base = base
|
|
165
|
+
end
|
|
166
|
+
|
|
65
167
|
# The metrics #metric added, in order (classes: a Duel::Metric
|
|
66
168
|
# subclass, or running totals' Snapshot), for a plugin's tests to
|
|
67
169
|
# look at (their .measures and .settings).
|
|
@@ -81,6 +183,9 @@ module Duel
|
|
|
81
183
|
@library = :conventional
|
|
82
184
|
@build = :conventional
|
|
83
185
|
@release_tags = "v*"
|
|
186
|
+
@comment = :failures
|
|
187
|
+
@fail_on = :every_run
|
|
188
|
+
@push_base = :before
|
|
84
189
|
@prepares = SCOPES.to_h { |scope| [scope, []] }
|
|
85
190
|
@metrics = []
|
|
86
191
|
@in_workers = []
|
data/lib/duel/init.rb
CHANGED
|
@@ -10,15 +10,17 @@ require_relative "../duel"
|
|
|
10
10
|
module Duel
|
|
11
11
|
# `duel init`: sets Duel up in a gem or an app that isn't a Rails app
|
|
12
12
|
# (duel-rails's generator sets a Rails app up). It writes .duel.rb, a
|
|
13
|
-
# benchmark to start from, and
|
|
13
|
+
# benchmark to start from, and GitHub Actions workflows (Duel's, and
|
|
14
|
+
# one that comments on a pull request with its report), or with
|
|
15
|
+
# --gitlab, GitLab CI jobs to include, adds a Rake
|
|
14
16
|
# task to the Rakefile, and adds StackProf to the Gemfile (for profiles
|
|
15
17
|
# of regressions, on CRuby). It overwrites nothing without --force, and
|
|
16
18
|
# writes nothing with --dry-run.
|
|
17
19
|
# @api private
|
|
18
20
|
class Init
|
|
19
21
|
USAGE = "Usage: duel init [options]\n\n" \
|
|
20
|
-
"Sets Duel up in this project: .duel.rb, a benchmark to start from in duel/,
|
|
21
|
-
"a Rake task, and StackProf in the Gemfile.\n\n"
|
|
22
|
+
"Sets Duel up in this project: .duel.rb, a benchmark to start from in duel/, GitHub Actions workflows\n" \
|
|
23
|
+
"(or GitLab CI jobs), a Rake task, and StackProf in the Gemfile.\n\n"
|
|
22
24
|
|
|
23
25
|
TEMPLATES = File.join(__dir__, "templates")
|
|
24
26
|
|
|
@@ -61,8 +63,13 @@ module Duel
|
|
|
61
63
|
|
|
62
64
|
# The files it writes, from the project's root, to what's in them.
|
|
63
65
|
def files
|
|
64
|
-
|
|
65
|
-
{".duel.rb" => render("duel.rb"), benchmark_file => render("benchmark_duel.rb")
|
|
66
|
+
top = Pathname(File.realpath(repository)).relative_path_from(File.realpath(@root))
|
|
67
|
+
written = {".duel.rb" => render("duel.rb"), benchmark_file => render("benchmark_duel.rb")}
|
|
68
|
+
return written.merge(top.join(gitlab_file).to_s => render("duel.gitlab-ci.yml")) if @gitlab
|
|
69
|
+
|
|
70
|
+
workflows = top.join(".github", "workflows")
|
|
71
|
+
written.merge(workflows.join(workflow_name).to_s => render("perf.yml"),
|
|
72
|
+
workflows.join(workflow_name.sub(/\.yml\z/, "-comment.yml")).to_s => render("perf-comment.yml"))
|
|
66
73
|
end
|
|
67
74
|
|
|
68
75
|
private
|
|
@@ -72,6 +79,7 @@ module Duel
|
|
|
72
79
|
o.banner = USAGE
|
|
73
80
|
o.on("--force", "Overwrite files that exist") { @force = true }
|
|
74
81
|
o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
|
|
82
|
+
o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
|
|
75
83
|
o.on("-h", "--help", "Print this help") do
|
|
76
84
|
@out.puts o
|
|
77
85
|
throw :help
|
|
@@ -156,14 +164,27 @@ module Duel
|
|
|
156
164
|
steps << "Set c.library in .duel.rb to what each version's workers require." unless required
|
|
157
165
|
steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
|
|
158
166
|
steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
|
|
159
|
-
steps <<
|
|
167
|
+
steps << if @gitlab
|
|
168
|
+
"Include #{gitlab_file} from .gitlab-ci.yml, and add the tokens it says to, for comments. On a merge request, the job " \
|
|
169
|
+
"compares the code with the branch it targets."
|
|
170
|
+
else
|
|
171
|
+
"Commit and push. On a pull request, the workflow compares the code with the branch it targets, and comments with the " \
|
|
172
|
+
"report when a benchmark fails (c.comment in .duel.rb says when)."
|
|
173
|
+
end
|
|
160
174
|
"\nNext:\n#{steps.each_with_index.map { |step, i| " #{i + 1}. #{step}\n" }.join}"
|
|
161
175
|
end
|
|
162
176
|
|
|
163
177
|
def render(name) = ERB.new(File.read(File.join(TEMPLATES, "#{name}.erb")), trim_mode: "-").result(binding)
|
|
164
178
|
|
|
179
|
+
# The project's configuration, as the conventions have it.
|
|
180
|
+
def config = Config.new(@root)
|
|
181
|
+
|
|
165
182
|
# The library the gemspec names (see Config#library), or nil.
|
|
166
|
-
def library =
|
|
183
|
+
def library = config.library
|
|
184
|
+
|
|
185
|
+
# The glob release tags match (Config#release_tags), for the workflow
|
|
186
|
+
# to find the last release as Duel does.
|
|
187
|
+
def release_tags = config.release_tags
|
|
167
188
|
|
|
168
189
|
# The library, without a gemspec: lib/'s only Ruby file, or nil.
|
|
169
190
|
def guess
|
|
@@ -207,5 +228,26 @@ module Duel
|
|
|
207
228
|
|
|
208
229
|
# What marks the workflow's comment on a pull request.
|
|
209
230
|
def marker = ["duel", working_directory].compact.join(" ")
|
|
231
|
+
|
|
232
|
+
# Where the GitLab CI jobs are, from the repository's root:
|
|
233
|
+
# .gitlab/duel.gitlab-ci.yml, or for a project in a subdirectory,
|
|
234
|
+
# .gitlab/duel-<its path>.gitlab-ci.yml.
|
|
235
|
+
def gitlab_file = File.join(".gitlab", "#{["duel", working_directory&.tr("/", "-")].compact.join("-")}.gitlab-ci.yml")
|
|
236
|
+
|
|
237
|
+
# The GitLab CI job's name: duel, or for a project in a subdirectory,
|
|
238
|
+
# duel:<its path>, so each of a monorepo's projects has its own.
|
|
239
|
+
def gitlab_job = ["duel", working_directory].compact.join(":")
|
|
240
|
+
|
|
241
|
+
# The Ruby image's tag for GitLab CI: the version .ruby-version names,
|
|
242
|
+
# exactly (so it's the Ruby that c.ruby = :version_file asks for), or
|
|
243
|
+
# this one's minor version.
|
|
244
|
+
def gitlab_ruby
|
|
245
|
+
named = File.read(File.join(@root, ".ruby-version")).split.first if version_file
|
|
246
|
+
named&.delete_prefix("ruby-") || RUBY_VERSION.rpartition(".").first
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# What names the branches that mark the last commit Duel passed in a
|
|
250
|
+
# push, before the branch pushed's name (see Baseline#passed_branch).
|
|
251
|
+
def passed_branch = ["duel/passed", working_directory&.tr("/", "-")].compact.join("-")
|
|
210
252
|
end
|
|
211
253
|
end
|