duel 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +134 -1
- data/README.md +42 -7
- data/docs/benchmarks.md +2 -1
- data/docs/ci.md +630 -61
- data/docs/configuration.md +91 -4
- data/docs/how-it-works.md +34 -5
- data/docs/json.md +2 -0
- data/docs/plugins.md +2 -2
- data/docs/troubleshooting.md +14 -0
- data/lib/duel/baseline.rb +34 -8
- data/lib/duel/cli.rb +174 -29
- data/lib/duel/config.rb +84 -6
- data/lib/duel/context.rb +6 -5
- data/lib/duel/init.rb +45 -7
- data/lib/duel/json_report.rb +4 -3
- data/lib/duel/lock.rb +52 -8
- data/lib/duel/native.rb +20 -0
- data/lib/duel/resource.rb +37 -2
- data/lib/duel/runner.rb +98 -9
- data/lib/duel/summary.rb +8 -3
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +167 -0
- data/lib/duel/templates/duel.rb.erb +13 -0
- data/lib/duel/templates/perf-comment.yml.erb +59 -0
- data/lib/duel/templates/perf.yml.erb +196 -46
- data/lib/duel/testing.rb +2 -2
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +1 -1
- data/sig/duel/baseline.rbs +14 -1
- data/sig/duel/cli.rbs +64 -2
- data/sig/duel/config.rbs +39 -0
- data/sig/duel/init.rbs +16 -0
- data/sig/duel/lock.rbs +11 -0
- data/sig/duel/native.rbs +2 -0
- data/sig/duel/resource.rbs +10 -1
- data/sig/duel/runner.rbs +50 -3
- data/sig/duel/summary.rbs +6 -1
- metadata +3 -1
data/lib/duel/cli.rb
CHANGED
|
@@ -18,7 +18,8 @@ module Duel
|
|
|
18
18
|
# On GitHub Actions (GITHUB_STEP_SUMMARY), it adds its report to the
|
|
19
19
|
# job's summary, in Markdown (see Summary), and tells the step's
|
|
20
20
|
# outputs (GITHUB_OUTPUT) what to do about the comment on a pull
|
|
21
|
-
# request (see .comment)
|
|
21
|
+
# request (see .comment), and for a push, whether to mark it passed
|
|
22
|
+
# (see Config#push_base). --markdown writes the Markdown to a file, and
|
|
22
23
|
# --json the report as JSON (see JSONReport), whatever the run's
|
|
23
24
|
# outcome, but not for a command line that isn't valid.
|
|
24
25
|
# @api private
|
|
@@ -32,6 +33,10 @@ module Duel
|
|
|
32
33
|
# Where GitHub Actions has a step set its outputs.
|
|
33
34
|
STEP_OUTPUT = "GITHUB_OUTPUT"
|
|
34
35
|
|
|
36
|
+
# Where else to write them, as variables (DUEL_COMMENT, DUEL_STATUS,
|
|
37
|
+
# DUEL_PASSED): on GitLab CI, a dotenv report, for the jobs after.
|
|
38
|
+
OUTPUT = "DUEL_OUTPUT"
|
|
39
|
+
|
|
35
40
|
# The statuses (as the JSON report has them) that the workflow
|
|
36
41
|
# comments on, by when it does (Config#comment, but :never, which
|
|
37
42
|
# comments on none).
|
|
@@ -41,6 +46,16 @@ module Duel
|
|
|
41
46
|
# has it: passed, failed (or couldn't compare), or neutral.
|
|
42
47
|
MARKED = {"passed" => "passed", "failed" => "failed", "error" => "failed", "nothing_to_compare" => "neutral"}.freeze
|
|
43
48
|
|
|
49
|
+
# The GitHub Actions events of a pull request, as GITHUB_EVENT_NAME
|
|
50
|
+
# has them: a merge queue's too (see Config#fail_on). (A GitLab merge
|
|
51
|
+
# request's is pull_request: see #event.)
|
|
52
|
+
PULL_REQUESTS = %w[pull_request pull_request_target merge_group].freeze
|
|
53
|
+
|
|
54
|
+
# The exit status of a run that was slower outside a merge request on
|
|
55
|
+
# GitLab CI, where only a merge request fails (Config#fail_on): the
|
|
56
|
+
# job that `duel init --gitlab` writes lets it fail, as a warning.
|
|
57
|
+
WARNED = 3
|
|
58
|
+
|
|
44
59
|
# The variables that set options, to their values, and the one that
|
|
45
60
|
# sets a flag by being set at all.
|
|
46
61
|
ENV_OPTIONS = {"DUEL_BASE" => "--base", "DUEL_FILTER" => "--filter"}.freeze
|
|
@@ -54,13 +69,14 @@ module Duel
|
|
|
54
69
|
end
|
|
55
70
|
|
|
56
71
|
# Runs, and returns the exit status: 0 not slower (or nothing to compare),
|
|
57
|
-
# 1 slower (or a benchmark failed), 2 couldn't compare
|
|
72
|
+
# 1 slower (or a benchmark failed), 2 couldn't compare, 3 slower but
|
|
73
|
+
# warned (see WARNED).
|
|
58
74
|
def run
|
|
59
75
|
return Init.new(@argv.drop(1), out: @out, err: @err).run if @argv.first == "init"
|
|
60
76
|
|
|
61
77
|
settings = @settings = catch(:version) { parse } or return 0
|
|
62
78
|
config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
63
|
-
Lock.hold(
|
|
79
|
+
Lock.hold(config.state) { compare(config, settings) }
|
|
64
80
|
rescue Error => e
|
|
65
81
|
@err.puts "duel: #{e}"
|
|
66
82
|
unfinished(Summary::FAILED_MARK, "couldn't compare", "error", e.to_s, 2)
|
|
@@ -91,9 +107,17 @@ module Duel
|
|
|
91
107
|
|
|
92
108
|
# The mode --comment names (see Config#comment), or an invalid
|
|
93
109
|
# argument.
|
|
94
|
-
def self.comment_mode(name)
|
|
95
|
-
|
|
96
|
-
|
|
110
|
+
def self.comment_mode(name) = named(name, Config::COMMENTS)
|
|
111
|
+
|
|
112
|
+
# The runs --fail-on names (see Config#fail_on), or an invalid
|
|
113
|
+
# argument.
|
|
114
|
+
def self.fail_on(name) = named(name, Config::FAIL_ON)
|
|
115
|
+
|
|
116
|
+
# Which of +values+ (symbols) an option's argument names, or an
|
|
117
|
+
# invalid argument.
|
|
118
|
+
def self.named(name, values)
|
|
119
|
+
values.find { |value| value.to_s == name } or
|
|
120
|
+
raise OptionParser::InvalidArgument, "#{name} (give #{values.join(", ")})"
|
|
97
121
|
end
|
|
98
122
|
|
|
99
123
|
# The settings, from the environment and then the options.
|
|
@@ -101,6 +125,11 @@ module Duel
|
|
|
101
125
|
settings = Settings.defaults
|
|
102
126
|
settings.jobs = [Etc.nprocessors / 2, 1].max
|
|
103
127
|
parser(settings).parse!(env_options + @argv)
|
|
128
|
+
branch = variable("GITHUB_REF_NAME", "CI_COMMIT_BRANCH")
|
|
129
|
+
base = variable("DUEL_BASE")
|
|
130
|
+
settings.pushed = branch if pushed? && !base.nil? && settings.base == base
|
|
131
|
+
target = variable("GITHUB_BASE_REF", "CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
|
|
132
|
+
settings.target = "origin/#{target}" if target && pull_request?
|
|
104
133
|
settings.config_path ||= (".duel.rb" if File.exist?(".duel.rb"))
|
|
105
134
|
settings.config_path &&= File.expand_path(settings.config_path)
|
|
106
135
|
settings
|
|
@@ -117,9 +146,12 @@ module Duel
|
|
|
117
146
|
return 0
|
|
118
147
|
end
|
|
119
148
|
if settings.base_dir.nil? && baseline.ref.nil?
|
|
149
|
+
return nothing("no commit to compare against", "no commit before this push to compare against") if settings.pushed
|
|
150
|
+
|
|
120
151
|
return nothing("no release to compare against", "no release tag (#{config.release_tags}) before HEAD to compare against")
|
|
121
152
|
end
|
|
122
153
|
if !settings.force && baseline.unchanged?
|
|
154
|
+
@passed = marks_passed?(config, settings)
|
|
123
155
|
return nothing("nothing to compare",
|
|
124
156
|
"#{baseline.watched.join(", ")} unchanged since the baseline, so there's nothing to compare (--force to anyway)")
|
|
125
157
|
end
|
|
@@ -130,12 +162,14 @@ module Duel
|
|
|
130
162
|
uneven(base_root, candidate)
|
|
131
163
|
runner = Runner.new(config, settings, base_root, out: @out)
|
|
132
164
|
results = runner.run
|
|
133
|
-
|
|
165
|
+
passed = runner.passed?(results)
|
|
166
|
+
status = passed ? 0 : slower(config, settings)
|
|
167
|
+
@passed = (passed || accepted?(config)) && marks_passed?(config, settings)
|
|
134
168
|
publish(
|
|
135
|
-
|
|
169
|
+
passed ? "passed" : "failed",
|
|
136
170
|
-> {
|
|
137
171
|
Summary.new(runner, results, settings: settings, baseline: described_baseline(config, settings, baseline),
|
|
138
|
-
candidate: described_candidate(config, settings)).markdown
|
|
172
|
+
candidate: described_candidate(config, settings), note: @note).markdown
|
|
139
173
|
},
|
|
140
174
|
->(comment) {
|
|
141
175
|
JSONReport.new(runner, results, settings: settings, exit: status, comment: comment,
|
|
@@ -145,6 +179,107 @@ module Duel
|
|
|
145
179
|
status
|
|
146
180
|
end
|
|
147
181
|
|
|
182
|
+
# The exit status of a run that was slower (or whose benchmark
|
|
183
|
+
# failed): 1, or 0 where the pull request's label accepts it
|
|
184
|
+
# (Config#accept_label), or on GitHub Actions outside a pull request,
|
|
185
|
+
# where c.fail_on (or --fail-on) says only a pull request fails, with
|
|
186
|
+
# a warning that says so, and a note in the report.
|
|
187
|
+
def slower(config, settings)
|
|
188
|
+
if accepted?(config)
|
|
189
|
+
# (Set: see #accepted?.)
|
|
190
|
+
label = config.accept_label #: String
|
|
191
|
+
return passing("accepted with the #{label} label", "Accepted with the #{Summary.code(label)} label.")
|
|
192
|
+
end
|
|
193
|
+
return 1 unless pull_requests_only?(config, settings)
|
|
194
|
+
return 1 unless outside_pull_request?
|
|
195
|
+
|
|
196
|
+
# (Outside a pull request on CI, so there's one: see #event.)
|
|
197
|
+
run = event #: String
|
|
198
|
+
passing("only a pull request fails (c.fail_on), so this #{run} passes",
|
|
199
|
+
"Only a pull request fails (#{Summary.code("c.fail_on")}), so this #{run} passes.", gitlab? ? WARNED : 0)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Passes a run that was slower (or on GitLab CI, warns: +exit+), as
|
|
203
|
+
# +why+ says, in a warning (on GitHub Actions, an annotation), and as
|
|
204
|
+
# +note+ says in the report.
|
|
205
|
+
def passing(why, note, exit = 0)
|
|
206
|
+
@out.puts "#{gitlab? ? "duel: " : "::warning::Duel: "}slower, but #{why}"
|
|
207
|
+
@note = note
|
|
208
|
+
exit
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Whether the pull request's labels have the one that accepts a
|
|
212
|
+
# regression (Config#accept_label).
|
|
213
|
+
def accepted?(config)
|
|
214
|
+
label = config.accept_label
|
|
215
|
+
!label.nil? && labels.include?(label)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
# The pull request's labels: DUEL_LABELS (a JSON array of their names)
|
|
219
|
+
# on GitHub Actions, or a GitLab merge request's
|
|
220
|
+
# (CI_MERGE_REQUEST_LABELS, by commas), or none, where neither is
|
|
221
|
+
# set, or DUEL_LABELS isn't a JSON array.
|
|
222
|
+
def labels
|
|
223
|
+
set = variables
|
|
224
|
+
return set.fetch("CI_MERGE_REQUEST_LABELS").split(",") if !set.key?("DUEL_LABELS") && set.key?("CI_MERGE_REQUEST_LABELS")
|
|
225
|
+
return [] unless set.key?("DUEL_LABELS")
|
|
226
|
+
|
|
227
|
+
names = JSON.parse(set.fetch("DUEL_LABELS"))
|
|
228
|
+
names.instance_of?(Array) ? names : []
|
|
229
|
+
rescue JSON::ParserError
|
|
230
|
+
[]
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# Whether the workflow marks a push that passed (or had nothing to
|
|
234
|
+
# compare) as the last commit Duel passed: where Config#push_base
|
|
235
|
+
# says to compare a push with that commit.
|
|
236
|
+
def marks_passed?(config, settings) = !settings.pushed.nil? && config.push_base == :last_passed
|
|
237
|
+
|
|
238
|
+
# Whether a regression fails only a pull request (--fail-on, or else
|
|
239
|
+
# Config#fail_on).
|
|
240
|
+
def pull_requests_only?(config, settings) = (settings.fail_on || config.fail_on) == :pull_requests
|
|
241
|
+
|
|
242
|
+
# What a run on CI is for, as GitHub Actions names it
|
|
243
|
+
# (GITHUB_EVENT_NAME): push, pull_request, schedule, say. On GitLab
|
|
244
|
+
# CI, its pipeline's source (CI_PIPELINE_SOURCE), as GitHub would
|
|
245
|
+
# name it: a merge request's is pull_request. Outside CI, nil.
|
|
246
|
+
def event
|
|
247
|
+
github = variable("GITHUB_EVENT_NAME")
|
|
248
|
+
return github if github
|
|
249
|
+
return unless gitlab?
|
|
250
|
+
|
|
251
|
+
source = variable("CI_PIPELINE_SOURCE")
|
|
252
|
+
(source == "merge_request_event") ? "pull_request" : source
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Whether this is a run on GitLab CI.
|
|
256
|
+
def gitlab? = variables.key?("GITLAB_CI")
|
|
257
|
+
|
|
258
|
+
# Whether this is a run on CI for something other than a pull request
|
|
259
|
+
# (PULL_REQUESTS): a push, say.
|
|
260
|
+
def outside_pull_request?
|
|
261
|
+
run = event
|
|
262
|
+
!run.nil? && !PULL_REQUESTS.include?(run)
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
# Whether this is a run on CI for a pull request (or a merge queue:
|
|
266
|
+
# see PULL_REQUESTS).
|
|
267
|
+
def pull_request? = PULL_REQUESTS.include?(event)
|
|
268
|
+
|
|
269
|
+
# Whether the baseline is a merge queue's base: on GitHub Actions for
|
|
270
|
+
# a merge queue, as DUEL_BASE gives it.
|
|
271
|
+
def queued?(settings) = event == "merge_group" && settings.base == variable("DUEL_BASE")
|
|
272
|
+
|
|
273
|
+
# Whether this is a run on CI for a push.
|
|
274
|
+
def pushed? = event == "push"
|
|
275
|
+
|
|
276
|
+
# The first of the variables +names+ that's set, or nil.
|
|
277
|
+
def variable(*names)
|
|
278
|
+
set = variables
|
|
279
|
+
name = names.find { |n| set.key?(n) }
|
|
280
|
+
set.fetch(name) if name
|
|
281
|
+
end
|
|
282
|
+
|
|
148
283
|
# Says there's nothing to compare (+title+, in Markdown), and +why+,
|
|
149
284
|
# and passes.
|
|
150
285
|
def nothing(title, why)
|
|
@@ -161,14 +296,19 @@ module Duel
|
|
|
161
296
|
exit
|
|
162
297
|
end
|
|
163
298
|
|
|
164
|
-
# What the workflow does about its comment on a pull request
|
|
165
|
-
# .comment), for a run whose +status+ is the
|
|
166
|
-
# --comment says, or else the configuration, or the
|
|
167
|
-
# configuration didn't load.
|
|
299
|
+
# What the workflow does about its comment on a pull request, or on
|
|
300
|
+
# a pushed commit (see .comment), for a run whose +status+ is the
|
|
301
|
+
# JSON report's: as --comment says, or else the configuration, or the
|
|
302
|
+
# default where the configuration didn't load. It comments on a push
|
|
303
|
+
# only where a regression in one passes (Config#fail_on), so the
|
|
304
|
+
# comment says what the check doesn't.
|
|
168
305
|
def commenting(status)
|
|
169
306
|
# (Parsed, so set: see #run.)
|
|
170
307
|
settings = @settings #: Settings
|
|
171
|
-
|
|
308
|
+
config = @config || Config.new
|
|
309
|
+
return "none" if pushed? && !pull_requests_only?(config, settings)
|
|
310
|
+
|
|
311
|
+
CLI.comment(settings.comment || config.comment, status)
|
|
172
312
|
end
|
|
173
313
|
|
|
174
314
|
# Writes the reports of a run whose +status+ is the JSON report's
|
|
@@ -177,13 +317,17 @@ module Duel
|
|
|
177
317
|
# --json names, and the Markdown +markdown+ makes to the file
|
|
178
318
|
# --markdown names, and added to the job's summary on GitHub Actions
|
|
179
319
|
# (STEP_SUMMARY). Each is only made if it's written. On GitHub
|
|
180
|
-
# Actions, it also sets the step's outputs (STEP_OUTPUT) for the
|
|
181
|
-
# that
|
|
182
|
-
#
|
|
320
|
+
# Actions, it also sets the step's outputs (STEP_OUTPUT) for the jobs
|
|
321
|
+
# that comment: +comment+, and +status+ as the comment's marker has
|
|
322
|
+
# it (MARKED), and for the job that marks a push passed, "passed"
|
|
323
|
+
# (see #marks_passed?). Where DUEL_OUTPUT names a file (OUTPUT), it
|
|
324
|
+
# writes them there too, as variables.
|
|
183
325
|
def publish(status, markdown, json)
|
|
184
326
|
comment = commenting(status)
|
|
185
327
|
set = variables
|
|
186
|
-
|
|
328
|
+
outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed=true\n" if @passed}"
|
|
329
|
+
File.write(set.fetch(STEP_OUTPUT), outputs, mode: "a") if set.key?(STEP_OUTPUT)
|
|
330
|
+
File.write(set.fetch(OUTPUT), outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if set.key?(OUTPUT)
|
|
187
331
|
path = @json
|
|
188
332
|
write(path, "#{JSON.pretty_generate(json.call(comment))}\n") if path
|
|
189
333
|
path = @markdown
|
|
@@ -204,7 +348,9 @@ module Duel
|
|
|
204
348
|
# The baseline, in Markdown: its directory, or its commit, bare (so
|
|
205
349
|
# GitHub links it) and followed by a space, never punctuation, and
|
|
206
350
|
# what chose it: the main branch, the push (on GitHub Actions, the
|
|
207
|
-
# commit before it, from DUEL_BASE
|
|
351
|
+
# commit before it, from DUEL_BASE, or the last commit Duel passed),
|
|
352
|
+
# the merge queue (its base, from DUEL_BASE), the last release, or
|
|
353
|
+
# --base.
|
|
208
354
|
def described_baseline(config, settings, baseline)
|
|
209
355
|
dir = settings.base_dir
|
|
210
356
|
return Summary.code(dir) if dir
|
|
@@ -212,15 +358,13 @@ module Duel
|
|
|
212
358
|
# (There's one: #compare checked.)
|
|
213
359
|
ref = baseline.ref #: String
|
|
214
360
|
commit = git(config.root, "rev-parse", "--short", "#{ref}^{commit}").first.chomp
|
|
215
|
-
set = variables
|
|
216
361
|
chosen = case settings.base
|
|
217
362
|
when nil then "where HEAD left #{Summary.code(baseline.main_branch)}"
|
|
218
363
|
when Baseline::LAST_RELEASE then "the last release, #{Summary.code(ref)}"
|
|
219
364
|
else
|
|
220
|
-
pushed
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
elsif pushed then "the commit before the push"
|
|
365
|
+
if settings.pushed then (ref == baseline.passed) ? "the last commit Duel passed" : "the commit before the push"
|
|
366
|
+
elsif queued?(settings) then "the merge queue's base"
|
|
367
|
+
elsif !ref.match?(/\A\h+\z/) then Summary.code(ref)
|
|
224
368
|
else "from --base"
|
|
225
369
|
end
|
|
226
370
|
end
|
|
@@ -248,11 +392,7 @@ module Duel
|
|
|
248
392
|
|
|
249
393
|
# (There's one: #compare checked.)
|
|
250
394
|
ref = baseline.ref #: String
|
|
251
|
-
named =
|
|
252
|
-
when nil then baseline.main_branch
|
|
253
|
-
when Baseline::LAST_RELEASE then ref
|
|
254
|
-
else settings.base
|
|
255
|
-
end
|
|
395
|
+
named = settings.base ? ref : baseline.main_branch
|
|
256
396
|
described.merge("ref" => named, "commit" => git(config.root, "rev-parse", "#{ref}^{commit}").first.chomp)
|
|
257
397
|
end
|
|
258
398
|
|
|
@@ -311,6 +451,9 @@ module Duel
|
|
|
311
451
|
o.on("--jobs N", Integer, "Benchmarks measured at once (default: half the CPUs)") { |v| settings.jobs = v }
|
|
312
452
|
o.on("--processes N", Integer, "Worker processes per version (default: 3)") { |v| settings.processes = v }
|
|
313
453
|
o.on("--timeout SECONDS", Float, "Most a worker may take to load or answer (default: 60)") { |v| settings.timeout = v }
|
|
454
|
+
o.on("--time-limit SECONDS", Float, "Most a run measures for, then fails what's unmeasured or looks worse (default: 1800)") do |v|
|
|
455
|
+
settings.time_limit = v
|
|
456
|
+
end
|
|
314
457
|
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: 1.02)") { |v| settings.limit = v }
|
|
315
458
|
o.on("--warm SECONDS", Float, "Warm-up per benchmark (default: 5 on JRuby and TruffleRuby, 2 with YJIT on, else 0)") { |v| settings.warm = v }
|
|
316
459
|
o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
|
|
@@ -322,6 +465,8 @@ module Duel
|
|
|
322
465
|
o.on("--json PATH", "Also write the report to PATH as JSON, for a program to read") { |v| @json = v }
|
|
323
466
|
o.on("--comment MODE", "When the workflow comments on a pull request: always, results (passed or failed),",
|
|
324
467
|
"failures, or never (default: c.comment, or failures)") { |v| settings.comment = CLI.comment_mode(v) }
|
|
468
|
+
o.on("--fail-on RUNS", "Which runs a regression fails: every_run, or pull_requests (on GitHub Actions, another",
|
|
469
|
+
"warns and passes) (default: c.fail_on, or every_run)") { |v| settings.fail_on = CLI.fail_on(v) }
|
|
325
470
|
o.on("--clean", "Remove tmp/duel, and the baseline's git worktree, and stop") { @clean = true }
|
|
326
471
|
o.on("-v", "--version", "Print the version") do
|
|
327
472
|
@out.puts "duel #{VERSION}"
|
data/lib/duel/config.rb
CHANGED
|
@@ -31,6 +31,14 @@ module Duel
|
|
|
31
31
|
# Where the project is (and its config file).
|
|
32
32
|
attr_reader :root
|
|
33
33
|
|
|
34
|
+
# Where Duel keeps what it keeps for the project, tmp/duel: the lock
|
|
35
|
+
# that keeps two runs from sharing it (see Lock), what prepare blocks
|
|
36
|
+
# build once (Resource#cached), in cache/, and the baseline's checkout
|
|
37
|
+
# where it can't go next to the repository (see Baseline#checkout), in
|
|
38
|
+
# base/. A run removes the lock when it ends, and what that leaves
|
|
39
|
+
# empty. `duel --clean` removes the rest.
|
|
40
|
+
def state = File.join(root, "tmp", "duel")
|
|
41
|
+
|
|
34
42
|
# A glob, from the root, of the files defining benchmarks.
|
|
35
43
|
attr_accessor :benchmarks
|
|
36
44
|
|
|
@@ -95,6 +103,75 @@ module Duel
|
|
|
95
103
|
@comment = mode
|
|
96
104
|
end
|
|
97
105
|
|
|
106
|
+
# Which runs a regression fails (`duel --fail-on` overrides it):
|
|
107
|
+
#
|
|
108
|
+
# - :every_run (the default)
|
|
109
|
+
# - :pull_requests, only a pull request's (or a merge queue's), and a
|
|
110
|
+
# run outside GitHub Actions. On GitHub Actions, a regression in
|
|
111
|
+
# any other run (a push, the schedule, run by hand) passes, with a
|
|
112
|
+
# warning, and the workflow that `duel init` writes comments on a
|
|
113
|
+
# pushed commit with the report, as #comment says, which notifies
|
|
114
|
+
# its author.
|
|
115
|
+
#
|
|
116
|
+
# A run that couldn't compare fails either way.
|
|
117
|
+
#
|
|
118
|
+
# c.fail_on = :pull_requests
|
|
119
|
+
attr_reader :fail_on
|
|
120
|
+
|
|
121
|
+
# The values #fail_on takes.
|
|
122
|
+
FAIL_ON = %i[every_run pull_requests].freeze #: Array[fail_on]
|
|
123
|
+
|
|
124
|
+
def fail_on=(runs)
|
|
125
|
+
raise Error, "c.fail_on = #{runs.inspect}: give #{FAIL_ON.map(&:inspect).join(", ")}" unless FAIL_ON.include?(runs)
|
|
126
|
+
|
|
127
|
+
@fail_on = runs
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# The name of a label that accepts a regression in a pull request
|
|
131
|
+
# (or nil, the default, for none): on GitHub Actions, the workflow
|
|
132
|
+
# that `duel init` writes passes Duel the pull request's labels
|
|
133
|
+
# (DUEL_LABELS), and with this one, a regression passes, with a
|
|
134
|
+
# warning, and the report says it was accepted. For a push, it's the
|
|
135
|
+
# labels of the pull request that merged it, so where #push_base is
|
|
136
|
+
# :last_passed, the push is marked passed too. The labels are read
|
|
137
|
+
# when the workflow runs, so after adding the label, run it again.
|
|
138
|
+
#
|
|
139
|
+
# c.accept_label = "duel: accept"
|
|
140
|
+
attr_reader :accept_label
|
|
141
|
+
|
|
142
|
+
def accept_label=(label)
|
|
143
|
+
raise Error, "c.accept_label = #{label.inspect}: give a label's name, or nil" unless label.nil? || label.instance_of?(String)
|
|
144
|
+
|
|
145
|
+
@accept_label = label
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
# What the workflow that `duel init` writes compares a push with:
|
|
149
|
+
#
|
|
150
|
+
# - :before (the default), the commit before the push
|
|
151
|
+
# - :last_passed, the last commit Duel passed, which the workflow
|
|
152
|
+
# marks with a branch, duel/passed/main for main (see
|
|
153
|
+
# Baseline#passed_branch),
|
|
154
|
+
# when a push passes, or has nothing to compare. Slowdowns too
|
|
155
|
+
# small for any one push to fail then add up until they fail one,
|
|
156
|
+
# and a push the workflow didn't compare (its run cancelled, say)
|
|
157
|
+
# is compared with the next. After a regression, every push is
|
|
158
|
+
# compared with the commit before it, until one is no slower, or
|
|
159
|
+
# the branch is moved to accept the regression. Without the branch,
|
|
160
|
+
# or where HEAD doesn't have its commit (after a force push, say),
|
|
161
|
+
# a push is compared with the commit before it.
|
|
162
|
+
#
|
|
163
|
+
# c.push_base = :last_passed
|
|
164
|
+
attr_reader :push_base
|
|
165
|
+
|
|
166
|
+
# The values #push_base takes.
|
|
167
|
+
PUSH_BASES = %i[before last_passed].freeze #: Array[push_base]
|
|
168
|
+
|
|
169
|
+
def push_base=(base)
|
|
170
|
+
raise Error, "c.push_base = #{base.inspect}: give #{PUSH_BASES.map(&:inspect).join(", ")}" unless PUSH_BASES.include?(base)
|
|
171
|
+
|
|
172
|
+
@push_base = base
|
|
173
|
+
end
|
|
174
|
+
|
|
98
175
|
# The metrics #metric added, in order (classes: a Duel::Metric
|
|
99
176
|
# subclass, or running totals' Snapshot), for a plugin's tests to
|
|
100
177
|
# look at (their .measures and .settings).
|
|
@@ -115,6 +192,8 @@ module Duel
|
|
|
115
192
|
@build = :conventional
|
|
116
193
|
@release_tags = "v*"
|
|
117
194
|
@comment = :failures
|
|
195
|
+
@fail_on = :every_run
|
|
196
|
+
@push_base = :before
|
|
118
197
|
@prepares = SCOPES.to_h { |scope| [scope, []] }
|
|
119
198
|
@metrics = []
|
|
120
199
|
@in_workers = []
|
|
@@ -241,18 +320,17 @@ module Duel
|
|
|
241
320
|
# stopped when it's done with. The block returns the environment
|
|
242
321
|
# variables it gets: a worker's, under its version's.
|
|
243
322
|
#
|
|
244
|
-
# A version's (named "
|
|
245
|
-
#
|
|
246
|
-
#
|
|
247
|
-
#
|
|
248
|
-
# its process benchmarks').
|
|
323
|
+
# A version's (named "base" or "head", its #root the version's tree
|
|
324
|
+
# and its #ruby the Ruby it runs on) is prepared once for a run, in
|
|
325
|
+
# the runner, before any of its workers start: installing its bundle,
|
|
326
|
+
# say. Its environment is its workers' (and its process benchmarks').
|
|
249
327
|
#
|
|
250
328
|
# c.prepare(:version) do |v|
|
|
251
329
|
# v.cached("bundle-#{Digest::SHA256.file(File.join(v.root, "Gemfile.lock"))}") { |_| v.run("bundle", "install", chdir: v.root) }
|
|
252
330
|
# {"BUNDLE_GEMFILE" => File.join(v.root, "Gemfile")}
|
|
253
331
|
# end
|
|
254
332
|
#
|
|
255
|
-
# A worker's (named "
|
|
333
|
+
# A worker's (named "base-1", say) is prepared by the runner as
|
|
256
334
|
# the worker starts: a server of its own, say.
|
|
257
335
|
#
|
|
258
336
|
# c.prepare(:worker) do |w|
|
data/lib/duel/context.rb
CHANGED
|
@@ -498,9 +498,10 @@ module Duel
|
|
|
498
498
|
end
|
|
499
499
|
|
|
500
500
|
# A stretch of a benchmark run for its peak memory: with the garbage
|
|
501
|
-
# collector working as usual, the most
|
|
502
|
-
#
|
|
503
|
-
# every
|
|
501
|
+
# collector working as usual, the most memory the process had in use
|
|
502
|
+
# (Native::Memory.used: resident memory, or on JRuby, the JVM's heap)
|
|
503
|
+
# above where it started (after a full collection), watched every
|
|
504
|
+
# millisecond.
|
|
504
505
|
# @api private
|
|
505
506
|
class MemorySample
|
|
506
507
|
attr_reader :started, :stopped, :peak
|
|
@@ -570,8 +571,8 @@ module Duel
|
|
|
570
571
|
end
|
|
571
572
|
|
|
572
573
|
def resident
|
|
573
|
-
# (
|
|
574
|
-
Native::Memory.
|
|
574
|
+
# (The memory in use is known: see Lane#peak_memory.)
|
|
575
|
+
Native::Memory.used #: Integer
|
|
575
576
|
end
|
|
576
577
|
|
|
577
578
|
def watch
|
data/lib/duel/init.rb
CHANGED
|
@@ -10,15 +10,17 @@ require_relative "../duel"
|
|
|
10
10
|
module Duel
|
|
11
11
|
# `duel init`: sets Duel up in a gem or an app that isn't a Rails app
|
|
12
12
|
# (duel-rails's generator sets a Rails app up). It writes .duel.rb, a
|
|
13
|
-
# benchmark to start from, and
|
|
13
|
+
# benchmark to start from, and GitHub Actions workflows (Duel's, and
|
|
14
|
+
# one that comments on a pull request with its report), or with
|
|
15
|
+
# --gitlab, GitLab CI jobs to include, adds a Rake
|
|
14
16
|
# task to the Rakefile, and adds StackProf to the Gemfile (for profiles
|
|
15
17
|
# of regressions, on CRuby). It overwrites nothing without --force, and
|
|
16
18
|
# writes nothing with --dry-run.
|
|
17
19
|
# @api private
|
|
18
20
|
class Init
|
|
19
21
|
USAGE = "Usage: duel init [options]\n\n" \
|
|
20
|
-
"Sets Duel up in this project: .duel.rb, a benchmark to start from in duel/,
|
|
21
|
-
"a Rake task, and StackProf in the Gemfile.\n\n"
|
|
22
|
+
"Sets Duel up in this project: .duel.rb, a benchmark to start from in duel/, GitHub Actions workflows\n" \
|
|
23
|
+
"(or GitLab CI jobs), a Rake task, and StackProf in the Gemfile.\n\n"
|
|
22
24
|
|
|
23
25
|
TEMPLATES = File.join(__dir__, "templates")
|
|
24
26
|
|
|
@@ -61,8 +63,13 @@ module Duel
|
|
|
61
63
|
|
|
62
64
|
# The files it writes, from the project's root, to what's in them.
|
|
63
65
|
def files
|
|
64
|
-
|
|
65
|
-
{".duel.rb" => render("duel.rb"), benchmark_file => render("benchmark_duel.rb")
|
|
66
|
+
top = Pathname(File.realpath(repository)).relative_path_from(File.realpath(@root))
|
|
67
|
+
written = {".duel.rb" => render("duel.rb"), benchmark_file => render("benchmark_duel.rb")}
|
|
68
|
+
return written.merge(top.join(gitlab_file).to_s => render("duel.gitlab-ci.yml")) if @gitlab
|
|
69
|
+
|
|
70
|
+
workflows = top.join(".github", "workflows")
|
|
71
|
+
written.merge(workflows.join(workflow_name).to_s => render("perf.yml"),
|
|
72
|
+
workflows.join(workflow_name.sub(/\.yml\z/, "-comment.yml")).to_s => render("perf-comment.yml"))
|
|
66
73
|
end
|
|
67
74
|
|
|
68
75
|
private
|
|
@@ -72,6 +79,7 @@ module Duel
|
|
|
72
79
|
o.banner = USAGE
|
|
73
80
|
o.on("--force", "Overwrite files that exist") { @force = true }
|
|
74
81
|
o.on("--dry-run", "Say what it would do, and write nothing") { @dry_run = true }
|
|
82
|
+
o.on("--gitlab", "Write GitLab CI jobs (to include from .gitlab-ci.yml), not GitHub Actions workflows") { @gitlab = true }
|
|
75
83
|
o.on("-h", "--help", "Print this help") do
|
|
76
84
|
@out.puts o
|
|
77
85
|
throw :help
|
|
@@ -156,8 +164,13 @@ module Duel
|
|
|
156
164
|
steps << "Set c.library in .duel.rb to what each version's workers require." unless required
|
|
157
165
|
steps << "Replace the example in #{benchmark_file} with benchmarks of your own code."
|
|
158
166
|
steps << "Run `bundle exec duel --force`. It compares the code with itself, and passes when the benchmarks are sound."
|
|
159
|
-
steps <<
|
|
160
|
-
"
|
|
167
|
+
steps << if @gitlab
|
|
168
|
+
"Include #{gitlab_file} from .gitlab-ci.yml, and add the tokens it says to, for comments. On a merge request, the job " \
|
|
169
|
+
"compares the code with the branch it targets."
|
|
170
|
+
else
|
|
171
|
+
"Commit and push. On a pull request, the workflow compares the code with the branch it targets, and comments with the " \
|
|
172
|
+
"report when a benchmark fails (c.comment in .duel.rb says when)."
|
|
173
|
+
end
|
|
161
174
|
"\nNext:\n#{steps.each_with_index.map { |step, i| " #{i + 1}. #{step}\n" }.join}"
|
|
162
175
|
end
|
|
163
176
|
|
|
@@ -215,5 +228,30 @@ module Duel
|
|
|
215
228
|
|
|
216
229
|
# What marks the workflow's comment on a pull request.
|
|
217
230
|
def marker = ["duel", working_directory].compact.join(" ")
|
|
231
|
+
|
|
232
|
+
# Where the GitLab CI jobs are, from the repository's root:
|
|
233
|
+
# .gitlab/duel.gitlab-ci.yml, or for a project in a subdirectory,
|
|
234
|
+
# .gitlab/duel-<its path>.gitlab-ci.yml.
|
|
235
|
+
def gitlab_file = File.join(".gitlab", "#{["duel", working_directory&.tr("/", "-")].compact.join("-")}.gitlab-ci.yml")
|
|
236
|
+
|
|
237
|
+
# The GitLab CI job's name: duel, or for a project in a subdirectory,
|
|
238
|
+
# duel:<its path>, so each of a monorepo's projects has its own.
|
|
239
|
+
def gitlab_job = ["duel", working_directory].compact.join(":")
|
|
240
|
+
|
|
241
|
+
# The Ruby image's tag for GitLab CI: the version .ruby-version names,
|
|
242
|
+
# exactly (so it's the Ruby that c.ruby = :version_file asks for), or
|
|
243
|
+
# this one's minor version.
|
|
244
|
+
def gitlab_ruby
|
|
245
|
+
named = File.read(File.join(@root, ".ruby-version")).split.first if version_file
|
|
246
|
+
named&.delete_prefix("ruby-") || RUBY_VERSION.rpartition(".").first
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# What names the branches that mark the last commit Duel passed in a
|
|
250
|
+
# push, before the branch pushed's name (see Baseline#passed_branch).
|
|
251
|
+
def passed_branch = ["duel/passed", working_directory&.tr("/", "-")].compact.join("-")
|
|
252
|
+
|
|
253
|
+
# The branch that holds the workflow's badge: duel/badge, or for a
|
|
254
|
+
# project in a subdirectory, duel/badge-<its path>.
|
|
255
|
+
def badge_branch = ["duel/badge", working_directory&.tr("/", "-")].compact.join("-")
|
|
218
256
|
end
|
|
219
257
|
end
|
data/lib/duel/json_report.rb
CHANGED
|
@@ -22,7 +22,7 @@ module Duel
|
|
|
22
22
|
none = [] #: Array[value]
|
|
23
23
|
nothing = {} #: report
|
|
24
24
|
head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
|
|
25
|
-
"benchmarks" => none, "across" => nothing, "errors" => none)
|
|
25
|
+
"benchmarks" => none, "across" => nothing, "errors" => none, "cut_short" => nil)
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
# What every report starts with.
|
|
@@ -30,7 +30,7 @@ module Duel
|
|
|
30
30
|
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
|
|
31
31
|
"settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
|
|
32
32
|
"jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
|
|
33
|
-
"warm" => settings.warm, "force" => settings.force}}
|
|
33
|
+
"time_limit" => settings.time_limit, "warm" => settings.warm, "force" => settings.force}}
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
@@ -54,7 +54,8 @@ module Duel
|
|
|
54
54
|
"plugins" => @runner.plugins.map(&:to_s),
|
|
55
55
|
"benchmarks" => @results.map { |result| benchmark(result) },
|
|
56
56
|
"across" => across,
|
|
57
|
-
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} }
|
|
57
|
+
"errors" => @results.select(&:error).map { |result| {"benchmark" => result.name, "message" => result.error} },
|
|
58
|
+
"cut_short" => @runner.cut_short
|
|
58
59
|
)
|
|
59
60
|
end
|
|
60
61
|
|