duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/cli.rb
CHANGED
|
@@ -11,25 +11,32 @@ require_relative "summary"
|
|
|
11
11
|
require_relative "json_report"
|
|
12
12
|
require_relative "init"
|
|
13
13
|
require_relative "comment"
|
|
14
|
+
require_relative "mark_passed"
|
|
15
|
+
require_relative "badge"
|
|
14
16
|
require_relative "spawn"
|
|
15
17
|
|
|
16
18
|
module Duel
|
|
17
|
-
# The `duel` command (and `duel init
|
|
18
|
-
#
|
|
19
|
+
# The `duel` command (and `duel init`, `duel comment`,
|
|
20
|
+
# `duel mark-passed` and `duel badge`: see Init, Comment, MarkPassed
|
|
21
|
+
# and Badge). What a run behind a rake task or a CI expression can't pass
|
|
19
22
|
# as an option can be set in the environment (DUEL_BASE, DUEL_FILTER,
|
|
20
|
-
# DUEL_FORCE), and options win.
|
|
23
|
+
# DUEL_FORCE...: see ENV_OPTIONS), and options win. On CI, the baseline
|
|
24
|
+
# is what the run's event says (see #event_base), unless one of them
|
|
25
|
+
# names one.
|
|
21
26
|
# On GitHub Actions (GITHUB_STEP_SUMMARY), it adds its report to the
|
|
22
27
|
# job's summary, in Markdown (see Summary), and tells the step's
|
|
23
28
|
# outputs (GITHUB_OUTPUT) what to do about the comment on a pull
|
|
24
|
-
# request (see .comment), and for a push,
|
|
25
|
-
# (see Config#push_base). --markdown writes the Markdown to a file, and
|
|
29
|
+
# request (see .comment), and for a push, the branch that marks it
|
|
30
|
+
# passed, where it's to be (see Config#push_base). --markdown writes the Markdown to a file, and
|
|
26
31
|
# --json the report as JSON (see JSONReport), whatever the run's
|
|
27
32
|
# outcome, but not for a command line that isn't valid.
|
|
28
33
|
# @api private
|
|
29
34
|
class CLI
|
|
30
|
-
USAGE = "Usage: duel [options]\n duel
|
|
31
|
-
"
|
|
32
|
-
"`duel
|
|
35
|
+
USAGE = "Usage: duel [options]\n duel base [options]\n duel init [options]\n duel clean [--config PATH]\n" \
|
|
36
|
+
" duel comment [options]\n duel mark-passed BRANCH\n duel badge BRANCH [options]\n\n" \
|
|
37
|
+
"Fails if this tree is slower than a baseline at any benchmark. `duel base` prints the commit it would compare\n" \
|
|
38
|
+
"with, `duel init` sets Duel up in a project, `duel clean` removes tmp/duel and the baseline's git worktree,\n" \
|
|
39
|
+
"and `duel comment`, `duel mark-passed` and `duel badge` are what the workflows it writes run.\n\n"
|
|
33
40
|
|
|
34
41
|
# Where GitHub Actions has a step add to its job's summary.
|
|
35
42
|
STEP_SUMMARY = "GITHUB_STEP_SUMMARY"
|
|
@@ -60,10 +67,16 @@ module Duel
|
|
|
60
67
|
# job that `duel init --gitlab` writes lets it fail, as a warning.
|
|
61
68
|
WARNED = 3
|
|
62
69
|
|
|
63
|
-
# The variables that set options, to their values
|
|
64
|
-
#
|
|
65
|
-
|
|
66
|
-
|
|
70
|
+
# The variables that set options, to their values: DUEL_ and the
|
|
71
|
+
# option's name (DUEL_LIMIT=1.05 is --limit 1.05, DUEL_TIME_LIMIT
|
|
72
|
+
# --time-limit), for --base, --candidate, --filter, --ruby, and each
|
|
73
|
+
# option .duel.rb can set too (Config::RUN_SETTINGS). An option given
|
|
74
|
+
# beats its variable, which beats .duel.rb.
|
|
75
|
+
ENV_OPTIONS = [:base, :candidate, :filter, :ruby, *Config::RUN_SETTINGS.keys].to_h { |name| ["DUEL_#{name.upcase}", "--#{name.to_s.tr("_", "-")}"] }.freeze
|
|
76
|
+
|
|
77
|
+
# What DUEL_FORCE's values say: to compare whatever changed
|
|
78
|
+
# (--force), or not.
|
|
79
|
+
FORCE = {"1" => true, "true" => true, "0" => false, "false" => false}.freeze
|
|
67
80
|
|
|
68
81
|
def initialize(argv, env: ENV, out: $stdout, err: $stderr)
|
|
69
82
|
@argv = argv
|
|
@@ -78,6 +91,10 @@ module Duel
|
|
|
78
91
|
def run
|
|
79
92
|
return Init.new(@argv.drop(1), out: @out, err: @err).run if @argv.first == "init"
|
|
80
93
|
return Comment.new(@argv.drop(1), env: @env, out: @out, err: @err).run if @argv.first == "comment"
|
|
94
|
+
return MarkPassed.new(@argv.drop(1), env: @env, out: @out, err: @err).run if @argv.first == "mark-passed"
|
|
95
|
+
return Badge.new(@argv.drop(1), env: @env, out: @out, err: @err).run if @argv.first == "badge"
|
|
96
|
+
return clean(@argv.drop(1)) if @argv.first == "clean"
|
|
97
|
+
return show_base(@argv.drop(1)) if @argv.first == "base"
|
|
81
98
|
|
|
82
99
|
settings = @settings = catch(:version) { parse } or return 0
|
|
83
100
|
config = @config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
@@ -112,34 +129,25 @@ module Duel
|
|
|
112
129
|
COMMENTED.fetch(mode).include?(status) ? "post" : "change"
|
|
113
130
|
end
|
|
114
131
|
|
|
115
|
-
# The mode --comment names (see Config#comment), or an invalid
|
|
116
|
-
# argument.
|
|
117
|
-
def self.comment_mode(name) = named(name, Config::COMMENTS)
|
|
118
|
-
|
|
119
|
-
# The runs --fail-on names (see Config#fail_on), or an invalid
|
|
120
|
-
# argument.
|
|
121
|
-
def self.fail_on(name) = named(name, Config::FAIL_ON)
|
|
122
|
-
|
|
123
|
-
# Which of +values+ (symbols) an option's argument names, or an
|
|
124
|
-
# invalid argument.
|
|
125
|
-
def self.named(name, values)
|
|
126
|
-
values.find { |value| value.to_s == name } or
|
|
127
|
-
raise OptionParser::InvalidArgument, "#{name} (give #{values.join(", ")})"
|
|
128
|
-
end
|
|
129
|
-
|
|
130
132
|
# The settings, from the environment and then the options.
|
|
131
133
|
def parse
|
|
132
134
|
@given = [] # (the RUN_SETTINGS options given: see Config)
|
|
133
135
|
settings = Settings.defaults
|
|
134
136
|
settings.jobs = [Etc.nprocessors / CPUS_PER_LANE, 1].max
|
|
135
137
|
settings.fit = %i[jobs processes]
|
|
136
|
-
parser(settings).parse
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
138
|
+
rest = parser(settings).parse(env_options + @argv)
|
|
139
|
+
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
140
|
+
|
|
141
|
+
# (A base given that's the event's is the event's too: the workflows
|
|
142
|
+
# duel init wrote before give DUEL_BASE as the event says.)
|
|
143
|
+
settings.base = event_base unless settings.base || settings.base_dir
|
|
144
|
+
settings.pushed = variable("GITHUB_REF_NAME", "CI_COMMIT_BRANCH") if pushed? && from_event?(settings)
|
|
140
145
|
target = variable("GITHUB_BASE_REF", "CI_MERGE_REQUEST_TARGET_BRANCH_NAME")
|
|
141
146
|
settings.target = "origin/#{target}" if target && pull_request?
|
|
142
147
|
settings.accepted = variable("DUEL_ACCEPTED")
|
|
148
|
+
# (Where Duel finds what a label accepted since the last release, on
|
|
149
|
+
# a schedule: see Baseline#merged_accepted.)
|
|
150
|
+
settings.repository = variable("GITHUB_REPOSITORY") if event == "schedule" && !gitlab?
|
|
143
151
|
settings.config_path ||= (".duel.rb" if File.exist?(".duel.rb"))
|
|
144
152
|
settings.config_path &&= File.expand_path(settings.config_path)
|
|
145
153
|
settings
|
|
@@ -147,21 +155,60 @@ module Duel
|
|
|
147
155
|
|
|
148
156
|
private
|
|
149
157
|
|
|
158
|
+
# `duel base`: prints the commit a run given +argv+ (a run's options:
|
|
159
|
+
# --base, say) compares with, or nothing, where it has none (no release
|
|
160
|
+
# yet, say, or a directory): for what needs it before the run, as the
|
|
161
|
+
# workflow's step that installs the baseline's Ruby does.
|
|
162
|
+
def show_base(argv)
|
|
163
|
+
@argv = argv
|
|
164
|
+
settings = @settings = catch(:version) { parse } or return 0
|
|
165
|
+
config = settings.config_path ? Duel.load_config(settings.config_path) : Config.new
|
|
166
|
+
ref = Baseline.new(config, settings).ref unless settings.base_dir
|
|
167
|
+
commit, found = git(config.root, "rev-parse", "#{ref}^{commit}") if ref
|
|
168
|
+
raise Error, "#{ref} isn't a commit here" if found && !found.success?
|
|
169
|
+
|
|
170
|
+
@out.puts commit
|
|
171
|
+
0
|
|
172
|
+
rescue Error, OptionParser::ParseError => e
|
|
173
|
+
@err.puts "duel base: #{e}"
|
|
174
|
+
2
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
# `duel clean`: removes what Duel keeps (tmp/duel), and the baseline's
|
|
178
|
+
# git worktree (see Baseline#clean), holding the project's lock, for
|
|
179
|
+
# the configuration at --config, or .duel.rb.
|
|
180
|
+
def clean(argv)
|
|
181
|
+
path = nil
|
|
182
|
+
parser = OptionParser.new("Usage: duel clean [--config PATH]") do |o|
|
|
183
|
+
o.on("--config PATH", "Config file (default: .duel.rb)") { |v| path = v }
|
|
184
|
+
o.on("-h", "--help", "Print this help") do
|
|
185
|
+
@out.puts o
|
|
186
|
+
return 0
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
rest = parser.parse(argv)
|
|
190
|
+
raise OptionParser::NeedlessArgument, rest.first unless rest.empty?
|
|
191
|
+
|
|
192
|
+
path ||= (".duel.rb" if File.exist?(".duel.rb"))
|
|
193
|
+
config = path ? Duel.load_config(path) : Config.new
|
|
194
|
+
Lock.hold(config.state) { Baseline.new(config, Settings.defaults).clean }
|
|
195
|
+
@out.puts "duel: removed tmp/duel and its baseline's worktree"
|
|
196
|
+
0
|
|
197
|
+
rescue Error, OptionParser::ParseError => e
|
|
198
|
+
@err.puts "duel clean: #{e}"
|
|
199
|
+
2
|
|
200
|
+
end
|
|
201
|
+
|
|
150
202
|
# What #run does, holding the project's lock (see Lock).
|
|
151
203
|
def compare(config, settings)
|
|
152
204
|
baseline = Baseline.new(config, settings)
|
|
153
|
-
if @clean
|
|
154
|
-
baseline.clean
|
|
155
|
-
@out.puts "duel: removed tmp/duel and its baseline's worktree"
|
|
156
|
-
return 0
|
|
157
|
-
end
|
|
158
205
|
if settings.base_dir.nil? && baseline.ref.nil?
|
|
159
206
|
return nothing("no commit to compare against", "no commit before this push to compare against") if settings.pushed
|
|
160
207
|
|
|
161
|
-
return nothing("no release to compare against", "no release tag (#{config.release_tags}) before HEAD to compare against")
|
|
208
|
+
return nothing("no release to compare against", "no release tag (#{Config.list(config.release_tags).join(", ")}) before HEAD to compare against")
|
|
162
209
|
end
|
|
163
210
|
if !settings.force && baseline.unchanged?
|
|
164
|
-
@passed = marks_passed?(config, settings)
|
|
211
|
+
@passed = (baseline.passed_branch if marks_passed?(config, settings))
|
|
165
212
|
return nothing("nothing to compare",
|
|
166
213
|
"#{baseline.watched.join(", ")} unchanged since the baseline, so there's nothing to compare (--force to anyway)")
|
|
167
214
|
end
|
|
@@ -173,17 +220,17 @@ module Duel
|
|
|
173
220
|
runner = Runner.new(config, settings, base_root, out: @out)
|
|
174
221
|
results = runner.run
|
|
175
222
|
passed = runner.passed?(results)
|
|
176
|
-
status = passed ? 0 : slower(config
|
|
177
|
-
@passed = (passed || accepted?(config)) && marks_passed?(config, settings)
|
|
223
|
+
status = passed ? 0 : slower(config)
|
|
224
|
+
@passed = (baseline.passed_branch if (passed || accepted?(config)) && marks_passed?(config, settings))
|
|
178
225
|
publish(
|
|
179
226
|
passed ? "passed" : "failed",
|
|
180
227
|
-> {
|
|
181
228
|
Summary.new(runner, results, settings: settings, baseline: described_baseline(config, settings, baseline),
|
|
182
|
-
candidate: described_candidate(config, settings), note: @note, accepted: @accepted).markdown
|
|
229
|
+
candidate: described_candidate(config, settings, baseline), note: @note, accepted: @accepted).markdown
|
|
183
230
|
},
|
|
184
|
-
->
|
|
185
|
-
JSONReport.new(runner, results, settings: settings, exit: status,
|
|
186
|
-
baseline: baseline_json(config, settings, baseline, base_root), candidate: candidate_json(config, settings)).to_h
|
|
231
|
+
-> {
|
|
232
|
+
JSONReport.new(runner, results, settings: settings, passed: passed, exit: status, passed_anyway: @passed_anyway,
|
|
233
|
+
baseline: baseline_json(config, settings, baseline, base_root), candidate: candidate_json(config, settings, baseline)).to_h
|
|
187
234
|
}
|
|
188
235
|
)
|
|
189
236
|
status
|
|
@@ -192,20 +239,21 @@ module Duel
|
|
|
192
239
|
# The exit status of a run that was slower (or whose benchmark
|
|
193
240
|
# failed): 1, or 0 where the pull request's label accepts it
|
|
194
241
|
# (Config#accept_label), or on GitHub Actions outside a pull request,
|
|
195
|
-
# where c.fail_on
|
|
242
|
+
# where c.fail_on says only a pull request fails, with
|
|
196
243
|
# a warning that says so, and a note in the report.
|
|
197
|
-
def slower(config
|
|
244
|
+
def slower(config)
|
|
198
245
|
if accepted?(config)
|
|
199
246
|
# (Set: see #accepted?.)
|
|
200
247
|
label = config.accept_label #: String
|
|
201
248
|
@accepted = true
|
|
249
|
+
@passed_anyway = "accept_label"
|
|
202
250
|
return passing("accepted with the #{label} label", "Accepted with the #{Summary.code(label)} label.")
|
|
203
251
|
end
|
|
204
|
-
return 1 unless pull_requests_only?(config
|
|
205
|
-
return 1 unless outside_pull_request?
|
|
252
|
+
return 1 unless pull_requests_only?(config) && outside_pull_request?
|
|
206
253
|
|
|
207
254
|
# (Outside a pull request on CI, so there's one: see #event.)
|
|
208
255
|
run = event #: String
|
|
256
|
+
@passed_anyway = "fail_on"
|
|
209
257
|
passing("only a pull request fails (c.fail_on), so this #{run} passes",
|
|
210
258
|
"Only a pull request fails (#{Summary.code("c.fail_on")}), so this #{run} passes.", gitlab? ? WARNED : 0)
|
|
211
259
|
end
|
|
@@ -245,9 +293,8 @@ module Duel
|
|
|
245
293
|
# says to compare a push with that commit.
|
|
246
294
|
def marks_passed?(config, settings) = !settings.pushed.nil? && config.push_base == :last_passed
|
|
247
295
|
|
|
248
|
-
# Whether a regression fails only a pull request (
|
|
249
|
-
|
|
250
|
-
def pull_requests_only?(config, settings) = (settings.fail_on || config.fail_on) == :pull_requests
|
|
296
|
+
# Whether a regression fails only a pull request (Config#fail_on).
|
|
297
|
+
def pull_requests_only?(config) = config.fail_on == :pull_requests
|
|
251
298
|
|
|
252
299
|
# What a run on CI is for, as GitHub Actions names it
|
|
253
300
|
# (GITHUB_EVENT_NAME): push, pull_request, schedule, say. On GitLab
|
|
@@ -263,7 +310,7 @@ module Duel
|
|
|
263
310
|
end
|
|
264
311
|
|
|
265
312
|
# Whether this is a run on GitLab CI.
|
|
266
|
-
def gitlab? =
|
|
313
|
+
def gitlab? = !variable("GITLAB_CI").nil?
|
|
267
314
|
|
|
268
315
|
# Whether this is a run on CI for something other than a pull request
|
|
269
316
|
# (PULL_REQUESTS): a push, say.
|
|
@@ -276,19 +323,34 @@ module Duel
|
|
|
276
323
|
# see PULL_REQUESTS).
|
|
277
324
|
def pull_request? = PULL_REQUESTS.include?(event)
|
|
278
325
|
|
|
279
|
-
# Whether
|
|
280
|
-
#
|
|
281
|
-
|
|
326
|
+
# Whether +settings+' baseline is the one the run's event says (see
|
|
327
|
+
# #event_base): the event's, or a base given that's the event's (as
|
|
328
|
+
# the workflows duel init wrote before give DUEL_BASE).
|
|
329
|
+
def from_event?(settings)
|
|
330
|
+
found = event_base
|
|
331
|
+
!found.nil? && settings.base == found
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
# The baseline a run on CI is for, as its event says, where neither
|
|
335
|
+
# --base nor DUEL_BASE names one: for a push, the commit before it
|
|
336
|
+
# (on GitHub Actions, the event's before, and on GitLab CI,
|
|
337
|
+
# CI_COMMIT_BEFORE_SHA), for a merge queue, its base, and on a
|
|
338
|
+
# schedule, the last release (Baseline::LAST_RELEASE). Nil for the
|
|
339
|
+
# rest: a pull request, whose baseline is where HEAD left the branch
|
|
340
|
+
# it targets, as outside CI.
|
|
341
|
+
def event_base
|
|
342
|
+
case event
|
|
343
|
+
when "push" then gitlab? ? variable("CI_COMMIT_BEFORE_SHA") : event_payload["before"]
|
|
344
|
+
when "merge_group" then event_payload.dig("merge_group", "base_sha")
|
|
345
|
+
when "schedule" then Baseline::LAST_RELEASE
|
|
346
|
+
end
|
|
347
|
+
end
|
|
282
348
|
|
|
283
349
|
# Whether this is a run on CI for a push.
|
|
284
350
|
def pushed? = event == "push"
|
|
285
351
|
|
|
286
352
|
# The first of the variables +names+ that's set, or nil.
|
|
287
|
-
def variable(*names)
|
|
288
|
-
set = variables
|
|
289
|
-
name = names.find { |n| set.key?(n) }
|
|
290
|
-
set.fetch(name) if name
|
|
291
|
-
end
|
|
353
|
+
def variable(*names) = names.map { |name| @env[name] }.find { |value| value && !value.empty? }
|
|
292
354
|
|
|
293
355
|
# Says there's nothing to compare (+title+, in Markdown), and +why+,
|
|
294
356
|
# and passes.
|
|
@@ -302,48 +364,45 @@ module Duel
|
|
|
302
364
|
def unfinished(mark, title, status, why, exit)
|
|
303
365
|
# (Parsed, so set: see #run.)
|
|
304
366
|
settings = @settings #: Settings
|
|
305
|
-
publish(status, -> { Summary.note(mark, title, why) }, ->
|
|
367
|
+
publish(status, -> { Summary.note(mark, title, why) }, -> { JSONReport.note(status, exit, "duel: #{why}", settings) })
|
|
306
368
|
exit
|
|
307
369
|
end
|
|
308
370
|
|
|
309
371
|
# What the workflow does about its comment on a pull request, or on
|
|
310
372
|
# a pushed commit (see .comment), for a run whose +status+ is the
|
|
311
|
-
# JSON report's: as
|
|
312
|
-
# default where
|
|
373
|
+
# JSON report's: as the configuration says (Config#comment), or the
|
|
374
|
+
# default where it didn't load. It comments on a push
|
|
313
375
|
# only where a regression in one passes (Config#fail_on), so the
|
|
314
376
|
# comment says what the check doesn't. A regression the label
|
|
315
377
|
# accepts (Config#accept_label) is commented on as a run that passed
|
|
316
378
|
# would be: whoever added the label knows, so by default it posts
|
|
317
379
|
# only where the last comment's status isn't its own.
|
|
318
380
|
def commenting(status)
|
|
319
|
-
# (Parsed, so set: see #run.)
|
|
320
|
-
settings = @settings #: Settings
|
|
321
381
|
config = @config || Config.new
|
|
322
|
-
return "none" if pushed? && !pull_requests_only?(config
|
|
382
|
+
return "none" if pushed? && !pull_requests_only?(config)
|
|
323
383
|
|
|
324
|
-
CLI.comment(
|
|
384
|
+
CLI.comment(config.comment, @accepted ? "passed" : status)
|
|
325
385
|
end
|
|
326
386
|
|
|
327
387
|
# Writes the reports of a run whose +status+ is the JSON report's
|
|
328
|
-
# where they're asked for: the JSON +json+ makes
|
|
329
|
-
#
|
|
330
|
-
#
|
|
331
|
-
# --markdown names, and added to the job's summary on GitHub Actions
|
|
388
|
+
# where they're asked for: the JSON +json+ makes to the file --json
|
|
389
|
+
# names, and the Markdown +markdown+ makes to the file --markdown
|
|
390
|
+
# names, and added to the job's summary on GitHub Actions
|
|
332
391
|
# (STEP_SUMMARY). Each is only made if it's written. On GitHub
|
|
333
392
|
# Actions, it also sets the step's outputs (STEP_OUTPUT) for the jobs
|
|
334
393
|
# that comment: +comment+, and +status+ as the comment's marker has
|
|
335
|
-
# it (MARKED), and for the job that marks a push passed, "passed"
|
|
336
|
-
# (see #marks_passed?). Where DUEL_OUTPUT
|
|
337
|
-
# writes them there too, as variables.
|
|
394
|
+
# it (MARKED), and for the job that marks a push passed, "passed",
|
|
395
|
+
# the branch that marks it (see #marks_passed?). Where DUEL_OUTPUT
|
|
396
|
+
# names a file (OUTPUT), it writes them there too, as variables.
|
|
338
397
|
def publish(status, markdown, json)
|
|
339
398
|
comment = commenting(status)
|
|
340
|
-
outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed
|
|
399
|
+
outputs = "comment=#{comment}\nstatus=#{MARKED.fetch(status)}\n#{"passed=#{@passed}\n" if @passed}"
|
|
341
400
|
step_output = variable(STEP_OUTPUT)
|
|
342
401
|
File.write(step_output, outputs, mode: "a") if step_output
|
|
343
402
|
output = variable(OUTPUT)
|
|
344
403
|
File.write(output, outputs.gsub(/^\w+=/) { |name| "DUEL_#{name.upcase}" }, mode: "a") if output
|
|
345
404
|
path = @json
|
|
346
|
-
write(path, "#{JSON.pretty_generate(json.call
|
|
405
|
+
write(path, "#{JSON.pretty_generate(json.call)}\n") if path
|
|
347
406
|
path = @markdown
|
|
348
407
|
summary = variable(STEP_SUMMARY)
|
|
349
408
|
return unless path || summary
|
|
@@ -361,10 +420,9 @@ module Duel
|
|
|
361
420
|
|
|
362
421
|
# The baseline, in Markdown: its directory, or its commit, bare (so
|
|
363
422
|
# GitHub links it) and followed by a space, never punctuation, and
|
|
364
|
-
# what chose it: the main branch, the push (
|
|
365
|
-
#
|
|
366
|
-
#
|
|
367
|
-
# --base.
|
|
423
|
+
# what chose it: the main branch, the push (the commit before it, or
|
|
424
|
+
# the last commit Duel passed), the merge queue (its base), the last
|
|
425
|
+
# release, or --base.
|
|
368
426
|
def described_baseline(config, settings, baseline)
|
|
369
427
|
dir = settings.base_dir
|
|
370
428
|
return Summary.code(dir) if dir
|
|
@@ -376,8 +434,9 @@ module Duel
|
|
|
376
434
|
when nil then "where HEAD left #{Summary.code(baseline.main_branch)}"
|
|
377
435
|
when Baseline::LAST_RELEASE then last_release(config, baseline)
|
|
378
436
|
else
|
|
379
|
-
if
|
|
380
|
-
|
|
437
|
+
if ref == baseline.passed then "the last commit Duel passed"
|
|
438
|
+
# (The event's: a push's, or a merge queue's, as a schedule's is the last release.)
|
|
439
|
+
elsif from_event?(settings) then pushed? ? "the commit before the push" : "the merge queue's base"
|
|
381
440
|
elsif !ref.match?(/\A\h+\z/) then Summary.code(ref)
|
|
382
441
|
else "from --base"
|
|
383
442
|
end
|
|
@@ -398,13 +457,12 @@ module Duel
|
|
|
398
457
|
# The candidate, in Markdown: its directory, or its commit, bare (see
|
|
399
458
|
# #described_baseline), and whether what's watched (Baseline#watched)
|
|
400
459
|
# has changes not committed.
|
|
401
|
-
def described_candidate(config, settings)
|
|
460
|
+
def described_candidate(config, settings, baseline)
|
|
402
461
|
dir = settings.candidate_dir || config.root
|
|
403
462
|
head, found = git(dir, "rev-parse", "--short", "HEAD")
|
|
404
463
|
return Summary.code(dir) unless found.success?
|
|
405
464
|
|
|
406
|
-
|
|
407
|
-
return "#{head.chomp} (with changes not committed)" unless clean.success?
|
|
465
|
+
return "#{head.chomp} (with changes not committed)" if uncommitted?(dir, baseline)
|
|
408
466
|
|
|
409
467
|
merged(dir) || head.chomp
|
|
410
468
|
end
|
|
@@ -429,16 +487,21 @@ module Duel
|
|
|
429
487
|
"base" => {"ref" => variable("CI_MERGE_REQUEST_TARGET_BRANCH_NAME")}}
|
|
430
488
|
|
|
431
489
|
# The pull request a pull_request event's run is for, as GitHub
|
|
432
|
-
# Actions describes it (
|
|
490
|
+
# Actions describes it (see #event_payload), or nil.
|
|
433
491
|
def pull_request
|
|
434
|
-
|
|
435
|
-
return unless path && File.file?(path)
|
|
436
|
-
|
|
437
|
-
payload = JSON.parse(File.read(path))
|
|
438
|
-
pull = payload["pull_request"] if payload.instance_of?(Hash)
|
|
492
|
+
pull = event_payload["pull_request"] if event == "pull_request"
|
|
439
493
|
pull if pull.instance_of?(Hash)
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
# The event a run on GitHub Actions is for, as its file
|
|
497
|
+
# (GITHUB_EVENT_PATH) describes it, or nothing ({}), where there's
|
|
498
|
+
# none, or it isn't a JSON object.
|
|
499
|
+
def event_payload
|
|
500
|
+
path = variable("GITHUB_EVENT_PATH")
|
|
501
|
+
payload = JSON.parse(File.read(path)) if path && File.file?(path)
|
|
502
|
+
payload.instance_of?(Hash) ? payload : {}
|
|
440
503
|
rescue JSON::ParserError
|
|
441
|
-
|
|
504
|
+
{}
|
|
442
505
|
end
|
|
443
506
|
|
|
444
507
|
# The baseline, for JSON: the ref that named it (the main branch,
|
|
@@ -457,13 +520,19 @@ module Duel
|
|
|
457
520
|
# The candidate, for JSON: its commit, and whether what's watched
|
|
458
521
|
# (Baseline#watched) has changes not committed (each nil outside a
|
|
459
522
|
# repository), and its tree.
|
|
460
|
-
def candidate_json(config, settings)
|
|
523
|
+
def candidate_json(config, settings, baseline)
|
|
461
524
|
dir = File.expand_path(settings.candidate_dir || config.root)
|
|
462
525
|
head, found = git(dir, "rev-parse", "HEAD")
|
|
463
526
|
return {"commit" => nil, "uncommitted_changes" => nil, "root" => dir} unless found.success?
|
|
464
527
|
|
|
465
|
-
|
|
466
|
-
|
|
528
|
+
{"commit" => head.chomp, "uncommitted_changes" => uncommitted?(dir, baseline), "root" => dir}
|
|
529
|
+
end
|
|
530
|
+
|
|
531
|
+
# Whether what +baseline+ watches (Baseline#watched) has changes not
|
|
532
|
+
# committed in +dir+, a repository.
|
|
533
|
+
def uncommitted?(dir, baseline)
|
|
534
|
+
_, clean = git(dir, "diff", "--quiet", "HEAD", "--", *baseline.watched)
|
|
535
|
+
!clean.success?
|
|
467
536
|
end
|
|
468
537
|
|
|
469
538
|
# Git's output, and its status, in +dir+.
|
|
@@ -487,8 +556,12 @@ module Duel
|
|
|
487
556
|
# which CI expressions find handy).
|
|
488
557
|
def env_options
|
|
489
558
|
set = variables
|
|
490
|
-
|
|
491
|
-
|
|
559
|
+
force = variable("DUEL_FORCE")
|
|
560
|
+
if force && !FORCE.key?(force.downcase)
|
|
561
|
+
raise OptionParser::InvalidArgument, "DUEL_FORCE=#{force} (give 1 or true, or 0 or false)"
|
|
562
|
+
end
|
|
563
|
+
|
|
564
|
+
ENV_OPTIONS.flat_map { |name, option| set.key?(name) ? [option, set.fetch(name)] : [] } + (FORCE[force.to_s.downcase] ? ["--force"] : [])
|
|
492
565
|
end
|
|
493
566
|
|
|
494
567
|
# The environment's variables that are set (not empty).
|
|
@@ -523,10 +596,10 @@ module Duel
|
|
|
523
596
|
settings.base = (v unless directory)
|
|
524
597
|
settings.base_dir = (v if directory)
|
|
525
598
|
end
|
|
526
|
-
o.on("--candidate
|
|
599
|
+
o.on("--candidate DIR", "Candidate: a directory, already built (default: this tree)") { |v| settings.candidate_dir = v }
|
|
527
600
|
o.on("--filter REGEXP", "Only benchmarks whose names match") { |v| settings.filter = CLI.regexp(v) }
|
|
528
|
-
o.on("--jobs N", Integer, "Benchmarks measured at once (default: one for every #{CPUS_PER_LANE} CPUs, as many as fit
|
|
529
|
-
|
|
601
|
+
o.on("--jobs N", Integer, "Benchmarks measured at once (default: c.jobs, or one for every #{CPUS_PER_LANE} CPUs, as many as fit)") do |v|
|
|
602
|
+
given(settings, :jobs, v)
|
|
530
603
|
end
|
|
531
604
|
o.on("--processes N", Integer, "Worker processes per version (default: c.processes, or #{PROCESSES}, as many as fit in memory)") do |v|
|
|
532
605
|
given(settings, :processes, v)
|
|
@@ -539,21 +612,16 @@ module Duel
|
|
|
539
612
|
end
|
|
540
613
|
o.on("--limit RATIO", Float, "Ratio above which a timing is slower (default: c.limit, or 1.02)") { |v| given(settings, :limit, v) }
|
|
541
614
|
o.on("--confidence LEVEL", Float, "Confidence of every interval (default: c.confidence, or 0.99)") { |v| given(settings, :confidence, v) }
|
|
542
|
-
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: c.warm, or until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT on, else 0)") do |v|
|
|
615
|
+
o.on("--warm SECONDS", Float, "Warm-up per benchmark, in seconds (default: c.warm, or until steady, up to 60 on JRuby and TruffleRuby, 10 with YJIT or ZJIT on, else 0)") do |v|
|
|
543
616
|
given(settings, :warm, v)
|
|
544
617
|
end
|
|
545
618
|
o.on("--ruby version-file|PATH", "Each version's Ruby: the one its version file names (.ruby-version, say), or a",
|
|
546
|
-
"path (default: c.ruby, or this one)") { |v| settings.ruby = (v == "version-file") ? :version_file : v }
|
|
619
|
+
"path (default: c.ruby, or this one)") { |v| settings.ruby = (v.tr("_", "-") == "version-file") ? :version_file : v }
|
|
547
620
|
o.on("--force", "Compare even if nothing watched changed") { settings.force = true }
|
|
548
621
|
o.on("--config PATH", "Config file (default: .duel.rb)") { |v| settings.config_path = v }
|
|
549
622
|
o.on("--markdown PATH", "Also write the report to PATH in Markdown, for a pull request comment (on GitHub",
|
|
550
623
|
"Actions, it's added to the job's summary too)") { |v| @markdown = v }
|
|
551
624
|
o.on("--json PATH", "Also write the report to PATH as JSON, for a program to read") { |v| @json = v }
|
|
552
|
-
o.on("--comment MODE", "When the workflow comments on a pull request: always, results (passed or failed),",
|
|
553
|
-
"failures, or never (default: c.comment, or failures)") { |v| settings.comment = CLI.comment_mode(v) }
|
|
554
|
-
o.on("--fail-on RUNS", "Which runs a regression fails: every_run, or pull_requests (on GitHub Actions, another",
|
|
555
|
-
"warns and passes) (default: c.fail_on, or every_run)") { |v| settings.fail_on = CLI.fail_on(v) }
|
|
556
|
-
o.on("--clean", "Remove tmp/duel, and the baseline's git worktree, and stop") { @clean = true }
|
|
557
625
|
o.on("-v", "--version", "Print the version") do
|
|
558
626
|
@out.puts "duel #{VERSION}"
|
|
559
627
|
throw :version
|