duel 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +20 -16
- data/CHANGELOG.md +148 -1
- data/README.md +59 -26
- data/docs/ci.md +612 -54
- data/docs/configuration.md +146 -4
- data/docs/how-it-works.md +8 -3
- data/docs/json.md +5 -2
- data/docs/troubleshooting.md +6 -5
- data/lib/duel/baseline.rb +61 -7
- data/lib/duel/cli.rb +259 -48
- data/lib/duel/config.rb +108 -3
- data/lib/duel/init.rb +49 -7
- data/lib/duel/json_report.rb +11 -9
- data/lib/duel/runner.rb +1 -1
- data/lib/duel/summary.rb +148 -74
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +148 -0
- data/lib/duel/templates/duel.rb.erb +20 -2
- data/lib/duel/templates/perf-comment.yml.erb +59 -0
- data/lib/duel/templates/perf.yml.erb +174 -37
- data/lib/duel/version.rb +1 -1
- data/sig/duel/baseline.rbs +30 -2
- data/sig/duel/cli.rbs +88 -5
- data/sig/duel/config.rbs +52 -1
- data/sig/duel/init.rbs +19 -0
- data/sig/duel/json_report.rbs +7 -5
- data/sig/duel/runner.rbs +18 -3
- data/sig/duel/summary.rbs +50 -18
- metadata +3 -1
data/lib/duel/json_report.rb
CHANGED
|
@@ -16,17 +16,18 @@ module Duel
|
|
|
16
16
|
SCHEMA = 1
|
|
17
17
|
|
|
18
18
|
# A run that didn't compare: +status+ ("nothing_to_compare", or
|
|
19
|
-
# "error"), its +exit+ status,
|
|
20
|
-
|
|
19
|
+
# "error"), its +exit+ status, +message+, what it said, and what the
|
|
20
|
+
# workflow does about its +comment+ (see CLI.comment).
|
|
21
|
+
def self.note(status, exit, message, settings, comment)
|
|
21
22
|
none = [] #: Array[value]
|
|
22
23
|
nothing = {} #: report
|
|
23
|
-
head(status, exit, message, settings).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
|
|
24
|
+
head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
|
|
24
25
|
"benchmarks" => none, "across" => nothing, "errors" => none)
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
# What every report starts with.
|
|
28
|
-
def self.head(status, exit, message, settings)
|
|
29
|
-
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message,
|
|
29
|
+
def self.head(status, exit, message, settings, comment)
|
|
30
|
+
{"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
|
|
30
31
|
"settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
|
|
31
32
|
"jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
|
|
32
33
|
"warm" => settings.warm, "force" => settings.force}}
|
|
@@ -34,19 +35,20 @@ module Duel
|
|
|
34
35
|
|
|
35
36
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
36
37
|
# +candidate+ (each {"ref", "commit", "root"} or {"commit",
|
|
37
|
-
# "uncommitted_changes", "root"}) as +settings+ say,
|
|
38
|
-
# +exit
|
|
39
|
-
def initialize(runner, results, settings:, exit:, baseline:, candidate:)
|
|
38
|
+
# "uncommitted_changes", "root"}) as +settings+ say, exits with
|
|
39
|
+
# +exit+, and has the workflow do +comment+ (see CLI.comment).
|
|
40
|
+
def initialize(runner, results, settings:, exit:, comment:, baseline:, candidate:)
|
|
40
41
|
@runner = runner
|
|
41
42
|
@results = results
|
|
42
43
|
@settings = settings
|
|
43
44
|
@exit = exit
|
|
45
|
+
@comment = comment
|
|
44
46
|
@versions = {"baseline" => baseline, "candidate" => candidate}
|
|
45
47
|
end
|
|
46
48
|
|
|
47
49
|
def to_h
|
|
48
50
|
rubies = @runner.rubies
|
|
49
|
-
JSONReport.head(@exit.zero? ? "passed" : "failed", @exit, nil, @settings).merge(
|
|
51
|
+
JSONReport.head(@exit.zero? ? "passed" : "failed", @exit, nil, @settings, @comment).merge(
|
|
50
52
|
"seconds" => @runner.seconds,
|
|
51
53
|
**@versions.to_h { |side, version| [side, version.merge("ruby" => rubies.fetch(side))] },
|
|
52
54
|
"plugins" => @runner.plugins.map(&:to_s),
|
data/lib/duel/runner.rb
CHANGED
|
@@ -30,7 +30,7 @@ module Duel
|
|
|
30
30
|
# How a run is set up (see Duel::CLI for where each comes from).
|
|
31
31
|
# @api private
|
|
32
32
|
Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
|
|
33
|
-
:config_path, :timeout, :ruby) do
|
|
33
|
+
:config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target) do
|
|
34
34
|
# @type self: singleton(Settings)
|
|
35
35
|
def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
|
|
36
36
|
timeout: 60.0)
|
data/lib/duel/summary.rb
CHANGED
|
@@ -4,11 +4,11 @@ require_relative "runner"
|
|
|
4
4
|
|
|
5
5
|
module Duel
|
|
6
6
|
# A run's report in Markdown, for a pull request: GitHub Actions' job
|
|
7
|
-
# summary, or a comment (see CLI).
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
7
|
+
# summary, or a comment (see CLI). In view, only what the author needs:
|
|
8
|
+
# the verdict, the two commits, and what failed, in a table. The rest is
|
|
9
|
+
# collapsed: what made each regression, every benchmark's timings, the
|
|
10
|
+
# verdicts across the board, each version's Ruby where they differ, and
|
|
11
|
+
# how Duel decides. So it stays short with hundreds of benchmarks.
|
|
12
12
|
# @api private
|
|
13
13
|
class Summary
|
|
14
14
|
# The most characters it has (a GitHub comment has at most 65,536):
|
|
@@ -26,6 +26,19 @@ module Duel
|
|
|
26
26
|
# The verdicts that fail a run, which stand out.
|
|
27
27
|
FAILED = %w[regressed error].freeze
|
|
28
28
|
|
|
29
|
+
# What the heading starts with: a run that passed, one that failed (or
|
|
30
|
+
# couldn't compare), and one with nothing compared.
|
|
31
|
+
PASSED_MARK = "🟢"
|
|
32
|
+
FAILED_MARK = "🔻"
|
|
33
|
+
NEUTRAL_MARK = "⬜"
|
|
34
|
+
|
|
35
|
+
# What an unsettled timing means.
|
|
36
|
+
UNSETTLED = "An unsettled timing was too close to the limit to settle. It doesn't fail the run, but one that's unsettled " \
|
|
37
|
+
"in run after run is noisy."
|
|
38
|
+
|
|
39
|
+
# How Duel decides that a timing is slower, in its docs.
|
|
40
|
+
DECIDES = "https://github.com/sferik/duel-ruby/blob/main/docs/how-it-works.md#how-duel-tells-a-regression-from-noise"
|
|
41
|
+
|
|
29
42
|
# +text+ as code, in Markdown: in a span of more backticks than any it
|
|
30
43
|
# has (and spaced from one at either end).
|
|
31
44
|
def self.code(text)
|
|
@@ -34,8 +47,17 @@ module Duel
|
|
|
34
47
|
"#{ticks}#{padded}#{ticks}"
|
|
35
48
|
end
|
|
36
49
|
|
|
37
|
-
#
|
|
38
|
-
|
|
50
|
+
# +text+ as it is, in Markdown: what Markdown, or the HTML in it, would
|
|
51
|
+
# take for something else, escaped.
|
|
52
|
+
def self.escape(text) = text.gsub(/[\\`*_\[\]<>|~&#]/) { |char| "\\#{char}" }
|
|
53
|
+
|
|
54
|
+
# A run that didn't compare: a heading of +mark+ and +title+, and the
|
|
55
|
+
# first line of +reason+, with all of it collapsed if it has more.
|
|
56
|
+
def self.note(mark, title, reason)
|
|
57
|
+
first, _, rest = reason.partition("\n")
|
|
58
|
+
more = "\n\n<details>\n<summary>The whole message</summary>\n\n#{block(reason)}\n\n</details>" unless rest.empty?
|
|
59
|
+
"### #{mark} Duel: #{title}\n\n#{escape(first)}#{more}\n"
|
|
60
|
+
end
|
|
39
61
|
|
|
40
62
|
# +text+ in a fenced block of its own: fenced with more backticks than
|
|
41
63
|
# any run of them it has, three at least.
|
|
@@ -67,13 +89,18 @@ module Duel
|
|
|
67
89
|
end
|
|
68
90
|
|
|
69
91
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
70
|
-
# +candidate+ (each a description, in Markdown
|
|
71
|
-
|
|
92
|
+
# +candidate+ (each a description, in Markdown, whose commit stands
|
|
93
|
+
# bare, so GitHub links it) as +settings+ say, with a +note+ (in
|
|
94
|
+
# Markdown) under the heading, if there's one.
|
|
95
|
+
def initialize(runner, results, settings:, baseline:, candidate:, note: nil)
|
|
96
|
+
@note = note
|
|
72
97
|
@runner = runner
|
|
73
98
|
@results = results
|
|
74
99
|
@settings = settings
|
|
75
|
-
@
|
|
100
|
+
@baseline = baseline
|
|
101
|
+
@candidate = candidate
|
|
76
102
|
@regressed = results.select { |r| Summary.verdict(r) == "regressed" }
|
|
103
|
+
@failed = @regressed + results.select(&:error)
|
|
77
104
|
end
|
|
78
105
|
|
|
79
106
|
# The Markdown: every regression explained, or as many as LIMIT has
|
|
@@ -89,38 +116,89 @@ module Duel
|
|
|
89
116
|
|
|
90
117
|
# The Markdown, with the first +explained+ regressions explained.
|
|
91
118
|
def render(explained)
|
|
92
|
-
|
|
93
|
-
[heading, versions, overview, table(changed), regressions(explained), across, unsettled, errors, unchanged(same)]
|
|
94
|
-
.compact.join("\n\n") + "\n"
|
|
119
|
+
[heading, @note, compared, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
|
|
95
120
|
end
|
|
96
121
|
|
|
122
|
+
# The verdict: what failed (benchmarks, or a metric across the board),
|
|
123
|
+
# or that every benchmark compared passed, and those skipped.
|
|
97
124
|
def heading
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
125
|
+
skipped = @results.count(&:skipped)
|
|
126
|
+
compared = @results.size > skipped
|
|
127
|
+
slower = @runner.regressed_across
|
|
128
|
+
said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
|
|
129
|
+
("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
|
|
130
|
+
mark = if said.any? then FAILED_MARK
|
|
131
|
+
elsif compared then PASSED_MARK
|
|
132
|
+
else NEUTRAL_MARK
|
|
133
|
+
end
|
|
134
|
+
said << (compared ? "100% of benchmarks passed" : "no benchmarks compared") if said.empty?
|
|
135
|
+
said << "#{skipped} skipped" if skipped.positive?
|
|
136
|
+
"### #{mark} Duel: #{said.join(", ")}"
|
|
137
|
+
end
|
|
103
138
|
|
|
104
|
-
|
|
139
|
+
# The two versions, and why any benchmark was skipped.
|
|
140
|
+
def compared
|
|
141
|
+
skipped = @results.count(&:skipped)
|
|
142
|
+
why = " The baseline can't run the skipped #{(skipped == 1) ? "benchmark" : "benchmarks"}." if skipped.positive?
|
|
143
|
+
"Compared #{@candidate} with #{@baseline}.#{why}"
|
|
105
144
|
end
|
|
106
145
|
|
|
107
|
-
#
|
|
108
|
-
#
|
|
109
|
-
def
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
146
|
+
# The benchmarks that failed: what got worse and by how much, or the
|
|
147
|
+
# error, and where the most extra CPU went, if a profile says.
|
|
148
|
+
def failures
|
|
149
|
+
return if @failed.empty?
|
|
150
|
+
|
|
151
|
+
header = ["Benchmark", "Failure"]
|
|
152
|
+
header << "Most extra CPU in" unless @failed.all? { |r| top(r).empty? }
|
|
153
|
+
rows = @failed.map { |r| row([Summary.code(cell(r.name)), failure(r), top(r)].first(header.size)) }
|
|
154
|
+
([row(header), "|#{"---|" * header.size}"] + rows).join("\n")
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# What got worse, or the error's first line.
|
|
158
|
+
def failure(result)
|
|
159
|
+
error = result.error
|
|
160
|
+
return "error: #{Summary.code(cell(error.partition("\n").first))}" if error
|
|
161
|
+
|
|
162
|
+
worse = result.regressed.map do |metric|
|
|
163
|
+
timed = result.timed[metric]
|
|
164
|
+
counted = result.counted[metric]
|
|
165
|
+
if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
|
|
166
|
+
elsif counted
|
|
167
|
+
n = counted.fetch("n")
|
|
168
|
+
format("%s %.4g → %.4g", metric, counted.fetch("baseline").fdiv(n), counted.fetch("candidate").fdiv(n))
|
|
169
|
+
else
|
|
170
|
+
# (Peak memory, which grew too much.)
|
|
171
|
+
memory = result.memory #: Hash[String, Integer]
|
|
172
|
+
"#{metric} #{mb(memory.fetch("baseline"))} → #{mb(memory.fetch("candidate"))}"
|
|
173
|
+
end
|
|
115
174
|
end
|
|
116
|
-
|
|
117
|
-
|
|
175
|
+
worse.join(result.mixed? ? " or " : ", ")
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# The frame whose own share of the CPU grew the most, and by how much,
|
|
179
|
+
# if there's a profile with one (or "").
|
|
180
|
+
def top(result)
|
|
181
|
+
profile = result.profile
|
|
182
|
+
return "" unless profile && !profile.key?("unavailable")
|
|
183
|
+
|
|
184
|
+
# (A Profile's: Steep doesn't narrow on key?.)
|
|
185
|
+
grew = profile.fetch("grew") #: Array[Profile::growth]
|
|
186
|
+
most = grew.first
|
|
187
|
+
return "" unless most
|
|
188
|
+
|
|
189
|
+
frame, own, = most
|
|
190
|
+
format("%s %+.1f%%", Summary.code(cell(frame)), own * 100)
|
|
118
191
|
end
|
|
119
192
|
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
193
|
+
# The rest, collapsed: each regression explained, the errors, every
|
|
194
|
+
# benchmark, the verdicts across the board, the Rubies, and how Duel
|
|
195
|
+
# decides.
|
|
196
|
+
def details(explained)
|
|
197
|
+
unsettled = @results.count { |r| Summary.verdict(r) == "unsettled" }
|
|
198
|
+
title = format("Details: %s in %.0fs%s", Runner.count(@results.size, "benchmark"), @runner.seconds,
|
|
199
|
+
(", #{unsettled} unsettled" if unsettled.positive?))
|
|
200
|
+
body = [*regressions(explained), *errors, table, (UNSETTLED if unsettled.positive?), across(confirmed: false), rubies, decides]
|
|
201
|
+
"<details>\n<summary>#{title}</summary>\n\n#{body.compact.join("\n\n")}\n\n</details>"
|
|
124
202
|
end
|
|
125
203
|
|
|
126
204
|
# The results, in the order of their VERDICTS, and the run's order in
|
|
@@ -130,36 +208,30 @@ module Duel
|
|
|
130
208
|
VERDICTS.flat_map { |shown| grouped.fetch(shown, []) }
|
|
131
209
|
end
|
|
132
210
|
|
|
133
|
-
|
|
134
|
-
|
|
211
|
+
# Every benchmark, with its timings, and its counts where any changed.
|
|
212
|
+
def table
|
|
213
|
+
return if @results.empty?
|
|
135
214
|
|
|
136
|
-
rows =
|
|
215
|
+
rows = sorted.map do |r|
|
|
137
216
|
shown = Summary.verdict(r)
|
|
138
|
-
|
|
217
|
+
[Summary.code(cell(r.name)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r)]
|
|
139
218
|
end
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
# Those that didn't change, collapsed.
|
|
144
|
-
def unchanged(same)
|
|
145
|
-
return if same.empty?
|
|
146
|
-
|
|
147
|
-
rows = same.map { |r| row(Summary.code(name(r)), timings(r)) }
|
|
148
|
-
"<details>\n<summary>#{Runner.count(same.size, "benchmark")} unchanged</summary>\n\n" \
|
|
149
|
-
"#{[row("Benchmark", "Timings (ratio [interval])"), "|---|---|", *rows].join("\n")}\n\n</details>"
|
|
219
|
+
header = ["Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"]
|
|
220
|
+
columns = (rows.any? { |cells| !cells.last.empty? }) ? 4 : 3
|
|
221
|
+
([row(header.first(columns)), "|#{"---|" * columns}"] + rows.map { |cells| row(cells.first(columns)) }).join("\n")
|
|
150
222
|
end
|
|
151
223
|
|
|
152
224
|
# A table's row of +cells+.
|
|
153
|
-
def row(
|
|
225
|
+
def row(cells) = "| #{cells.join(" | ")} |"
|
|
154
226
|
|
|
155
|
-
#
|
|
156
|
-
def
|
|
227
|
+
# +text+ for a table's cell, on one line.
|
|
228
|
+
def cell(text) = text.tr("\n", " ").gsub("|", "\\|")
|
|
157
229
|
|
|
158
230
|
# Each timed measure's ratio and its interval, with its verdict where
|
|
159
231
|
# it isn't the same (or why it was skipped).
|
|
160
232
|
def timings(result)
|
|
161
233
|
skipped = result.skipped
|
|
162
|
-
return skipped
|
|
234
|
+
return cell(skipped) if skipped
|
|
163
235
|
|
|
164
236
|
result.timed.map do |metric, t|
|
|
165
237
|
low, high = t.fetch("interval")
|
|
@@ -186,8 +258,6 @@ module Duel
|
|
|
186
258
|
# The first +explained+ regressions, explained as the terminal report
|
|
187
259
|
# explains them (and the rest named).
|
|
188
260
|
def regressions(explained)
|
|
189
|
-
return if @regressed.empty?
|
|
190
|
-
|
|
191
261
|
shown = @regressed.first(explained).map do |r|
|
|
192
262
|
what = r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement"
|
|
193
263
|
lines = @runner.explained(r)
|
|
@@ -196,42 +266,46 @@ module Duel
|
|
|
196
266
|
end
|
|
197
267
|
rest = @regressed.size - shown.size
|
|
198
268
|
shown << "#{Runner.count(rest, shown.empty? ? "regression" : "more regression")} explained in the job's log." if rest.positive?
|
|
199
|
-
|
|
269
|
+
shown
|
|
200
270
|
end
|
|
201
271
|
|
|
202
272
|
# The verdicts across the board, of the metrics with enough
|
|
203
|
-
# benchmarks to tell
|
|
204
|
-
|
|
273
|
+
# benchmarks to tell: those slower in both measurements, which fail
|
|
274
|
+
# the run (+confirmed+), or the rest.
|
|
275
|
+
def across(confirmed:)
|
|
205
276
|
lines = @runner.across.filter_map do |metric, v|
|
|
206
277
|
low, high = v.fetch("interval")
|
|
207
|
-
next unless low
|
|
278
|
+
next unless low && v.fetch("confirmed", false) == confirmed
|
|
208
279
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
v.fetch("limit"), Runner.count(v.fetch("benchmarks"), "benchmark"))
|
|
280
|
+
format("- %s %.3f [%.3f, %.3f] %s%s (limit %.4f, %s)", metric, v.fetch("median"), low, high, v.fetch("verdict"),
|
|
281
|
+
(" in both measurements" if confirmed), v.fetch("limit"), Runner.count(v.fetch("benchmarks"), "benchmark"))
|
|
212
282
|
end
|
|
213
|
-
"
|
|
214
|
-
end
|
|
215
|
-
|
|
216
|
-
def unsettled
|
|
217
|
-
names = @results.select { |r| Summary.verdict(r) == "unsettled" }.map { |r| Summary.code(r.name) }
|
|
218
|
-
return if names.empty?
|
|
219
|
-
|
|
220
|
-
"#### Unsettled\n\nA timing too close to the limit to settle doesn't fail the run, but one that's unsettled in run " \
|
|
221
|
-
"after run is noisy: #{names.join(", ")}."
|
|
283
|
+
"Across the board:\n\n#{lines.join("\n")}" unless lines.empty?
|
|
222
284
|
end
|
|
223
285
|
|
|
286
|
+
# The errors of more than a line, all of them (up to ERROR_LINES):
|
|
287
|
+
# their first lines are in view.
|
|
224
288
|
def errors
|
|
225
|
-
failed
|
|
226
|
-
return if failed.empty?
|
|
227
|
-
|
|
228
|
-
shown = failed.map do |r|
|
|
289
|
+
@failed.select { |r| r.error.to_s.chomp.include?("\n") }.map do |r|
|
|
229
290
|
error = r.error #: String
|
|
230
291
|
lines = error.lines
|
|
231
292
|
cut = "\n(#{Runner.count(lines.size - ERROR_LINES, "more line")} in the job's log)" if lines.size > ERROR_LINES
|
|
232
293
|
"**#{Summary.code(r.name)}**\n\n#{Summary.block(lines.first(ERROR_LINES).join)}#{cut}"
|
|
233
294
|
end
|
|
234
|
-
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# Each version's Ruby, where they differ.
|
|
298
|
+
def rubies
|
|
299
|
+
rubies = @runner.rubies
|
|
300
|
+
return if rubies.values.uniq.size == 1
|
|
301
|
+
|
|
302
|
+
"Baseline on #{Summary.code(rubies.fetch("baseline"))}, candidate on #{Summary.code(rubies.fetch("candidate"))}."
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
# How Duel decides that a timing is slower, as the settings have it.
|
|
306
|
+
def decides
|
|
307
|
+
format("A timing is slower when a %s%% confidence interval for its ratio, the candidate's to the baseline's, is above %s. " \
|
|
308
|
+
"[How Duel decides](%s)", (@settings.confidence * 100).round(1), @settings.limit, DECIDES)
|
|
235
309
|
end
|
|
236
310
|
end
|
|
237
311
|
end
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
# Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
|
|
2
|
+
#
|
|
3
|
+
# Include it from .gitlab-ci.yml:
|
|
4
|
+
#
|
|
5
|
+
# include:
|
|
6
|
+
# - local: <%= gitlab_file %>
|
|
7
|
+
#
|
|
8
|
+
# It compares a merge request with the branch it targets, a push to the
|
|
9
|
+
# default branch with the commit before it (or as c.push_base in .duel.rb
|
|
10
|
+
# says, the last commit Duel passed), and on a schedule, the last release.
|
|
11
|
+
# The Duel job's log, and its duel.md artifact, have the report. Comments
|
|
12
|
+
# need project access tokens, in masked CI/CD variables:
|
|
13
|
+
#
|
|
14
|
+
# - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
|
|
15
|
+
# protected), to comment on a merge request. Every job of a merge
|
|
16
|
+
# request's pipeline has it, the Duel job too, which runs the merge
|
|
17
|
+
# request's code, so the token can do no more than comment.
|
|
18
|
+
# - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
|
|
19
|
+
# only the default branch's pipelines have it), to comment on a pushed
|
|
20
|
+
# commit (see c.fail_on), and to mark a push passed (see c.push_base).
|
|
21
|
+
|
|
22
|
+
<%= gitlab_job %>:
|
|
23
|
+
image: ruby:<%= gitlab_ruby %>
|
|
24
|
+
stage: test
|
|
25
|
+
variables:
|
|
26
|
+
GIT_DEPTH: 0 # the baseline comes from history, tags included
|
|
27
|
+
DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
|
|
28
|
+
rules:
|
|
29
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
30
|
+
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
31
|
+
variables:
|
|
32
|
+
DUEL_BASE: $CI_COMMIT_BEFORE_SHA
|
|
33
|
+
- if: $CI_PIPELINE_SOURCE == "schedule"
|
|
34
|
+
variables:
|
|
35
|
+
DUEL_BASE: last-release
|
|
36
|
+
# (Run by hand, it compares anyway: with nothing changed, the code
|
|
37
|
+
# with itself, which checks the setup.)
|
|
38
|
+
- if: $CI_PIPELINE_SOURCE == "web"
|
|
39
|
+
variables:
|
|
40
|
+
DUEL_FORCE: "1"
|
|
41
|
+
# A regression outside a merge request, where only a merge request
|
|
42
|
+
# fails (c.fail_on in .duel.rb), exits 3: a warning.
|
|
43
|
+
allow_failure:
|
|
44
|
+
exit_codes: [3]
|
|
45
|
+
before_script:
|
|
46
|
+
<% if working_directory -%>
|
|
47
|
+
- cd <%= working_directory %>
|
|
48
|
+
<% end -%>
|
|
49
|
+
# The branch a merge request targets, the last commits Duel passed,
|
|
50
|
+
# and the releases.
|
|
51
|
+
- git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
|
|
52
|
+
- if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
|
|
53
|
+
- bundle install
|
|
54
|
+
script:
|
|
55
|
+
- bundle exec duel --markdown duel.md
|
|
56
|
+
artifacts:
|
|
57
|
+
when: always
|
|
58
|
+
paths:
|
|
59
|
+
- <%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
60
|
+
reports:
|
|
61
|
+
dotenv: <%= working_directory ? "#{working_directory}/" : "" %>duel.env
|
|
62
|
+
expire_in: 1 day
|
|
63
|
+
|
|
64
|
+
# The report, as a new comment on the merge request, when Duel says to:
|
|
65
|
+
# always (post), or only when its last comment's status differs from this
|
|
66
|
+
# run's (change), so the latest comment is never stale. It never edits or
|
|
67
|
+
# deletes a comment.
|
|
68
|
+
<%= gitlab_job %>:comment:
|
|
69
|
+
image: alpine:3
|
|
70
|
+
stage: .post
|
|
71
|
+
needs: [<%= gitlab_job %>]
|
|
72
|
+
rules:
|
|
73
|
+
- if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
74
|
+
when: always
|
|
75
|
+
variables:
|
|
76
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
77
|
+
script:
|
|
78
|
+
- apk add --no-cache --quiet curl jq
|
|
79
|
+
- |
|
|
80
|
+
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
81
|
+
report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
82
|
+
[ -f "$report" ] || exit 0
|
|
83
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
84
|
+
token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
|
|
85
|
+
marker="<!-- <%= marker %>"
|
|
86
|
+
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
87
|
+
# The first line of the last comment Duel posted here: its marker,
|
|
88
|
+
# with its status.
|
|
89
|
+
last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
|
|
90
|
+
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
|
|
91
|
+
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
92
|
+
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
|
|
93
|
+
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
|
|
94
|
+
fi
|
|
95
|
+
|
|
96
|
+
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
97
|
+
# for a merge request: only where a regression in a push passes (see
|
|
98
|
+
# c.fail_on), so the comment says what the check doesn't. Its last comment
|
|
99
|
+
# is the commit before's.
|
|
100
|
+
<%= gitlab_job %>:commit-comment:
|
|
101
|
+
image: alpine:3
|
|
102
|
+
stage: .post
|
|
103
|
+
needs: [<%= gitlab_job %>]
|
|
104
|
+
rules:
|
|
105
|
+
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
106
|
+
when: always
|
|
107
|
+
variables:
|
|
108
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
109
|
+
script:
|
|
110
|
+
- apk add --no-cache --quiet curl jq
|
|
111
|
+
- |
|
|
112
|
+
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
113
|
+
report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
114
|
+
[ -f "$report" ] || exit 0
|
|
115
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
116
|
+
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
117
|
+
marker="<!-- <%= marker %>"
|
|
118
|
+
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
119
|
+
last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
|
|
120
|
+
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
|
|
121
|
+
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
122
|
+
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
|
|
123
|
+
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
|
|
124
|
+
fi
|
|
125
|
+
|
|
126
|
+
# With c.push_base = :last_passed in .duel.rb, a push is compared with the
|
|
127
|
+
# last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
|
|
128
|
+
# main), so slowdowns too small for any one push to fail add up until they
|
|
129
|
+
# fail one. This job moves the branch to a push that passed (or makes it).
|
|
130
|
+
# To accept a regression, move the branch yourself (for main, git push
|
|
131
|
+
# origin main:<%= passed_branch %>/main).
|
|
132
|
+
<%= gitlab_job %>:passed:
|
|
133
|
+
image: alpine:3
|
|
134
|
+
stage: .post
|
|
135
|
+
needs: [<%= gitlab_job %>]
|
|
136
|
+
rules:
|
|
137
|
+
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
138
|
+
variables:
|
|
139
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
140
|
+
script:
|
|
141
|
+
- apk add --no-cache --quiet curl
|
|
142
|
+
- |
|
|
143
|
+
[ "$DUEL_PASSED" = true ] || exit 0
|
|
144
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
145
|
+
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
146
|
+
branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
|
|
147
|
+
curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
|
|
148
|
+
curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches"
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
#
|
|
5
5
|
# Duel follows conventions, so set only what differs: benchmarks are in
|
|
6
6
|
# duel/**/*_duel.rb, each version's lib/ is first on the load path, a
|
|
7
|
-
# change is compared when it touches lib/ or ext
|
|
8
|
-
# tagged v*.
|
|
7
|
+
# change is compared when it touches lib/ or ext/ (or the benchmarks, or
|
|
8
|
+
# this file), and releases are tagged v*.
|
|
9
9
|
Duel.configure do |c|
|
|
10
10
|
<% if library -%>
|
|
11
11
|
# The library each version's workers require (from the gemspec, unless
|
|
@@ -32,4 +32,22 @@ Duel.configure do |c|
|
|
|
32
32
|
# branch that upgrades Ruby is compared on each version's Ruby.
|
|
33
33
|
c.ruby = :version_file
|
|
34
34
|
<% end -%>
|
|
35
|
+
|
|
36
|
+
# When the workflow comments on a pull request with the report: :always,
|
|
37
|
+
# :results (passed or failed), :failures (failed, or couldn't compare),
|
|
38
|
+
# or :never.
|
|
39
|
+
# c.comment = :failures
|
|
40
|
+
|
|
41
|
+
# Which runs a regression fails: :every_run, or :pull_requests, for a
|
|
42
|
+
# project where people push to main directly (on GitHub Actions, another
|
|
43
|
+
# run warns and passes, and a pushed commit gets the comment instead).
|
|
44
|
+
# c.fail_on = :every_run
|
|
45
|
+
|
|
46
|
+
# A label that accepts a regression in a pull request (nil for none).
|
|
47
|
+
# c.accept_label = "duel: accept"
|
|
48
|
+
|
|
49
|
+
# What the workflow compares a push with: :before (the commit before it),
|
|
50
|
+
# or :last_passed (the last commit Duel passed, which a branch,
|
|
51
|
+
# duel/passed/main, marks), so small slowdowns add up until they fail.
|
|
52
|
+
# c.push_base = :before
|
|
35
53
|
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
name: Perf comment<%= " (#{working_directory})" if working_directory %>
|
|
2
|
+
|
|
3
|
+
# Duel's report, as a new comment on the pull request, when Duel says to:
|
|
4
|
+
# always (post), or only when its last comment's status differs from the
|
|
5
|
+
# run's (change), so the latest comment is never stale. It never edits or
|
|
6
|
+
# deletes a comment. It runs once the Perf workflow has, with this
|
|
7
|
+
# repository's own workflow and a token that can write, so a pull request
|
|
8
|
+
# from a fork gets a comment too. It checks nothing out and runs none of
|
|
9
|
+
# the pull request's code, and of what the Duel job kept (where that code
|
|
10
|
+
# ran), it trusts only what that job could have said: the pull request's
|
|
11
|
+
# number, if it names the pull request the run compared, and the comment
|
|
12
|
+
# and the status, if they're ones Duel says. (The workflow must be on the
|
|
13
|
+
# default branch before it runs at all.)
|
|
14
|
+
on:
|
|
15
|
+
workflow_run:
|
|
16
|
+
workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
|
|
17
|
+
types: [completed]
|
|
18
|
+
|
|
19
|
+
permissions: {}
|
|
20
|
+
|
|
21
|
+
jobs:
|
|
22
|
+
comment:
|
|
23
|
+
name: Comment
|
|
24
|
+
if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
permissions:
|
|
27
|
+
actions: read # to get the report from the Perf run
|
|
28
|
+
pull-requests: write # to comment on the pull request with the report
|
|
29
|
+
steps:
|
|
30
|
+
- name: Get the report
|
|
31
|
+
id: report
|
|
32
|
+
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
33
|
+
uses: actions/download-artifact@v8
|
|
34
|
+
with:
|
|
35
|
+
name: duel-report
|
|
36
|
+
path: ${{ runner.temp }}/report
|
|
37
|
+
run-id: ${{ github.event.workflow_run.id }}
|
|
38
|
+
github-token: ${{ github.token }}
|
|
39
|
+
- name: Comment on the pull request
|
|
40
|
+
if: steps.report.outcome == 'success'
|
|
41
|
+
env:
|
|
42
|
+
GH_TOKEN: ${{ github.token }}
|
|
43
|
+
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
44
|
+
run: |
|
|
45
|
+
said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
|
|
46
|
+
PR=$(said pr '[0-9]+')
|
|
47
|
+
COMMENT=$(said comment 'post|change')
|
|
48
|
+
STATUS=$(said status 'passed|failed|neutral')
|
|
49
|
+
[ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
|
|
50
|
+
[ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
|
|
51
|
+
marker="<!-- <%= marker %>"
|
|
52
|
+
# The first line of the last comment Duel posted here: its marker,
|
|
53
|
+
# with its status (none in Duel 0.1.0's, which counts as another).
|
|
54
|
+
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
55
|
+
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
56
|
+
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
57
|
+
{ echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
|
|
58
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
59
|
+
fi
|