duel 0.1.0 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -16,17 +16,18 @@ module Duel
16
16
  SCHEMA = 1
17
17
 
18
18
  # A run that didn't compare: +status+ ("nothing_to_compare", or
19
- # "error"), its +exit+ status, and +message+, what it said.
20
- def self.note(status, exit, message, settings)
19
+ # "error"), its +exit+ status, +message+, what it said, and what the
20
+ # workflow does about its +comment+ (see CLI.comment).
21
+ def self.note(status, exit, message, settings, comment)
21
22
  none = [] #: Array[value]
22
23
  nothing = {} #: report
23
- head(status, exit, message, settings).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
24
+ head(status, exit, message, settings, comment).merge("seconds" => nil, "baseline" => nil, "candidate" => nil, "plugins" => none,
24
25
  "benchmarks" => none, "across" => nothing, "errors" => none)
25
26
  end
26
27
 
27
28
  # What every report starts with.
28
- def self.head(status, exit, message, settings)
29
- {"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message,
29
+ def self.head(status, exit, message, settings, comment)
30
+ {"schema" => SCHEMA, "duel_version" => VERSION, "status" => status, "exit" => exit, "message" => message, "comment" => comment,
30
31
  "settings" => {"confidence" => settings.confidence, "limit" => settings.limit, "filter" => settings.filter.source,
31
32
  "jobs" => settings.jobs, "processes" => settings.processes, "timeout" => settings.timeout,
32
33
  "warm" => settings.warm, "force" => settings.force}}
@@ -34,19 +35,20 @@ module Duel
34
35
 
35
36
  # +runner+ ran, and made +results+, comparing +baseline+ with
36
37
  # +candidate+ (each {"ref", "commit", "root"} or {"commit",
37
- # "uncommitted_changes", "root"}) as +settings+ say, and exits with
38
- # +exit+.
39
- def initialize(runner, results, settings:, exit:, baseline:, candidate:)
38
+ # "uncommitted_changes", "root"}) as +settings+ say, exits with
39
+ # +exit+, and has the workflow do +comment+ (see CLI.comment).
40
+ def initialize(runner, results, settings:, exit:, comment:, baseline:, candidate:)
40
41
  @runner = runner
41
42
  @results = results
42
43
  @settings = settings
43
44
  @exit = exit
45
+ @comment = comment
44
46
  @versions = {"baseline" => baseline, "candidate" => candidate}
45
47
  end
46
48
 
47
49
  def to_h
48
50
  rubies = @runner.rubies
49
- JSONReport.head(@exit.zero? ? "passed" : "failed", @exit, nil, @settings).merge(
51
+ JSONReport.head(@exit.zero? ? "passed" : "failed", @exit, nil, @settings, @comment).merge(
50
52
  "seconds" => @runner.seconds,
51
53
  **@versions.to_h { |side, version| [side, version.merge("ruby" => rubies.fetch(side))] },
52
54
  "plugins" => @runner.plugins.map(&:to_s),
data/lib/duel/runner.rb CHANGED
@@ -30,7 +30,7 @@ module Duel
30
30
  # How a run is set up (see Duel::CLI for where each comes from).
31
31
  # @api private
32
32
  Settings = Struct.new(:base, :base_dir, :candidate_dir, :filter, :jobs, :processes, :limit, :confidence, :warm, :force,
33
- :config_path, :timeout, :ruby) do
33
+ :config_path, :timeout, :ruby, :comment, :fail_on, :pushed, :target) do
34
34
  # @type self: singleton(Settings)
35
35
  def self.defaults = new(filter: //, jobs: 1, processes: 3, limit: 1.02, confidence: 0.99, warm: nil, force: false,
36
36
  timeout: 60.0)
data/lib/duel/summary.rb CHANGED
@@ -4,11 +4,11 @@ require_relative "runner"
4
4
 
5
5
  module Duel
6
6
  # A run's report in Markdown, for a pull request: GitHub Actions' job
7
- # summary, or a comment (see CLI). It has the verdict, the two versions
8
- # (and their Rubies, where they differ), a table of the benchmarks that
9
- # changed, regressions first, each regression explained as the terminal
10
- # report explains it, those unsettled and those that failed. The rest
11
- # are collapsed, so it stays short with hundreds of benchmarks.
7
+ # summary, or a comment (see CLI). In view, only what the author needs:
8
+ # the verdict, the two commits, and what failed, in a table. The rest is
9
+ # collapsed: what made each regression, every benchmark's timings, the
10
+ # verdicts across the board, each version's Ruby where they differ, and
11
+ # how Duel decides. So it stays short with hundreds of benchmarks.
12
12
  # @api private
13
13
  class Summary
14
14
  # The most characters it has (a GitHub comment has at most 65,536):
@@ -26,6 +26,19 @@ module Duel
26
26
  # The verdicts that fail a run, which stand out.
27
27
  FAILED = %w[regressed error].freeze
28
28
 
29
+ # What the heading starts with: a run that passed, one that failed (or
30
+ # couldn't compare), and one with nothing compared.
31
+ PASSED_MARK = "🟢"
32
+ FAILED_MARK = "🔻"
33
+ NEUTRAL_MARK = "⬜"
34
+
35
+ # What an unsettled timing means.
36
+ UNSETTLED = "An unsettled timing was too close to the limit to settle. It doesn't fail the run, but one that's unsettled " \
37
+ "in run after run is noisy."
38
+
39
+ # How Duel decides that a timing is slower, in its docs.
40
+ DECIDES = "https://github.com/sferik/duel-ruby/blob/main/docs/how-it-works.md#how-duel-tells-a-regression-from-noise"
41
+
29
42
  # +text+ as code, in Markdown: in a span of more backticks than any it
30
43
  # has (and spaced from one at either end).
31
44
  def self.code(text)
@@ -34,8 +47,17 @@ module Duel
34
47
  "#{ticks}#{padded}#{ticks}"
35
48
  end
36
49
 
37
- # A run that didn't compare (+title+ says why), with what it said.
38
- def self.note(title, message) = "### Duel: #{title}\n\n#{block(message)}\n"
50
+ # +text+ as it is, in Markdown: what Markdown, or the HTML in it, would
51
+ # take for something else, escaped.
52
+ def self.escape(text) = text.gsub(/[\\`*_\[\]<>|~&#]/) { |char| "\\#{char}" }
53
+
54
+ # A run that didn't compare: a heading of +mark+ and +title+, and the
55
+ # first line of +reason+, with all of it collapsed if it has more.
56
+ def self.note(mark, title, reason)
57
+ first, _, rest = reason.partition("\n")
58
+ more = "\n\n<details>\n<summary>The whole message</summary>\n\n#{block(reason)}\n\n</details>" unless rest.empty?
59
+ "### #{mark} Duel: #{title}\n\n#{escape(first)}#{more}\n"
60
+ end
39
61
 
40
62
  # +text+ in a fenced block of its own: fenced with more backticks than
41
63
  # any run of them it has, three at least.
@@ -67,13 +89,18 @@ module Duel
67
89
  end
68
90
 
69
91
  # +runner+ ran, and made +results+, comparing +baseline+ with
70
- # +candidate+ (each a description, in Markdown) as +settings+ say.
71
- def initialize(runner, results, settings:, baseline:, candidate:)
92
+ # +candidate+ (each a description, in Markdown, whose commit stands
93
+ # bare, so GitHub links it) as +settings+ say, with a +note+ (in
94
+ # Markdown) under the heading, if there's one.
95
+ def initialize(runner, results, settings:, baseline:, candidate:, note: nil)
96
+ @note = note
72
97
  @runner = runner
73
98
  @results = results
74
99
  @settings = settings
75
- @versions = {"baseline" => baseline, "candidate" => candidate}
100
+ @baseline = baseline
101
+ @candidate = candidate
76
102
  @regressed = results.select { |r| Summary.verdict(r) == "regressed" }
103
+ @failed = @regressed + results.select(&:error)
77
104
  end
78
105
 
79
106
  # The Markdown: every regression explained, or as many as LIMIT has
@@ -89,38 +116,89 @@ module Duel
89
116
 
90
117
  # The Markdown, with the first +explained+ regressions explained.
91
118
  def render(explained)
92
- same, changed = sorted.partition { |r| Summary.verdict(r) == "same" }
93
- [heading, versions, overview, table(changed), regressions(explained), across, unsettled, errors, unchanged(same)]
94
- .compact.join("\n\n") + "\n"
119
+ [heading, @note, compared, failures, across(confirmed: true), details(explained)].compact.join("\n\n") + "\n"
95
120
  end
96
121
 
122
+ # The verdict: what failed (benchmarks, or a metric across the board),
123
+ # or that every benchmark compared passed, and those skipped.
97
124
  def heading
98
- errors = @results.count(&:error)
99
- failed = [(Runner.count(@regressed.size, "regression") unless @regressed.empty?),
100
- ("slower across the board" unless @runner.regressed_across.empty?),
101
- (Runner.count(errors, "error") unless errors.zero?)].compact
102
- return "### Duel failed: #{failed.join(", ")}" unless failed.empty?
125
+ skipped = @results.count(&:skipped)
126
+ compared = @results.size > skipped
127
+ slower = @runner.regressed_across
128
+ said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
129
+ ("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
130
+ mark = if said.any? then FAILED_MARK
131
+ elsif compared then PASSED_MARK
132
+ else NEUTRAL_MARK
133
+ end
134
+ said << (compared ? "100% of benchmarks passed" : "no benchmarks compared") if said.empty?
135
+ said << "#{skipped} skipped" if skipped.positive?
136
+ "### #{mark} Duel: #{said.join(", ")}"
137
+ end
103
138
 
104
- "### Duel passed: none of #{Runner.count(@results.size, "benchmark")} slower"
139
+ # The two versions, and why any benchmark was skipped.
140
+ def compared
141
+ skipped = @results.count(&:skipped)
142
+ why = " The baseline can't run the skipped #{(skipped == 1) ? "benchmark" : "benchmarks"}." if skipped.positive?
143
+ "Compared #{@candidate} with #{@baseline}.#{why}"
105
144
  end
106
145
 
107
- # Each version, on its Ruby where theirs differ, and the plugins
108
- # used.
109
- def versions
110
- rubies = @runner.rubies
111
- plugins = @runner.plugins
112
- lines = @versions.map do |side, described|
113
- ruby = " on #{Summary.code(rubies.fetch(side))}" if rubies.values.uniq.size > 1
114
- "- **#{side.capitalize}:** #{described}#{ruby}"
146
+ # The benchmarks that failed: what got worse and by how much, or the
147
+ # error, and where the most extra CPU went, if a profile says.
148
+ def failures
149
+ return if @failed.empty?
150
+
151
+ header = ["Benchmark", "Failure"]
152
+ header << "Most extra CPU in" unless @failed.all? { |r| top(r).empty? }
153
+ rows = @failed.map { |r| row([Summary.code(cell(r.name)), failure(r), top(r)].first(header.size)) }
154
+ ([row(header), "|#{"---|" * header.size}"] + rows).join("\n")
155
+ end
156
+
157
+ # What got worse, or the error's first line.
158
+ def failure(result)
159
+ error = result.error
160
+ return "error: #{Summary.code(cell(error.partition("\n").first))}" if error
161
+
162
+ worse = result.regressed.map do |metric|
163
+ timed = result.timed[metric]
164
+ counted = result.counted[metric]
165
+ if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
166
+ elsif counted
167
+ n = counted.fetch("n")
168
+ format("%s %.4g → %.4g", metric, counted.fetch("baseline").fdiv(n), counted.fetch("candidate").fdiv(n))
169
+ else
170
+ # (Peak memory, which grew too much.)
171
+ memory = result.memory #: Hash[String, Integer]
172
+ "#{metric} #{mb(memory.fetch("baseline"))} → #{mb(memory.fetch("candidate"))}"
173
+ end
115
174
  end
116
- lines << "- **Plugins:** #{plugins.map { |plugin| Summary.code(plugin.to_s) }.join(", ")}" unless plugins.empty?
117
- lines.join("\n")
175
+ worse.join(result.mixed? ? " or " : ", ")
176
+ end
177
+
178
+ # The frame whose own share of the CPU grew the most, and by how much,
179
+ # if there's a profile with one (or "").
180
+ def top(result)
181
+ profile = result.profile
182
+ return "" unless profile && !profile.key?("unavailable")
183
+
184
+ # (A Profile's: Steep doesn't narrow on key?.)
185
+ grew = profile.fetch("grew") #: Array[Profile::growth]
186
+ most = grew.first
187
+ return "" unless most
188
+
189
+ frame, own, = most
190
+ format("%s %+.1f%%", Summary.code(cell(frame)), own * 100)
118
191
  end
119
192
 
120
- def overview
121
- format("%s in %.0fs. A timing is slower when a %s%% confidence interval for its ratio, the candidate's to the " \
122
- "baseline's, is above %s.", Runner.count(@results.size, "benchmark"), @runner.seconds,
123
- (@settings.confidence * 100).round(1), @settings.limit)
193
+ # The rest, collapsed: each regression explained, the errors, every
194
+ # benchmark, the verdicts across the board, the Rubies, and how Duel
195
+ # decides.
196
+ def details(explained)
197
+ unsettled = @results.count { |r| Summary.verdict(r) == "unsettled" }
198
+ title = format("Details: %s in %.0fs%s", Runner.count(@results.size, "benchmark"), @runner.seconds,
199
+ (", #{unsettled} unsettled" if unsettled.positive?))
200
+ body = [*regressions(explained), *errors, table, (UNSETTLED if unsettled.positive?), across(confirmed: false), rubies, decides]
201
+ "<details>\n<summary>#{title}</summary>\n\n#{body.compact.join("\n\n")}\n\n</details>"
124
202
  end
125
203
 
126
204
  # The results, in the order of their VERDICTS, and the run's order in
@@ -130,36 +208,30 @@ module Duel
130
208
  VERDICTS.flat_map { |shown| grouped.fetch(shown, []) }
131
209
  end
132
210
 
133
- def table(changed)
134
- return if changed.empty?
211
+ # Every benchmark, with its timings, and its counts where any changed.
212
+ def table
213
+ return if @results.empty?
135
214
 
136
- rows = changed.map do |r|
215
+ rows = sorted.map do |r|
137
216
  shown = Summary.verdict(r)
138
- row(Summary.code(name(r)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r))
217
+ [Summary.code(cell(r.name)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r)]
139
218
  end
140
- ([row("Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"), "|---|---|---|---|"] + rows).join("\n")
141
- end
142
-
143
- # Those that didn't change, collapsed.
144
- def unchanged(same)
145
- return if same.empty?
146
-
147
- rows = same.map { |r| row(Summary.code(name(r)), timings(r)) }
148
- "<details>\n<summary>#{Runner.count(same.size, "benchmark")} unchanged</summary>\n\n" \
149
- "#{[row("Benchmark", "Timings (ratio [interval])"), "|---|---|", *rows].join("\n")}\n\n</details>"
219
+ header = ["Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"]
220
+ columns = (rows.any? { |cells| !cells.last.empty? }) ? 4 : 3
221
+ ([row(header.first(columns)), "|#{"---|" * columns}"] + rows.map { |cells| row(cells.first(columns)) }).join("\n")
150
222
  end
151
223
 
152
224
  # A table's row of +cells+.
153
- def row(*cells) = "| #{cells.join(" | ")} |"
225
+ def row(cells) = "| #{cells.join(" | ")} |"
154
226
 
155
- # A benchmark's name, for a table's cell.
156
- def name(result) = result.name.tr("\n", " ").gsub("|", "\\|")
227
+ # +text+ for a table's cell, on one line.
228
+ def cell(text) = text.tr("\n", " ").gsub("|", "\\|")
157
229
 
158
230
  # Each timed measure's ratio and its interval, with its verdict where
159
231
  # it isn't the same (or why it was skipped).
160
232
  def timings(result)
161
233
  skipped = result.skipped
162
- return skipped.gsub("|", "\\|") if skipped
234
+ return cell(skipped) if skipped
163
235
 
164
236
  result.timed.map do |metric, t|
165
237
  low, high = t.fetch("interval")
@@ -186,8 +258,6 @@ module Duel
186
258
  # The first +explained+ regressions, explained as the terminal report
187
259
  # explains them (and the rest named).
188
260
  def regressions(explained)
189
- return if @regressed.empty?
190
-
191
261
  shown = @regressed.first(explained).map do |r|
192
262
  what = r.mixed? ? "slower in every measurement, by one of #{r.regressed.join(", ")}" : "worse #{r.regressed.join(", ")} in every measurement"
193
263
  lines = @runner.explained(r)
@@ -196,42 +266,46 @@ module Duel
196
266
  end
197
267
  rest = @regressed.size - shown.size
198
268
  shown << "#{Runner.count(rest, shown.empty? ? "regression" : "more regression")} explained in the job's log." if rest.positive?
199
- "#### Regressions\n\n#{shown.join("\n\n")}"
269
+ shown
200
270
  end
201
271
 
202
272
  # The verdicts across the board, of the metrics with enough
203
- # benchmarks to tell.
204
- def across
273
+ # benchmarks to tell: those slower in both measurements, which fail
274
+ # the run (+confirmed+), or the rest.
275
+ def across(confirmed:)
205
276
  lines = @runner.across.filter_map do |metric, v|
206
277
  low, high = v.fetch("interval")
207
- next unless low
278
+ next unless low && v.fetch("confirmed", false) == confirmed
208
279
 
209
- confirmed = " in both measurements" if v["confirmed"]
210
- format("- %s %.3f [%.3f, %.3f] %s%s (limit %.4f, %s)", metric, v.fetch("median"), low, high, v.fetch("verdict"), confirmed,
211
- v.fetch("limit"), Runner.count(v.fetch("benchmarks"), "benchmark"))
280
+ format("- %s %.3f [%.3f, %.3f] %s%s (limit %.4f, %s)", metric, v.fetch("median"), low, high, v.fetch("verdict"),
281
+ (" in both measurements" if confirmed), v.fetch("limit"), Runner.count(v.fetch("benchmarks"), "benchmark"))
212
282
  end
213
- "#### Across the board\n\n#{lines.join("\n")}" unless lines.empty?
214
- end
215
-
216
- def unsettled
217
- names = @results.select { |r| Summary.verdict(r) == "unsettled" }.map { |r| Summary.code(r.name) }
218
- return if names.empty?
219
-
220
- "#### Unsettled\n\nA timing too close to the limit to settle doesn't fail the run, but one that's unsettled in run " \
221
- "after run is noisy: #{names.join(", ")}."
283
+ "Across the board:\n\n#{lines.join("\n")}" unless lines.empty?
222
284
  end
223
285
 
286
+ # The errors of more than a line, all of them (up to ERROR_LINES):
287
+ # their first lines are in view.
224
288
  def errors
225
- failed = @results.select(&:error)
226
- return if failed.empty?
227
-
228
- shown = failed.map do |r|
289
+ @failed.select { |r| r.error.to_s.chomp.include?("\n") }.map do |r|
229
290
  error = r.error #: String
230
291
  lines = error.lines
231
292
  cut = "\n(#{Runner.count(lines.size - ERROR_LINES, "more line")} in the job's log)" if lines.size > ERROR_LINES
232
293
  "**#{Summary.code(r.name)}**\n\n#{Summary.block(lines.first(ERROR_LINES).join)}#{cut}"
233
294
  end
234
- "#### Errors\n\n#{shown.join("\n\n")}"
295
+ end
296
+
297
+ # Each version's Ruby, where they differ.
298
+ def rubies
299
+ rubies = @runner.rubies
300
+ return if rubies.values.uniq.size == 1
301
+
302
+ "Baseline on #{Summary.code(rubies.fetch("baseline"))}, candidate on #{Summary.code(rubies.fetch("candidate"))}."
303
+ end
304
+
305
+ # How Duel decides that a timing is slower, as the settings have it.
306
+ def decides
307
+ format("A timing is slower when a %s%% confidence interval for its ratio, the candidate's to the baseline's, is above %s. " \
308
+ "[How Duel decides](%s)", (@settings.confidence * 100).round(1), @settings.limit, DECIDES)
235
309
  end
236
310
  end
237
311
  end
@@ -0,0 +1,148 @@
1
+ # Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
2
+ #
3
+ # Include it from .gitlab-ci.yml:
4
+ #
5
+ # include:
6
+ # - local: <%= gitlab_file %>
7
+ #
8
+ # It compares a merge request with the branch it targets, a push to the
9
+ # default branch with the commit before it (or as c.push_base in .duel.rb
10
+ # says, the last commit Duel passed), and on a schedule, the last release.
11
+ # The Duel job's log, and its duel.md artifact, have the report. Comments
12
+ # need project access tokens, in masked CI/CD variables:
13
+ #
14
+ # - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
15
+ # protected), to comment on a merge request. Every job of a merge
16
+ # request's pipeline has it, the Duel job too, which runs the merge
17
+ # request's code, so the token can do no more than comment.
18
+ # - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
19
+ # only the default branch's pipelines have it), to comment on a pushed
20
+ # commit (see c.fail_on), and to mark a push passed (see c.push_base).
21
+
22
+ <%= gitlab_job %>:
23
+ image: ruby:<%= gitlab_ruby %>
24
+ stage: test
25
+ variables:
26
+ GIT_DEPTH: 0 # the baseline comes from history, tags included
27
+ DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
28
+ rules:
29
+ - if: $CI_PIPELINE_SOURCE == "merge_request_event"
30
+ - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
31
+ variables:
32
+ DUEL_BASE: $CI_COMMIT_BEFORE_SHA
33
+ - if: $CI_PIPELINE_SOURCE == "schedule"
34
+ variables:
35
+ DUEL_BASE: last-release
36
+ # (Run by hand, it compares anyway: with nothing changed, the code
37
+ # with itself, which checks the setup.)
38
+ - if: $CI_PIPELINE_SOURCE == "web"
39
+ variables:
40
+ DUEL_FORCE: "1"
41
+ # A regression outside a merge request, where only a merge request
42
+ # fails (c.fail_on in .duel.rb), exits 3: a warning.
43
+ allow_failure:
44
+ exit_codes: [3]
45
+ before_script:
46
+ <% if working_directory -%>
47
+ - cd <%= working_directory %>
48
+ <% end -%>
49
+ # The branch a merge request targets, the last commits Duel passed,
50
+ # and the releases.
51
+ - git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
52
+ - if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
53
+ - bundle install
54
+ script:
55
+ - bundle exec duel --markdown duel.md
56
+ artifacts:
57
+ when: always
58
+ paths:
59
+ - <%= working_directory ? "#{working_directory}/" : "" %>duel.md
60
+ reports:
61
+ dotenv: <%= working_directory ? "#{working_directory}/" : "" %>duel.env
62
+ expire_in: 1 day
63
+
64
+ # The report, as a new comment on the merge request, when Duel says to:
65
+ # always (post), or only when its last comment's status differs from this
66
+ # run's (change), so the latest comment is never stale. It never edits or
67
+ # deletes a comment.
68
+ <%= gitlab_job %>:comment:
69
+ image: alpine:3
70
+ stage: .post
71
+ needs: [<%= gitlab_job %>]
72
+ rules:
73
+ - if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
74
+ when: always
75
+ variables:
76
+ GIT_STRATEGY: none # it runs none of the code
77
+ script:
78
+ - apk add --no-cache --quiet curl jq
79
+ - |
80
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
81
+ report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
82
+ [ -f "$report" ] || exit 0
83
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
84
+ token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
85
+ marker="<!-- <%= marker %>"
86
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
87
+ # The first line of the last comment Duel posted here: its marker,
88
+ # with its status.
89
+ last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
90
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
91
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
92
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
93
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
94
+ fi
95
+
96
+ # The report, as a new comment on a pushed commit, when Duel says to, as
97
+ # for a merge request: only where a regression in a push passes (see
98
+ # c.fail_on), so the comment says what the check doesn't. Its last comment
99
+ # is the commit before's.
100
+ <%= gitlab_job %>:commit-comment:
101
+ image: alpine:3
102
+ stage: .post
103
+ needs: [<%= gitlab_job %>]
104
+ rules:
105
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
106
+ when: always
107
+ variables:
108
+ GIT_STRATEGY: none # it runs none of the code
109
+ script:
110
+ - apk add --no-cache --quiet curl jq
111
+ - |
112
+ case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
113
+ report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
114
+ [ -f "$report" ] || exit 0
115
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
116
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
117
+ marker="<!-- <%= marker %>"
118
+ me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
119
+ last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
120
+ jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
121
+ if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
122
+ { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
123
+ curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
124
+ fi
125
+
126
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with the
127
+ # last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
128
+ # main), so slowdowns too small for any one push to fail add up until they
129
+ # fail one. This job moves the branch to a push that passed (or makes it).
130
+ # To accept a regression, move the branch yourself (for main, git push
131
+ # origin main:<%= passed_branch %>/main).
132
+ <%= gitlab_job %>:passed:
133
+ image: alpine:3
134
+ stage: .post
135
+ needs: [<%= gitlab_job %>]
136
+ rules:
137
+ - if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
138
+ variables:
139
+ GIT_STRATEGY: none # it runs none of the code
140
+ script:
141
+ - apk add --no-cache --quiet curl
142
+ - |
143
+ [ "$DUEL_PASSED" = true ] || exit 0
144
+ api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
145
+ token="PRIVATE-TOKEN: $DUEL_TOKEN"
146
+ branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
147
+ curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
148
+ curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches"
@@ -4,8 +4,8 @@
4
4
  #
5
5
  # Duel follows conventions, so set only what differs: benchmarks are in
6
6
  # duel/**/*_duel.rb, each version's lib/ is first on the load path, a
7
- # change is compared when it touches lib/ or ext/, and releases are
8
- # tagged v*.
7
+ # change is compared when it touches lib/ or ext/ (or the benchmarks, or
8
+ # this file), and releases are tagged v*.
9
9
  Duel.configure do |c|
10
10
  <% if library -%>
11
11
  # The library each version's workers require (from the gemspec, unless
@@ -32,4 +32,22 @@ Duel.configure do |c|
32
32
  # branch that upgrades Ruby is compared on each version's Ruby.
33
33
  c.ruby = :version_file
34
34
  <% end -%>
35
+
36
+ # When the workflow comments on a pull request with the report: :always,
37
+ # :results (passed or failed), :failures (failed, or couldn't compare),
38
+ # or :never.
39
+ # c.comment = :failures
40
+
41
+ # Which runs a regression fails: :every_run, or :pull_requests, for a
42
+ # project where people push to main directly (on GitHub Actions, another
43
+ # run warns and passes, and a pushed commit gets the comment instead).
44
+ # c.fail_on = :every_run
45
+
46
+ # A label that accepts a regression in a pull request (nil for none).
47
+ # c.accept_label = "duel: accept"
48
+
49
+ # What the workflow compares a push with: :before (the commit before it),
50
+ # or :last_passed (the last commit Duel passed, which a branch,
51
+ # duel/passed/main, marks), so small slowdowns add up until they fail.
52
+ # c.push_base = :before
35
53
  end
@@ -0,0 +1,59 @@
1
+ name: Perf comment<%= " (#{working_directory})" if working_directory %>
2
+
3
+ # Duel's report, as a new comment on the pull request, when Duel says to:
4
+ # always (post), or only when its last comment's status differs from the
5
+ # run's (change), so the latest comment is never stale. It never edits or
6
+ # deletes a comment. It runs once the Perf workflow has, with this
7
+ # repository's own workflow and a token that can write, so a pull request
8
+ # from a fork gets a comment too. It checks nothing out and runs none of
9
+ # the pull request's code, and of what the Duel job kept (where that code
10
+ # ran), it trusts only what that job could have said: the pull request's
11
+ # number, if it names the pull request the run compared, and the comment
12
+ # and the status, if they're ones Duel says. (The workflow must be on the
13
+ # default branch before it runs at all.)
14
+ on:
15
+ workflow_run:
16
+ workflows: ["Perf<%= " (#{working_directory})" if working_directory %>"]
17
+ types: [completed]
18
+
19
+ permissions: {}
20
+
21
+ jobs:
22
+ comment:
23
+ name: Comment
24
+ if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
25
+ runs-on: ubuntu-latest
26
+ permissions:
27
+ actions: read # to get the report from the Perf run
28
+ pull-requests: write # to comment on the pull request with the report
29
+ steps:
30
+ - name: Get the report
31
+ id: report
32
+ continue-on-error: true # there's none when the Duel job failed before comparing
33
+ uses: actions/download-artifact@v8
34
+ with:
35
+ name: duel-report
36
+ path: ${{ runner.temp }}/report
37
+ run-id: ${{ github.event.workflow_run.id }}
38
+ github-token: ${{ github.token }}
39
+ - name: Comment on the pull request
40
+ if: steps.report.outcome == 'success'
41
+ env:
42
+ GH_TOKEN: ${{ github.token }}
43
+ HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
44
+ run: |
45
+ said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
46
+ PR=$(said pr '[0-9]+')
47
+ COMMENT=$(said comment 'post|change')
48
+ STATUS=$(said status 'passed|failed|neutral')
49
+ [ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
50
+ [ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
51
+ marker="<!-- <%= marker %>"
52
+ # The first line of the last comment Duel posted here: its marker,
53
+ # with its status (none in Duel 0.1.0's, which counts as another).
54
+ last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
55
+ --jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
56
+ if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
57
+ { echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
58
+ gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
59
+ fi