duel 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/AGENT_PROMPT.md +51 -27
  3. data/CHANGELOG.md +247 -1
  4. data/README.md +30 -14
  5. data/docs/benchmarks.md +28 -24
  6. data/docs/ci.md +328 -362
  7. data/docs/configuration.md +40 -37
  8. data/docs/how-it-works.md +23 -10
  9. data/docs/json.md +21 -25
  10. data/docs/plugins.md +40 -22
  11. data/docs/troubleshooting.md +3 -2
  12. data/lib/duel/across.rb +18 -18
  13. data/lib/duel/badge.rb +174 -0
  14. data/lib/duel/baseline.rb +38 -15
  15. data/lib/duel/cli.rb +175 -107
  16. data/lib/duel/comment.rb +80 -71
  17. data/lib/duel/config.rb +145 -63
  18. data/lib/duel/context.rb +29 -24
  19. data/lib/duel/forge.rb +87 -0
  20. data/lib/duel/init.rb +44 -87
  21. data/lib/duel/json_report.rb +34 -25
  22. data/lib/duel/lane.rb +107 -61
  23. data/lib/duel/lock.rb +1 -1
  24. data/lib/duel/mark_passed.rb +132 -0
  25. data/lib/duel/metric.rb +28 -27
  26. data/lib/duel/native.rb +7 -5
  27. data/lib/duel/platform.rb +4 -0
  28. data/lib/duel/process_sample.rb +9 -6
  29. data/lib/duel/rake_task.rb +4 -5
  30. data/lib/duel/resource.rb +10 -11
  31. data/lib/duel/result.rb +22 -14
  32. data/lib/duel/rubies.rb +6 -2
  33. data/lib/duel/runner.rb +27 -15
  34. data/lib/duel/spawn.rb +29 -4
  35. data/lib/duel/stats.rb +14 -12
  36. data/lib/duel/summary.rb +31 -29
  37. data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
  38. data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
  39. data/lib/duel/templates/duel.rb.erb +7 -7
  40. data/lib/duel/templates/perf-caller.yml.erb +22 -5
  41. data/lib/duel/templates/perf-comment.yml.erb +11 -11
  42. data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
  43. data/lib/duel/templates/perf-push.yml.erb +50 -0
  44. data/lib/duel/templates/perf.yml.erb +113 -335
  45. data/lib/duel/testing.rb +20 -25
  46. data/lib/duel/version.rb +1 -1
  47. data/lib/duel/worker.rb +16 -9
  48. data/lib/duel/worker_process.rb +6 -6
  49. data/lib/duel.rb +88 -33
  50. data/sig/duel/across.rbs +4 -0
  51. data/sig/duel/badge.rbs +51 -0
  52. data/sig/duel/baseline.rbs +7 -2
  53. data/sig/duel/cli.rbs +31 -20
  54. data/sig/duel/comment.rbs +14 -15
  55. data/sig/duel/config.rbs +71 -29
  56. data/sig/duel/context.rbs +4 -4
  57. data/sig/duel/forge.rbs +42 -0
  58. data/sig/duel/init.rbs +8 -26
  59. data/sig/duel/json_report.rbs +7 -6
  60. data/sig/duel/lane.rbs +29 -6
  61. data/sig/duel/mark_passed.rbs +39 -0
  62. data/sig/duel/metric.rbs +10 -10
  63. data/sig/duel/native.rbs +3 -0
  64. data/sig/duel/platform.rbs +1 -0
  65. data/sig/duel/process_sample.rbs +3 -0
  66. data/sig/duel/resource.rbs +3 -3
  67. data/sig/duel/result.rbs +12 -5
  68. data/sig/duel/rubies.rbs +3 -0
  69. data/sig/duel/runner.rbs +23 -16
  70. data/sig/duel/spawn.rbs +7 -0
  71. data/sig/duel/stats.rbs +4 -1
  72. data/sig/duel/summary.rbs +8 -0
  73. data/sig/duel/testing.rbs +2 -2
  74. data/sig/duel/worker.rbs +6 -5
  75. data/sig/duel/worker_process.rbs +3 -3
  76. data/sig/duel.rbs +50 -31
  77. metadata +9 -1
data/lib/duel/stats.rb CHANGED
@@ -93,13 +93,7 @@ module Duel
93
93
  # [low, high] bounds of the +confidence+ interval for the ratio of
94
94
  # +candidates+ (a version's samples) to +bases+ (the other's), or nil
95
95
  # (see #shift_rank).
96
- def shift_interval(bases, candidates, confidence)
97
- r = shift_rank(bases.size, candidates.size, confidence)
98
- return unless r
99
-
100
- ratios = cross_ratios(bases, candidates)
101
- [ratios.fetch(r - 1), ratios.fetch(-r)]
102
- end
96
+ def shift_interval(bases, candidates, confidence) = rank_interval(bases, candidates, confidence) { cross_ratios(bases, candidates) }
103
97
 
104
98
  # The ratio of +candidates+ to +bases+ the interval is for: the
105
99
  # median of the cross ratios (the Hodges-Lehmann estimate).
@@ -109,11 +103,19 @@ module Duel
109
103
  # +candidates+ (a version's samples) are than +bases+ (the other's),
110
104
  # as #shift_interval's are for how many times as much, or nil.
111
105
  def difference_interval(bases, candidates, confidence)
106
+ rank_interval(bases, candidates, confidence) { candidates.flat_map { |c| bases.map { |b| c - b } }.sort }
107
+ end
108
+
109
+ # The bounds #shift_interval and #difference_interval take, of the
110
+ # sorted values the block gives (one for each pair of a candidate's
111
+ # and a base's): the rth least and the rth most (see #shift_rank),
112
+ # or nil, without asking for the values.
113
+ def rank_interval(bases, candidates, confidence)
112
114
  r = shift_rank(bases.size, candidates.size, confidence)
113
115
  return unless r
114
116
 
115
- differences = candidates.flat_map { |c| bases.map { |b| c - b } }.sort
116
- [differences.fetch(r - 1), differences.fetch(-r)]
117
+ sorted = yield
118
+ [sorted.fetch(r - 1), sorted.fetch(-r)]
117
119
  end
118
120
 
119
121
  # Each of +candidates+ over each of +bases+, sorted.
@@ -132,17 +134,17 @@ module Duel
132
134
  # What a confidence +interval+ for the median ratio (or nil, for too
133
135
  # few) says against +limit+: :slower when it lies above the limit,
134
136
  # :faster when it lies below 1 / limit, :same when it lies below the
135
- # limit (it isn't slower: nothing more needs proving), and :undecided
137
+ # limit (it isn't slower: nothing more needs proving), and :unsettled
136
138
  # while it straddles the limit.
137
139
  def decide(interval, limit)
138
- return :undecided unless interval
140
+ return :unsettled unless interval
139
141
 
140
142
  low, high = interval
141
143
  return :slower if low > limit
142
144
  return :faster if high < 1 / limit
143
145
  return :same if high < limit
144
146
 
145
- :undecided
147
+ :unsettled
146
148
  end
147
149
  end
148
150
  end
data/lib/duel/summary.rb CHANGED
@@ -83,12 +83,7 @@ module Duel
83
83
 
84
84
  # {metric => [baseline, candidate]}, an iteration, of the counted
85
85
  # measures that differ.
86
- def self.changed_counts(result)
87
- result.counted.to_h do |metric, c|
88
- n = c.fetch("n")
89
- [metric, [c.fetch("baseline").fdiv(n), c.fetch("candidate").fdiv(n)]] #: [String, [Float, Float]]
90
- end.reject { |_, (base, candidate)| base == candidate }
91
- end
86
+ def self.changed_counts(result) = result.counted.transform_values { |c| Result.per_iteration(c) }.reject { |_, (base, candidate)| base == candidate }
92
87
 
93
88
  # +runner+ ran, and made +results+, comparing +baseline+ with
94
89
  # +candidate+ (each a description, in Markdown, whose commit stands
@@ -103,7 +98,11 @@ module Duel
103
98
  @settings = settings
104
99
  @baseline = baseline
105
100
  @candidate = candidate
106
- @regressed = results.select { |r| Summary.verdict(r) == "regressed" }
101
+ # (Each result's verdict, worked out once: rendering again, to fit
102
+ # LIMIT, asks for them all again.)
103
+ @verdicts = results.to_h { |r| [r, Summary.verdict(r)] }
104
+ @regressed = results.select { |r| @verdicts.fetch(r) == "regressed" }
105
+ @skipped = results.count(&:skipped)
107
106
  @failed = @regressed + results.select(&:error)
108
107
  end
109
108
 
@@ -127,8 +126,7 @@ module Duel
127
126
  # and whether that was accepted, or that every benchmark compared
128
127
  # passed, and those skipped.
129
128
  def heading
130
- skipped = @results.count(&:skipped)
131
- compared = @results.size > skipped
129
+ compared = @results.size > @skipped
132
130
  slower = @runner.regressed_across
133
131
  said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
134
132
  ("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
@@ -138,14 +136,13 @@ module Duel
138
136
  else NEUTRAL_MARK
139
137
  end
140
138
  said << (compared ? "100% of benchmarks passed" : "no benchmarks compared") if said.empty?
141
- said << "#{skipped} skipped" if skipped.positive?
139
+ said << "#{@skipped} skipped" if @skipped.positive?
142
140
  "### #{mark} Duel: #{said.join(", ")}"
143
141
  end
144
142
 
145
143
  # The two versions, and why any benchmark was skipped.
146
144
  def compared
147
- skipped = @results.count(&:skipped)
148
- why = " The baseline can't run the skipped #{(skipped == 1) ? "benchmark" : "benchmarks"}." if skipped.positive?
145
+ why = " The baseline can't run the skipped #{(@skipped == 1) ? "benchmark" : "benchmarks"}." if @skipped.positive?
149
146
  "Compared #{@candidate} with #{@baseline}.#{why}"
150
147
  end
151
148
 
@@ -170,12 +167,11 @@ module Duel
170
167
  counted = result.counted[metric]
171
168
  if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
172
169
  elsif counted
173
- n = counted.fetch("n")
174
- "#{metric} #{Runner.number(counted.fetch("baseline").fdiv(n))} → #{Runner.number(counted.fetch("candidate").fdiv(n))}"
170
+ base, candidate = Result.per_iteration(counted)
171
+ count_change(metric, base, candidate)
175
172
  else
176
173
  # (Peak memory, which grew too much.)
177
- memory = result.memory #: Hash[String, Integer]
178
- "#{metric} #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
174
+ memory_change(result)
179
175
  end
180
176
  end
181
177
  worse.join(result.mixed? ? " or " : ", ")
@@ -200,7 +196,7 @@ module Duel
200
196
  # benchmark, the verdicts across the board, the Rubies, and how Duel
201
197
  # decides.
202
198
  def details(explained)
203
- unsettled = @results.count { |r| Summary.verdict(r) == "unsettled" }
199
+ unsettled = @results.count { |r| @verdicts.fetch(r) == "unsettled" }
204
200
  title = format("Details: %s in %.0fs%s", Runner.count(@results.size, "benchmark"), @runner.seconds,
205
201
  (", #{unsettled} unsettled" if unsettled.positive?))
206
202
  body = [*regressions(explained), *errors, table, (UNSETTLED if unsettled.positive?), across(confirmed: false), rubies, decides]
@@ -210,7 +206,7 @@ module Duel
210
206
  # The results, in the order of their VERDICTS, and the run's order in
211
207
  # each (grouped, not sorted: Ruby's sort isn't stable everywhere).
212
208
  def sorted
213
- grouped = @results.group_by { |r| Summary.verdict(r) }
209
+ grouped = @results.group_by { |r| @verdicts.fetch(r) }
214
210
  VERDICTS.flat_map { |shown| grouped.fetch(shown, []) }
215
211
  end
216
212
 
@@ -219,7 +215,7 @@ module Duel
219
215
  return if @results.empty?
220
216
 
221
217
  rows = sorted.map do |r|
222
- shown = Summary.verdict(r)
218
+ shown = @verdicts.fetch(r)
223
219
  [Summary.code(cell(r.name)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r)]
224
220
  end
225
221
  grid(["Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"], rows)
@@ -254,16 +250,21 @@ module Duel
254
250
  # Each counted measure that changed, an iteration, and peak memory
255
251
  # that grew too much.
256
252
  def counts(result)
257
- counted = Summary.changed_counts(result).map do |metric, (base, candidate)|
258
- "#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
259
- end
260
- if result.more_memory?
261
- memory = result.memory #: Hash[String, Integer]
262
- counted << "peak_memory #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
263
- end
253
+ counted = Summary.changed_counts(result).map { |metric, (base, candidate)| count_change(metric, base, candidate) }
254
+ counted << memory_change(result) if result.more_memory?
264
255
  counted.join("<br>")
265
256
  end
266
257
 
258
+ # How a counted measure (+metric+) changed: each version's count, an
259
+ # iteration.
260
+ def count_change(metric, base, candidate) = "#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
261
+
262
+ # How peak memory changed: each version's, in MB.
263
+ def memory_change(result)
264
+ memory = result.memory #: Hash[String, Integer]
265
+ "peak_memory #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
266
+ end
267
+
267
268
  # The first +explained+ regressions, explained as the terminal report
268
269
  # explains them (and the rest named).
269
270
  def regressions(explained)
@@ -295,9 +296,10 @@ module Duel
295
296
  # The errors of more than a line, all of them (up to ERROR_LINES):
296
297
  # their first lines are in view.
297
298
  def errors
298
- @failed.select { |r| r.error.to_s.chomp.include?("\n") }.map do |r|
299
- error = r.error #: String
300
- lines = error.lines
299
+ @failed.filter_map do |r|
300
+ lines = r.error.to_s.lines
301
+ next unless lines.size > 1
302
+
301
303
  cut = "\n(#{Runner.count(lines.size - ERROR_LINES, "more line")} in the job's log)" if lines.size > ERROR_LINES
302
304
  "**#{Summary.code(r.name)}**\n\n#{Summary.block(lines.first(ERROR_LINES).join)}#{cut}"
303
305
  end
@@ -22,8 +22,8 @@
22
22
  # with a `before_sample` hook.
23
23
  # - Aim for microseconds to a few milliseconds an iteration, and split
24
24
  # anything slower.
25
- # - Use `only: :count` for work that's mostly waiting (on a socket, say),
26
- # and `only: :timing` for work on threads of its own.
25
+ # - Use `judge: :counted` for work that's mostly waiting (on a socket,
26
+ # say), and `judge: :timed` for work on threads of its own.
27
27
  # - Use only the API that the main branch has: a benchmark that the
28
28
  # baseline can't run is skipped.
29
29
 
@@ -27,12 +27,10 @@
27
27
  DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
28
28
  rules:
29
29
  - if: $CI_PIPELINE_SOURCE == "merge_request_event"
30
+ # (Duel compares a push with the commit before it, and a schedule's
31
+ # run with the last release, as CI_PIPELINE_SOURCE says.)
30
32
  - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
31
- variables:
32
- DUEL_BASE: $CI_COMMIT_BEFORE_SHA
33
33
  - if: $CI_PIPELINE_SOURCE == "schedule"
34
- variables:
35
- DUEL_BASE: last-release
36
34
  # (Run by hand, it compares anyway: with nothing changed, the code
37
35
  # with itself, which checks the setup.)
38
36
  - if: $CI_PIPELINE_SOURCE == "web"
@@ -77,17 +75,18 @@
77
75
  variables:
78
76
  GIT_STRATEGY: none # it runs none of the code
79
77
  script:
80
- - gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
78
+ - gem install duel --version '<%= Duel::VERSION %>' --no-document
81
79
  # (The merge request's head: in a merged results pipeline, not the
82
80
  # commit that merges it.)
83
- - duel comment --gitlab --report <%= working_directory || "." %> --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker '<%= marker.gsub("'") { "'\\''" } %>'
81
+ - duel comment --report <%= working_directory || "." %> --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker '<%= marker.gsub("'") { "'\\''" } %>'
84
82
 
85
83
  # The report, as a new comment on a pushed commit, when Duel says to, as
86
84
  # for a merge request: only where a regression in a push passes (see
87
85
  # c.fail_on), so the comment says what the check doesn't. Its last comment
88
- # is the commit before's.
86
+ # is the commit before's. It runs `duel comment`, from RubyGems, signed in
87
+ # with DUEL_TOKEN.
89
88
  <%= gitlab_job %>:commit-comment:
90
- image: alpine:3
89
+ image: ruby:slim
91
90
  stage: .post
92
91
  needs: [<%= gitlab_job %>]
93
92
  rules:
@@ -96,31 +95,19 @@
96
95
  variables:
97
96
  GIT_STRATEGY: none # it runs none of the code
98
97
  script:
99
- - apk add --no-cache --quiet curl jq
100
- - |
101
- case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
102
- report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
103
- [ -f "$report" ] || exit 0
104
- api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
105
- token="PRIVATE-TOKEN: $DUEL_TOKEN"
106
- marker="<!-- <%= marker %>"
107
- me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
108
- last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
109
- jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
110
- if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
111
- { echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
112
- curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
113
- fi
98
+ - gem install duel --version '<%= Duel::VERSION %>' --no-document
99
+ - duel comment --report <%= working_directory || "." %> --commit "$CI_COMMIT_SHA" --marker '<%= marker.gsub("'") { "'\\''" } %>'
114
100
 
115
101
  # With c.push_base = :last_passed in .duel.rb, a push is compared with the
116
102
  # last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
117
103
  # main), so slowdowns too small for any one push to fail add up until they
118
- # fail one. This job moves the branch to a push that passed (or makes it),
119
- # but never back: where a newer push's job moved it first, it stays. To
120
- # accept a regression, move the branch yourself (for main, git push
121
- # origin main:<%= passed_branch %>/main).
104
+ # fail one. This job moves the branch (as Duel names it, DUEL_PASSED) to a
105
+ # push that passed (or makes it), but never back: where a newer push's job
106
+ # moved it first, it stays. It runs `duel mark-passed`, from RubyGems,
107
+ # signed in with DUEL_TOKEN. To accept a regression, move the branch
108
+ # yourself (for main, git push origin main:<%= passed_branch %>/main).
122
109
  <%= gitlab_job %>:passed:
123
- image: alpine:3
110
+ image: ruby:slim
124
111
  stage: .post
125
112
  needs: [<%= gitlab_job %>]
126
113
  # One at a time, as each deletes the branch and makes it again: two at
@@ -131,29 +118,6 @@
131
118
  variables:
132
119
  GIT_STRATEGY: none # it runs none of the code
133
120
  script:
134
- - apk add --no-cache --quiet curl jq
135
- - |
136
- [ "$DUEL_PASSED" = true ] || exit 0
137
- api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
138
- token="PRIVATE-TOKEN: $DUEL_TOKEN"
139
- branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
140
- at=$(curl -sf -H "$token" "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')" | jq -r '.commit.id // empty') || at=
141
- if [ -n "$at" ]; then
142
- base=$(curl -sf -H "$token" "$api/repository/merge_base?refs%5B%5D=$CI_COMMIT_SHA&refs%5B%5D=$at" | jq -r '.id // empty') || base=
143
- [ -n "$base" ] || { echo "Couldn't compare $branch with this push" && exit 1; }
144
- # (A newer push's job moved it, or this one's did, run again.)
145
- [ "$base" = "$CI_COMMIT_SHA" ] && exit 0
146
- fi
147
- curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
148
- curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches" && exit 0
149
- # (Git can't make a branch where another's name is a folder of its
150
- # name: a branch named duel blocks duel/passed/main. If one does,
151
- # this names it.)
152
- prefix=$branch
153
- while [ "${prefix%/*}" != "$prefix" ]; do
154
- prefix=${prefix%/*}
155
- if curl -sf -o /dev/null -H "$token" "$api/repository/branches/$(echo "$prefix" | sed 's|/|%2F|g')"; then
156
- echo "The branch $prefix keeps Duel from making $branch, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
157
- fi
158
- done
159
- exit 1
121
+ - '[ -n "$DUEL_PASSED" ] || exit 0'
122
+ - gem install duel --version '<%= Duel::VERSION %>' --no-document
123
+ - duel mark-passed "$DUEL_PASSED"
@@ -17,14 +17,15 @@ Duel.configure do |c|
17
17
  <% else -%>
18
18
  # The library each version's workers require, from its lib/ (there's no
19
19
  # gemspec to name it). Or, to load each version some other way:
20
- # c.load { |root| require File.join(root, "config", "boot") }
20
+ # c.library = ->(root) { require File.join(root, "config", "boot") }
21
21
  # c.library = "my_library"
22
22
  <% end -%>
23
23
  <% if ext -%>
24
24
 
25
25
  # Builds each version's C extension, so that each is measured with its
26
- # own build.
27
- c.build = "bundle exec rake compile"
26
+ # own build: with this, by default, for a project with ext/ and a
27
+ # Rakefile.
28
+ # c.build = "bundle exec rake compile"
28
29
  <% end -%>
29
30
  <% if version_file -%>
30
31
 
@@ -41,17 +42,16 @@ Duel.configure do |c|
41
42
  # Which runs a regression fails: :every_run, or :pull_requests, for a
42
43
  # project where people push to main directly (on GitHub Actions, another
43
44
  # run warns and passes, and a pushed commit gets the comment instead,
44
- # from a workflow job written only where this is set: see docs/ci.md).
45
+ # from a workflow job that runs where this is set: see docs/ci.md).
45
46
  # c.fail_on = :every_run
46
47
 
47
48
  # A label that accepts a regression in a pull request, for a change
48
49
  # that's slower on purpose (nil for none).
49
- c.accept_label = "duel: accept"
50
+ # c.accept_label = "duel: accept"
50
51
 
51
52
  # What the workflow compares a push with: :before (the commit before it),
52
53
  # or :last_passed (the last commit Duel passed, which a branch,
53
54
  # duel/passed/main, marks), so small slowdowns add up until they fail.
54
- # (A workflow job moves the branch, written only where this is set: see
55
- # docs/ci.md.)
55
+ # (A workflow job moves the branch where this is set: see docs/ci.md.)
56
56
  # c.push_base = :before
57
57
  end
@@ -25,15 +25,11 @@ concurrency:
25
25
  permissions: {}
26
26
 
27
27
  jobs:
28
+ # Duel's job, which runs the branch's code, so it only reads.
28
29
  duel:
29
30
  uses: <%= REUSABLE %>/duel.yml@v<%= Duel::VERSION %>
30
- # (Its Duel job, which runs the branch's code, only reads.)
31
31
  permissions:
32
- <% if writes? -%>
33
- contents: write # to mark a push passed, comment on it, or keep the badge
34
- <% else -%>
35
32
  contents: read
36
- <% end -%>
37
33
  pull-requests: read # for the pull request's labels (see c.accept_label)
38
34
  # (And with setup:, shell commands that set up whatever else your
39
35
  # tests need: packages, services.)
@@ -41,3 +37,24 @@ jobs:
41
37
  <% perf_inputs.each do |name, value| -%>
42
38
  <%= name %>: <%= value %>
43
39
  <% end -%>
40
+
41
+ # For a push, the jobs that comment on the commit, mark it passed, or
42
+ # keep the badge, as the Duel job says, which run none of the code.
43
+ # They write, so they're a call of their own, and run only for a push:
44
+ # GitHub checks what each job of a workflow it calls asks for against
45
+ # what's granted, run or not, so the Duel job's call grants reading
46
+ # alone.
47
+ push:
48
+ needs: duel
49
+ if: ${{ !cancelled() && github.event_name == 'push' }}
50
+ uses: <%= REUSABLE %>/duel-push.yml@v<%= Duel::VERSION %>
51
+ permissions:
52
+ contents: write # to comment on the commit, move the passed branch, and the badge's
53
+ with:
54
+ comment: ${{ needs.duel.outputs.comment }}
55
+ status: ${{ needs.duel.outputs.status }}
56
+ passed: ${{ needs.duel.outputs.passed }}
57
+ result: ${{ needs.duel.result }}
58
+ <% push_inputs.each do |name, value| -%>
59
+ <%= name %>: <%= value %>
60
+ <% end -%>
@@ -2,14 +2,14 @@
2
2
  name: Duel comment
3
3
 
4
4
  # Duel's Perf comment workflow, reusable: a project's own Perf comment
5
- # workflow (as `duel init --reusable` writes it) calls it, triggered as
6
- # that workflow is (once the Perf workflow has run, and where there's a
7
- # label that accepts a regression, when it's added or taken off), and
8
- # passes on what .duel.rb sets that it needs as its inputs. Its jobs run
9
- # as the Perf comment workflow's that `duel init` writes do, and the
10
- # caller grants what they need (actions: read, or write to run the Duel
11
- # job again, and pull-requests: write), which each job narrows. See
12
- # Duel's docs/ci.md.
5
+ # workflow (as `duel init` writes it) calls it, triggered as that
6
+ # workflow is (once the Perf workflow has run, and where there's a label
7
+ # that accepts a regression, when it's added or taken off), and passes
8
+ # on what .duel.rb sets that it needs as its inputs. Its jobs run as the
9
+ # whole Perf comment workflow's that `duel init --standalone` writes do,
10
+ # and the caller grants what they need (actions: read, or write to run
11
+ # the Duel job again, and pull-requests: write), which each job narrows.
12
+ # See Duel's docs/ci.md.
13
13
  on:
14
14
  workflow_call:
15
15
  inputs:
@@ -77,7 +77,7 @@ jobs:
77
77
  - name: Get the report
78
78
  id: report
79
79
  continue-on-error: true # there's none when the Duel job failed before comparing
80
- uses: actions/download-artifact@v8
80
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
81
81
  with:
82
82
  <% if reusable? -%>
83
83
  # (Every call's, where the Perf workflow called Duel's more than
@@ -89,7 +89,7 @@ jobs:
89
89
  path: ${{ runner.temp }}/report
90
90
  run-id: ${{ github.event.workflow_run.id }}
91
91
  github-token: ${{ github.token }}
92
- - uses: ruby/setup-ruby@v1
92
+ - uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
93
93
  if: steps.report.outcome == 'success'
94
94
  with:
95
95
  ruby-version: ruby
@@ -104,7 +104,7 @@ jobs:
104
104
  MARKER: ${{ inputs.marker }}
105
105
  <% end -%>
106
106
  run: |
107
- gem install duel --version '~> <%= Duel::Comment::SINCE %>' --no-document
107
+ gem install duel --version '<%= Duel::VERSION %>' --no-document
108
108
  duel comment --report "$RUNNER_TEMP/report" --head "$HEAD_SHA" --marker <%= reusable? ? '"$MARKER"' : "'#{marker.gsub("'") { "'\\''" }}'" %>
109
109
  <% if accept_label -%>
110
110
 
@@ -0,0 +1,127 @@
1
+ # The report, as a new comment on a pushed commit, which notifies its
2
+ # author, when Duel says to, as for a pull request (see the Perf comment
3
+ # workflow). It does only where a regression in a push passes, with
4
+ # c.fail_on = :pull_requests in .duel.rb, so the comment says what the
5
+ # check doesn't. Its last comment is the last one on any commit. This
6
+ # job checks nothing out and runs none of the code.
7
+ commit-comment:
8
+ name: Comment on the commit
9
+ needs: duel
10
+ if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
11
+ runs-on: ubuntu-latest
12
+ permissions:
13
+ contents: write # to comment on the commit with the report
14
+ steps:
15
+ - name: Get the report
16
+ id: report
17
+ continue-on-error: true # there's none when the Duel job failed before comparing
18
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
19
+ with:
20
+ name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
21
+ path: ${{ runner.temp }}/report
22
+ - uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
23
+ if: steps.report.outcome == 'success'
24
+ with:
25
+ ruby-version: ruby
26
+ # Duel's own code, from RubyGems (none of the push's).
27
+ - name: Comment on the commit
28
+ if: steps.report.outcome == 'success'
29
+ env:
30
+ GH_TOKEN: ${{ github.token }}
31
+ DUEL_COMMENT: ${{ needs.duel.outputs.comment }}
32
+ DUEL_STATUS: ${{ needs.duel.outputs.status }}
33
+ <% if reusable? -%>
34
+ # (duel, or for a project in a subdirectory, duel and its path.)
35
+ MARKER: ${{ inputs.working-directory == '.' && 'duel' || format('duel {0}', inputs.working-directory) }}
36
+ <% end -%>
37
+ run: |
38
+ gem install duel --version '<%= Duel::VERSION %>' --no-document
39
+ duel comment --report "$RUNNER_TEMP/report" --commit "$GITHUB_SHA" --marker <%= reusable? ? '"$MARKER"' : "'#{marker.gsub("'") { "'\\''" }}'" %>
40
+
41
+ # With c.push_base = :last_passed in .duel.rb, a push is compared with
42
+ # the last commit Duel passed, which a branch marks (each branch pushed
43
+ # has its own), so slowdowns too small for any one push to fail add up
44
+ # until they fail one. This job moves the branch (as Duel names it) to
45
+ # a push that passed (or makes it), but never back: where a newer
46
+ # push's run moved it first, it stays. It checks nothing out and runs
47
+ # none of the code. To accept a regression, move the branch yourself
48
+ # (for main, git push origin main:<%= passed_branch %>/main).
49
+ passed:
50
+ name: Mark the commit passed
51
+ needs: duel
52
+ # One at a time for each branch, so that two runs can't both find no
53
+ # branch and race to make it (a push's run doesn't wait for the one
54
+ # before it). One that's running is never cancelled, and one that's
55
+ # waiting gives way to a newer push's, which moves it further anyway.
56
+ concurrency:
57
+ group: ${{ github.workflow }}-passed-${{ github.ref_name }}
58
+ cancel-in-progress: false
59
+ if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed != '' }}
60
+ runs-on: ubuntu-latest
61
+ permissions:
62
+ contents: write # to move the branch
63
+ steps:
64
+ - uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
65
+ with:
66
+ ruby-version: ruby
67
+ # Duel's own code, from RubyGems (none of the push's).
68
+ - name: Move the branch to the commit
69
+ env:
70
+ GH_TOKEN: ${{ github.token }}
71
+ BRANCH: ${{ needs.duel.outputs.passed }}
72
+ run: |
73
+ gem install duel --version '<%= Duel::VERSION %>' --no-document
74
+ duel mark-passed "$BRANCH"
75
+ <% if badge? -%>
76
+
77
+ # Duel's badge, for the README: green (perf guarded), red (perf
78
+ # regressed) or grey (not measured), as the last push to the default
79
+ # branch that Duel compared left it. It's red where Duel failed the push
80
+ # or found a regression, even one that passed (with
81
+ # c.fail_on = :pull_requests in .duel.rb), and grey where the Duel job
82
+ # failed before Duel could judge the push (its runner ran out of memory,
83
+ # say). A push with nothing to compare leaves it as it was, and so does
84
+ # a run for an older push than the badge's. This job writes the badge,
85
+ # alone, to a branch of its own, which the README shows:
86
+ #
87
+ # [![Duel](https://raw.githubusercontent.com/OWNER/REPO/<%= badge_branch %>/badge.svg)](https://github.com/OWNER/REPO/actions/workflows/<%= workflow_name %>)
88
+ #
89
+ # It checks nothing out, runs none of the code, and fetches nothing.
90
+ badge:
91
+ name: Update the badge
92
+ needs: duel
93
+ # One at a time, so that one run's check of which push the badge
94
+ # shows can't interleave with another run's write (a push's run
95
+ # doesn't wait for the one before it). A badge job that's running is
96
+ # never cancelled, and one that's waiting gives way to a newer push's,
97
+ # whose result the badge should show anyway.
98
+ concurrency:
99
+ group: ${{ github.workflow }}-badge
100
+ cancel-in-progress: false
101
+ if: ${{ <%= "inputs.badge && " if reusable? %>!cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
102
+ runs-on: ubuntu-latest
103
+ permissions:
104
+ contents: write # to move the badge's branch
105
+ steps:
106
+ - uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
107
+ with:
108
+ ruby-version: ruby
109
+ # Duel's own code, from RubyGems (none of the push's).
110
+ - name: Update the badge
111
+ env:
112
+ GH_TOKEN: ${{ github.token }}
113
+ <% if reusable? -%>
114
+ DIRECTORY: ${{ inputs.working-directory }}
115
+ <% end -%>
116
+ RESULT: ${{ needs.duel.result }}
117
+ STATUS: ${{ needs.duel.outputs.status }}
118
+ run: |
119
+ <% if reusable? -%>
120
+ # (duel/badge, or for a project in a subdirectory, duel/badge-<its
121
+ # path>.)
122
+ branch=duel/badge
123
+ if [ "$DIRECTORY" != . ]; then branch="duel/badge-$(printf %s "$DIRECTORY" | tr / -)"; fi
124
+ <% end -%>
125
+ gem install duel --version '<%= Duel::VERSION %>' --no-document
126
+ duel badge <%= reusable? ? '"$branch"' : badge_branch %> --status "$STATUS" --result "$RESULT"
127
+ <% end -%>
@@ -0,0 +1,50 @@
1
+ name: Duel push
2
+
3
+ # Duel's jobs for a push, reusable: a project's own Perf workflow (as
4
+ # `duel init` writes it) calls it once Duel's Perf workflow (duel.yml)
5
+ # has compared the push, with what its Duel job said (its outputs, and
6
+ # how it ended). They comment on the pushed commit, mark it passed, and
7
+ # keep the badge, as the whole Perf workflow's jobs that
8
+ # `duel init --standalone` writes do. They write to the repository, so
9
+ # they're apart from the Duel job, which runs the branch's code: the
10
+ # caller grants contents: write to this call alone, and only runs it for
11
+ # a push, so a pull request's run is granted nothing more than reading.
12
+ # None of them checks anything out or runs any of the code. See Duel's
13
+ # docs/ci.md.
14
+ on:
15
+ workflow_call:
16
+ inputs:
17
+ comment:
18
+ description: Whether Duel said to comment (post or change), or nothing, as duel.yml's comment output says
19
+ type: string
20
+ default: ""
21
+ status:
22
+ description: The run's status (passed, failed or neutral), or nothing, as duel.yml's status output says
23
+ type: string
24
+ default: ""
25
+ passed:
26
+ description: The branch to mark the push passed on, or nothing, as duel.yml's passed output says
27
+ type: string
28
+ default: ""
29
+ result:
30
+ description: How the call of duel.yml ended (success or failure, say), as needs.<job>.result says
31
+ type: string
32
+ required: true
33
+ working-directory:
34
+ description: The project's directory, from the repository's root, as given to duel.yml
35
+ type: string
36
+ default: .
37
+ report:
38
+ description: The name of the report, as given to duel.yml
39
+ type: string
40
+ default: ""
41
+ badge:
42
+ description: Whether to keep a badge for the README
43
+ type: boolean
44
+ default: false
45
+
46
+ # (Each job asks for what it needs.)
47
+ permissions: {}
48
+
49
+ jobs:
50
+ <%= push_jobs -%>