duel 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AGENT_PROMPT.md +51 -27
- data/CHANGELOG.md +247 -1
- data/README.md +30 -14
- data/docs/benchmarks.md +28 -24
- data/docs/ci.md +328 -362
- data/docs/configuration.md +40 -37
- data/docs/how-it-works.md +23 -10
- data/docs/json.md +21 -25
- data/docs/plugins.md +40 -22
- data/docs/troubleshooting.md +3 -2
- data/lib/duel/across.rb +18 -18
- data/lib/duel/badge.rb +174 -0
- data/lib/duel/baseline.rb +38 -15
- data/lib/duel/cli.rb +175 -107
- data/lib/duel/comment.rb +80 -71
- data/lib/duel/config.rb +145 -63
- data/lib/duel/context.rb +29 -24
- data/lib/duel/forge.rb +87 -0
- data/lib/duel/init.rb +44 -87
- data/lib/duel/json_report.rb +34 -25
- data/lib/duel/lane.rb +107 -61
- data/lib/duel/lock.rb +1 -1
- data/lib/duel/mark_passed.rb +132 -0
- data/lib/duel/metric.rb +28 -27
- data/lib/duel/native.rb +7 -5
- data/lib/duel/platform.rb +4 -0
- data/lib/duel/process_sample.rb +9 -6
- data/lib/duel/rake_task.rb +4 -5
- data/lib/duel/resource.rb +10 -11
- data/lib/duel/result.rb +22 -14
- data/lib/duel/rubies.rb +6 -2
- data/lib/duel/runner.rb +27 -15
- data/lib/duel/spawn.rb +29 -4
- data/lib/duel/stats.rb +14 -12
- data/lib/duel/summary.rb +31 -29
- data/lib/duel/templates/benchmark_duel.rb.erb +2 -2
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +18 -54
- data/lib/duel/templates/duel.rb.erb +7 -7
- data/lib/duel/templates/perf-caller.yml.erb +22 -5
- data/lib/duel/templates/perf-comment.yml.erb +11 -11
- data/lib/duel/templates/perf-push-jobs.yml.erb +127 -0
- data/lib/duel/templates/perf-push.yml.erb +50 -0
- data/lib/duel/templates/perf.yml.erb +113 -335
- data/lib/duel/testing.rb +20 -25
- data/lib/duel/version.rb +1 -1
- data/lib/duel/worker.rb +16 -9
- data/lib/duel/worker_process.rb +6 -6
- data/lib/duel.rb +88 -33
- data/sig/duel/across.rbs +4 -0
- data/sig/duel/badge.rbs +51 -0
- data/sig/duel/baseline.rbs +7 -2
- data/sig/duel/cli.rbs +31 -20
- data/sig/duel/comment.rbs +14 -15
- data/sig/duel/config.rbs +71 -29
- data/sig/duel/context.rbs +4 -4
- data/sig/duel/forge.rbs +42 -0
- data/sig/duel/init.rbs +8 -26
- data/sig/duel/json_report.rbs +7 -6
- data/sig/duel/lane.rbs +29 -6
- data/sig/duel/mark_passed.rbs +39 -0
- data/sig/duel/metric.rbs +10 -10
- data/sig/duel/native.rbs +3 -0
- data/sig/duel/platform.rbs +1 -0
- data/sig/duel/process_sample.rbs +3 -0
- data/sig/duel/resource.rbs +3 -3
- data/sig/duel/result.rbs +12 -5
- data/sig/duel/rubies.rbs +3 -0
- data/sig/duel/runner.rbs +23 -16
- data/sig/duel/spawn.rbs +7 -0
- data/sig/duel/stats.rbs +4 -1
- data/sig/duel/summary.rbs +8 -0
- data/sig/duel/testing.rbs +2 -2
- data/sig/duel/worker.rbs +6 -5
- data/sig/duel/worker_process.rbs +3 -3
- data/sig/duel.rbs +50 -31
- metadata +9 -1
data/lib/duel/stats.rb
CHANGED
|
@@ -93,13 +93,7 @@ module Duel
|
|
|
93
93
|
# [low, high] bounds of the +confidence+ interval for the ratio of
|
|
94
94
|
# +candidates+ (a version's samples) to +bases+ (the other's), or nil
|
|
95
95
|
# (see #shift_rank).
|
|
96
|
-
def shift_interval(bases, candidates, confidence)
|
|
97
|
-
r = shift_rank(bases.size, candidates.size, confidence)
|
|
98
|
-
return unless r
|
|
99
|
-
|
|
100
|
-
ratios = cross_ratios(bases, candidates)
|
|
101
|
-
[ratios.fetch(r - 1), ratios.fetch(-r)]
|
|
102
|
-
end
|
|
96
|
+
def shift_interval(bases, candidates, confidence) = rank_interval(bases, candidates, confidence) { cross_ratios(bases, candidates) }
|
|
103
97
|
|
|
104
98
|
# The ratio of +candidates+ to +bases+ the interval is for: the
|
|
105
99
|
# median of the cross ratios (the Hodges-Lehmann estimate).
|
|
@@ -109,11 +103,19 @@ module Duel
|
|
|
109
103
|
# +candidates+ (a version's samples) are than +bases+ (the other's),
|
|
110
104
|
# as #shift_interval's are for how many times as much, or nil.
|
|
111
105
|
def difference_interval(bases, candidates, confidence)
|
|
106
|
+
rank_interval(bases, candidates, confidence) { candidates.flat_map { |c| bases.map { |b| c - b } }.sort }
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# The bounds #shift_interval and #difference_interval take, of the
|
|
110
|
+
# sorted values the block gives (one for each pair of a candidate's
|
|
111
|
+
# and a base's): the rth least and the rth most (see #shift_rank),
|
|
112
|
+
# or nil, without asking for the values.
|
|
113
|
+
def rank_interval(bases, candidates, confidence)
|
|
112
114
|
r = shift_rank(bases.size, candidates.size, confidence)
|
|
113
115
|
return unless r
|
|
114
116
|
|
|
115
|
-
|
|
116
|
-
[
|
|
117
|
+
sorted = yield
|
|
118
|
+
[sorted.fetch(r - 1), sorted.fetch(-r)]
|
|
117
119
|
end
|
|
118
120
|
|
|
119
121
|
# Each of +candidates+ over each of +bases+, sorted.
|
|
@@ -132,17 +134,17 @@ module Duel
|
|
|
132
134
|
# What a confidence +interval+ for the median ratio (or nil, for too
|
|
133
135
|
# few) says against +limit+: :slower when it lies above the limit,
|
|
134
136
|
# :faster when it lies below 1 / limit, :same when it lies below the
|
|
135
|
-
# limit (it isn't slower: nothing more needs proving), and :
|
|
137
|
+
# limit (it isn't slower: nothing more needs proving), and :unsettled
|
|
136
138
|
# while it straddles the limit.
|
|
137
139
|
def decide(interval, limit)
|
|
138
|
-
return :
|
|
140
|
+
return :unsettled unless interval
|
|
139
141
|
|
|
140
142
|
low, high = interval
|
|
141
143
|
return :slower if low > limit
|
|
142
144
|
return :faster if high < 1 / limit
|
|
143
145
|
return :same if high < limit
|
|
144
146
|
|
|
145
|
-
:
|
|
147
|
+
:unsettled
|
|
146
148
|
end
|
|
147
149
|
end
|
|
148
150
|
end
|
data/lib/duel/summary.rb
CHANGED
|
@@ -83,12 +83,7 @@ module Duel
|
|
|
83
83
|
|
|
84
84
|
# {metric => [baseline, candidate]}, an iteration, of the counted
|
|
85
85
|
# measures that differ.
|
|
86
|
-
def self.changed_counts(result)
|
|
87
|
-
result.counted.to_h do |metric, c|
|
|
88
|
-
n = c.fetch("n")
|
|
89
|
-
[metric, [c.fetch("baseline").fdiv(n), c.fetch("candidate").fdiv(n)]] #: [String, [Float, Float]]
|
|
90
|
-
end.reject { |_, (base, candidate)| base == candidate }
|
|
91
|
-
end
|
|
86
|
+
def self.changed_counts(result) = result.counted.transform_values { |c| Result.per_iteration(c) }.reject { |_, (base, candidate)| base == candidate }
|
|
92
87
|
|
|
93
88
|
# +runner+ ran, and made +results+, comparing +baseline+ with
|
|
94
89
|
# +candidate+ (each a description, in Markdown, whose commit stands
|
|
@@ -103,7 +98,11 @@ module Duel
|
|
|
103
98
|
@settings = settings
|
|
104
99
|
@baseline = baseline
|
|
105
100
|
@candidate = candidate
|
|
106
|
-
|
|
101
|
+
# (Each result's verdict, worked out once: rendering again, to fit
|
|
102
|
+
# LIMIT, asks for them all again.)
|
|
103
|
+
@verdicts = results.to_h { |r| [r, Summary.verdict(r)] }
|
|
104
|
+
@regressed = results.select { |r| @verdicts.fetch(r) == "regressed" }
|
|
105
|
+
@skipped = results.count(&:skipped)
|
|
107
106
|
@failed = @regressed + results.select(&:error)
|
|
108
107
|
end
|
|
109
108
|
|
|
@@ -127,8 +126,7 @@ module Duel
|
|
|
127
126
|
# and whether that was accepted, or that every benchmark compared
|
|
128
127
|
# passed, and those skipped.
|
|
129
128
|
def heading
|
|
130
|
-
|
|
131
|
-
compared = @results.size > skipped
|
|
129
|
+
compared = @results.size > @skipped
|
|
132
130
|
slower = @runner.regressed_across
|
|
133
131
|
said = [("#{Runner.count(@failed.size, "benchmark")} failed" unless @failed.empty?),
|
|
134
132
|
("slower across the board in #{slower.join(", ")}" unless slower.empty?)].compact
|
|
@@ -138,14 +136,13 @@ module Duel
|
|
|
138
136
|
else NEUTRAL_MARK
|
|
139
137
|
end
|
|
140
138
|
said << (compared ? "100% of benchmarks passed" : "no benchmarks compared") if said.empty?
|
|
141
|
-
said << "#{skipped} skipped" if skipped.positive?
|
|
139
|
+
said << "#{@skipped} skipped" if @skipped.positive?
|
|
142
140
|
"### #{mark} Duel: #{said.join(", ")}"
|
|
143
141
|
end
|
|
144
142
|
|
|
145
143
|
# The two versions, and why any benchmark was skipped.
|
|
146
144
|
def compared
|
|
147
|
-
|
|
148
|
-
why = " The baseline can't run the skipped #{(skipped == 1) ? "benchmark" : "benchmarks"}." if skipped.positive?
|
|
145
|
+
why = " The baseline can't run the skipped #{(@skipped == 1) ? "benchmark" : "benchmarks"}." if @skipped.positive?
|
|
149
146
|
"Compared #{@candidate} with #{@baseline}.#{why}"
|
|
150
147
|
end
|
|
151
148
|
|
|
@@ -170,12 +167,11 @@ module Duel
|
|
|
170
167
|
counted = result.counted[metric]
|
|
171
168
|
if timed then format("%s %+.1f%%", metric, (timed.fetch("median") - 1) * 100)
|
|
172
169
|
elsif counted
|
|
173
|
-
|
|
174
|
-
|
|
170
|
+
base, candidate = Result.per_iteration(counted)
|
|
171
|
+
count_change(metric, base, candidate)
|
|
175
172
|
else
|
|
176
173
|
# (Peak memory, which grew too much.)
|
|
177
|
-
|
|
178
|
-
"#{metric} #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
|
|
174
|
+
memory_change(result)
|
|
179
175
|
end
|
|
180
176
|
end
|
|
181
177
|
worse.join(result.mixed? ? " or " : ", ")
|
|
@@ -200,7 +196,7 @@ module Duel
|
|
|
200
196
|
# benchmark, the verdicts across the board, the Rubies, and how Duel
|
|
201
197
|
# decides.
|
|
202
198
|
def details(explained)
|
|
203
|
-
unsettled = @results.count { |r|
|
|
199
|
+
unsettled = @results.count { |r| @verdicts.fetch(r) == "unsettled" }
|
|
204
200
|
title = format("Details: %s in %.0fs%s", Runner.count(@results.size, "benchmark"), @runner.seconds,
|
|
205
201
|
(", #{unsettled} unsettled" if unsettled.positive?))
|
|
206
202
|
body = [*regressions(explained), *errors, table, (UNSETTLED if unsettled.positive?), across(confirmed: false), rubies, decides]
|
|
@@ -210,7 +206,7 @@ module Duel
|
|
|
210
206
|
# The results, in the order of their VERDICTS, and the run's order in
|
|
211
207
|
# each (grouped, not sorted: Ruby's sort isn't stable everywhere).
|
|
212
208
|
def sorted
|
|
213
|
-
grouped = @results.group_by { |r|
|
|
209
|
+
grouped = @results.group_by { |r| @verdicts.fetch(r) }
|
|
214
210
|
VERDICTS.flat_map { |shown| grouped.fetch(shown, []) }
|
|
215
211
|
end
|
|
216
212
|
|
|
@@ -219,7 +215,7 @@ module Duel
|
|
|
219
215
|
return if @results.empty?
|
|
220
216
|
|
|
221
217
|
rows = sorted.map do |r|
|
|
222
|
-
shown =
|
|
218
|
+
shown = @verdicts.fetch(r)
|
|
223
219
|
[Summary.code(cell(r.name)), FAILED.include?(shown) ? "**#{shown}**" : shown, timings(r), counts(r)]
|
|
224
220
|
end
|
|
225
221
|
grid(["Benchmark", "Verdict", "Timings (ratio [interval])", "Counts an iteration"], rows)
|
|
@@ -254,16 +250,21 @@ module Duel
|
|
|
254
250
|
# Each counted measure that changed, an iteration, and peak memory
|
|
255
251
|
# that grew too much.
|
|
256
252
|
def counts(result)
|
|
257
|
-
counted = Summary.changed_counts(result).map
|
|
258
|
-
|
|
259
|
-
end
|
|
260
|
-
if result.more_memory?
|
|
261
|
-
memory = result.memory #: Hash[String, Integer]
|
|
262
|
-
counted << "peak_memory #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
|
|
263
|
-
end
|
|
253
|
+
counted = Summary.changed_counts(result).map { |metric, (base, candidate)| count_change(metric, base, candidate) }
|
|
254
|
+
counted << memory_change(result) if result.more_memory?
|
|
264
255
|
counted.join("<br>")
|
|
265
256
|
end
|
|
266
257
|
|
|
258
|
+
# How a counted measure (+metric+) changed: each version's count, an
|
|
259
|
+
# iteration.
|
|
260
|
+
def count_change(metric, base, candidate) = "#{metric} #{Runner.number(base)} → #{Runner.number(candidate)}"
|
|
261
|
+
|
|
262
|
+
# How peak memory changed: each version's, in MB.
|
|
263
|
+
def memory_change(result)
|
|
264
|
+
memory = result.memory #: Hash[String, Integer]
|
|
265
|
+
"peak_memory #{Runner.mb(memory.fetch("baseline"))} → #{Runner.mb(memory.fetch("candidate"))}"
|
|
266
|
+
end
|
|
267
|
+
|
|
267
268
|
# The first +explained+ regressions, explained as the terminal report
|
|
268
269
|
# explains them (and the rest named).
|
|
269
270
|
def regressions(explained)
|
|
@@ -295,9 +296,10 @@ module Duel
|
|
|
295
296
|
# The errors of more than a line, all of them (up to ERROR_LINES):
|
|
296
297
|
# their first lines are in view.
|
|
297
298
|
def errors
|
|
298
|
-
@failed.
|
|
299
|
-
|
|
300
|
-
lines
|
|
299
|
+
@failed.filter_map do |r|
|
|
300
|
+
lines = r.error.to_s.lines
|
|
301
|
+
next unless lines.size > 1
|
|
302
|
+
|
|
301
303
|
cut = "\n(#{Runner.count(lines.size - ERROR_LINES, "more line")} in the job's log)" if lines.size > ERROR_LINES
|
|
302
304
|
"**#{Summary.code(r.name)}**\n\n#{Summary.block(lines.first(ERROR_LINES).join)}#{cut}"
|
|
303
305
|
end
|
|
@@ -22,8 +22,8 @@
|
|
|
22
22
|
# with a `before_sample` hook.
|
|
23
23
|
# - Aim for microseconds to a few milliseconds an iteration, and split
|
|
24
24
|
# anything slower.
|
|
25
|
-
# - Use `
|
|
26
|
-
# and `
|
|
25
|
+
# - Use `judge: :counted` for work that's mostly waiting (on a socket,
|
|
26
|
+
# say), and `judge: :timed` for work on threads of its own.
|
|
27
27
|
# - Use only the API that the main branch has: a benchmark that the
|
|
28
28
|
# baseline can't run is skipped.
|
|
29
29
|
|
|
@@ -27,12 +27,10 @@
|
|
|
27
27
|
DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
|
|
28
28
|
rules:
|
|
29
29
|
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
30
|
+
# (Duel compares a push with the commit before it, and a schedule's
|
|
31
|
+
# run with the last release, as CI_PIPELINE_SOURCE says.)
|
|
30
32
|
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
31
|
-
variables:
|
|
32
|
-
DUEL_BASE: $CI_COMMIT_BEFORE_SHA
|
|
33
33
|
- if: $CI_PIPELINE_SOURCE == "schedule"
|
|
34
|
-
variables:
|
|
35
|
-
DUEL_BASE: last-release
|
|
36
34
|
# (Run by hand, it compares anyway: with nothing changed, the code
|
|
37
35
|
# with itself, which checks the setup.)
|
|
38
36
|
- if: $CI_PIPELINE_SOURCE == "web"
|
|
@@ -77,17 +75,18 @@
|
|
|
77
75
|
variables:
|
|
78
76
|
GIT_STRATEGY: none # it runs none of the code
|
|
79
77
|
script:
|
|
80
|
-
- gem install duel --version '
|
|
78
|
+
- gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
81
79
|
# (The merge request's head: in a merged results pipeline, not the
|
|
82
80
|
# commit that merges it.)
|
|
83
|
-
- duel comment --
|
|
81
|
+
- duel comment --report <%= working_directory || "." %> --head "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA:-$CI_COMMIT_SHA}" --marker '<%= marker.gsub("'") { "'\\''" } %>'
|
|
84
82
|
|
|
85
83
|
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
86
84
|
# for a merge request: only where a regression in a push passes (see
|
|
87
85
|
# c.fail_on), so the comment says what the check doesn't. Its last comment
|
|
88
|
-
# is the commit before's.
|
|
86
|
+
# is the commit before's. It runs `duel comment`, from RubyGems, signed in
|
|
87
|
+
# with DUEL_TOKEN.
|
|
89
88
|
<%= gitlab_job %>:commit-comment:
|
|
90
|
-
image:
|
|
89
|
+
image: ruby:slim
|
|
91
90
|
stage: .post
|
|
92
91
|
needs: [<%= gitlab_job %>]
|
|
93
92
|
rules:
|
|
@@ -96,31 +95,19 @@
|
|
|
96
95
|
variables:
|
|
97
96
|
GIT_STRATEGY: none # it runs none of the code
|
|
98
97
|
script:
|
|
99
|
-
-
|
|
100
|
-
-
|
|
101
|
-
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
102
|
-
report=<%= working_directory ? "#{working_directory}/" : "" %>duel.md
|
|
103
|
-
[ -f "$report" ] || exit 0
|
|
104
|
-
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
105
|
-
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
106
|
-
marker="<!-- <%= marker %>"
|
|
107
|
-
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
108
|
-
last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
|
|
109
|
-
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
|
|
110
|
-
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
111
|
-
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
|
|
112
|
-
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
|
|
113
|
-
fi
|
|
98
|
+
- gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
99
|
+
- duel comment --report <%= working_directory || "." %> --commit "$CI_COMMIT_SHA" --marker '<%= marker.gsub("'") { "'\\''" } %>'
|
|
114
100
|
|
|
115
101
|
# With c.push_base = :last_passed in .duel.rb, a push is compared with the
|
|
116
102
|
# last commit Duel passed, which a branch marks (<%= passed_branch %>/main for
|
|
117
103
|
# main), so slowdowns too small for any one push to fail add up until they
|
|
118
|
-
# fail one. This job moves the branch
|
|
119
|
-
# but never back: where a newer push's job
|
|
120
|
-
#
|
|
121
|
-
#
|
|
104
|
+
# fail one. This job moves the branch (as Duel names it, DUEL_PASSED) to a
|
|
105
|
+
# push that passed (or makes it), but never back: where a newer push's job
|
|
106
|
+
# moved it first, it stays. It runs `duel mark-passed`, from RubyGems,
|
|
107
|
+
# signed in with DUEL_TOKEN. To accept a regression, move the branch
|
|
108
|
+
# yourself (for main, git push origin main:<%= passed_branch %>/main).
|
|
122
109
|
<%= gitlab_job %>:passed:
|
|
123
|
-
image:
|
|
110
|
+
image: ruby:slim
|
|
124
111
|
stage: .post
|
|
125
112
|
needs: [<%= gitlab_job %>]
|
|
126
113
|
# One at a time, as each deletes the branch and makes it again: two at
|
|
@@ -131,29 +118,6 @@
|
|
|
131
118
|
variables:
|
|
132
119
|
GIT_STRATEGY: none # it runs none of the code
|
|
133
120
|
script:
|
|
134
|
-
-
|
|
135
|
-
-
|
|
136
|
-
|
|
137
|
-
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
138
|
-
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
139
|
-
branch="<%= passed_branch %>/$CI_COMMIT_BRANCH"
|
|
140
|
-
at=$(curl -sf -H "$token" "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')" | jq -r '.commit.id // empty') || at=
|
|
141
|
-
if [ -n "$at" ]; then
|
|
142
|
-
base=$(curl -sf -H "$token" "$api/repository/merge_base?refs%5B%5D=$CI_COMMIT_SHA&refs%5B%5D=$at" | jq -r '.id // empty') || base=
|
|
143
|
-
[ -n "$base" ] || { echo "Couldn't compare $branch with this push" && exit 1; }
|
|
144
|
-
# (A newer push's job moved it, or this one's did, run again.)
|
|
145
|
-
[ "$base" = "$CI_COMMIT_SHA" ] && exit 0
|
|
146
|
-
fi
|
|
147
|
-
curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
|
|
148
|
-
curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches" && exit 0
|
|
149
|
-
# (Git can't make a branch where another's name is a folder of its
|
|
150
|
-
# name: a branch named duel blocks duel/passed/main. If one does,
|
|
151
|
-
# this names it.)
|
|
152
|
-
prefix=$branch
|
|
153
|
-
while [ "${prefix%/*}" != "$prefix" ]; do
|
|
154
|
-
prefix=${prefix%/*}
|
|
155
|
-
if curl -sf -o /dev/null -H "$token" "$api/repository/branches/$(echo "$prefix" | sed 's|/|%2F|g')"; then
|
|
156
|
-
echo "The branch $prefix keeps Duel from making $branch, as Git can't have both. Delete it (git push origin --delete $prefix), or rename it."
|
|
157
|
-
fi
|
|
158
|
-
done
|
|
159
|
-
exit 1
|
|
121
|
+
- '[ -n "$DUEL_PASSED" ] || exit 0'
|
|
122
|
+
- gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
123
|
+
- duel mark-passed "$DUEL_PASSED"
|
|
@@ -17,14 +17,15 @@ Duel.configure do |c|
|
|
|
17
17
|
<% else -%>
|
|
18
18
|
# The library each version's workers require, from its lib/ (there's no
|
|
19
19
|
# gemspec to name it). Or, to load each version some other way:
|
|
20
|
-
# c.
|
|
20
|
+
# c.library = ->(root) { require File.join(root, "config", "boot") }
|
|
21
21
|
# c.library = "my_library"
|
|
22
22
|
<% end -%>
|
|
23
23
|
<% if ext -%>
|
|
24
24
|
|
|
25
25
|
# Builds each version's C extension, so that each is measured with its
|
|
26
|
-
# own build
|
|
27
|
-
|
|
26
|
+
# own build: with this, by default, for a project with ext/ and a
|
|
27
|
+
# Rakefile.
|
|
28
|
+
# c.build = "bundle exec rake compile"
|
|
28
29
|
<% end -%>
|
|
29
30
|
<% if version_file -%>
|
|
30
31
|
|
|
@@ -41,17 +42,16 @@ Duel.configure do |c|
|
|
|
41
42
|
# Which runs a regression fails: :every_run, or :pull_requests, for a
|
|
42
43
|
# project where people push to main directly (on GitHub Actions, another
|
|
43
44
|
# run warns and passes, and a pushed commit gets the comment instead,
|
|
44
|
-
# from a workflow job
|
|
45
|
+
# from a workflow job that runs where this is set: see docs/ci.md).
|
|
45
46
|
# c.fail_on = :every_run
|
|
46
47
|
|
|
47
48
|
# A label that accepts a regression in a pull request, for a change
|
|
48
49
|
# that's slower on purpose (nil for none).
|
|
49
|
-
c.accept_label = "duel: accept"
|
|
50
|
+
# c.accept_label = "duel: accept"
|
|
50
51
|
|
|
51
52
|
# What the workflow compares a push with: :before (the commit before it),
|
|
52
53
|
# or :last_passed (the last commit Duel passed, which a branch,
|
|
53
54
|
# duel/passed/main, marks), so small slowdowns add up until they fail.
|
|
54
|
-
# (A workflow job moves the branch
|
|
55
|
-
# docs/ci.md.)
|
|
55
|
+
# (A workflow job moves the branch where this is set: see docs/ci.md.)
|
|
56
56
|
# c.push_base = :before
|
|
57
57
|
end
|
|
@@ -25,15 +25,11 @@ concurrency:
|
|
|
25
25
|
permissions: {}
|
|
26
26
|
|
|
27
27
|
jobs:
|
|
28
|
+
# Duel's job, which runs the branch's code, so it only reads.
|
|
28
29
|
duel:
|
|
29
30
|
uses: <%= REUSABLE %>/duel.yml@v<%= Duel::VERSION %>
|
|
30
|
-
# (Its Duel job, which runs the branch's code, only reads.)
|
|
31
31
|
permissions:
|
|
32
|
-
<% if writes? -%>
|
|
33
|
-
contents: write # to mark a push passed, comment on it, or keep the badge
|
|
34
|
-
<% else -%>
|
|
35
32
|
contents: read
|
|
36
|
-
<% end -%>
|
|
37
33
|
pull-requests: read # for the pull request's labels (see c.accept_label)
|
|
38
34
|
# (And with setup:, shell commands that set up whatever else your
|
|
39
35
|
# tests need: packages, services.)
|
|
@@ -41,3 +37,24 @@ jobs:
|
|
|
41
37
|
<% perf_inputs.each do |name, value| -%>
|
|
42
38
|
<%= name %>: <%= value %>
|
|
43
39
|
<% end -%>
|
|
40
|
+
|
|
41
|
+
# For a push, the jobs that comment on the commit, mark it passed, or
|
|
42
|
+
# keep the badge, as the Duel job says, which run none of the code.
|
|
43
|
+
# They write, so they're a call of their own, and run only for a push:
|
|
44
|
+
# GitHub checks what each job of a workflow it calls asks for against
|
|
45
|
+
# what's granted, run or not, so the Duel job's call grants reading
|
|
46
|
+
# alone.
|
|
47
|
+
push:
|
|
48
|
+
needs: duel
|
|
49
|
+
if: ${{ !cancelled() && github.event_name == 'push' }}
|
|
50
|
+
uses: <%= REUSABLE %>/duel-push.yml@v<%= Duel::VERSION %>
|
|
51
|
+
permissions:
|
|
52
|
+
contents: write # to comment on the commit, move the passed branch, and the badge's
|
|
53
|
+
with:
|
|
54
|
+
comment: ${{ needs.duel.outputs.comment }}
|
|
55
|
+
status: ${{ needs.duel.outputs.status }}
|
|
56
|
+
passed: ${{ needs.duel.outputs.passed }}
|
|
57
|
+
result: ${{ needs.duel.result }}
|
|
58
|
+
<% push_inputs.each do |name, value| -%>
|
|
59
|
+
<%= name %>: <%= value %>
|
|
60
|
+
<% end -%>
|
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
name: Duel comment
|
|
3
3
|
|
|
4
4
|
# Duel's Perf comment workflow, reusable: a project's own Perf comment
|
|
5
|
-
# workflow (as `duel init
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
# caller grants what they need (actions: read, or write to run
|
|
11
|
-
# job again, and pull-requests: write), which each job narrows.
|
|
12
|
-
# Duel's docs/ci.md.
|
|
5
|
+
# workflow (as `duel init` writes it) calls it, triggered as that
|
|
6
|
+
# workflow is (once the Perf workflow has run, and where there's a label
|
|
7
|
+
# that accepts a regression, when it's added or taken off), and passes
|
|
8
|
+
# on what .duel.rb sets that it needs as its inputs. Its jobs run as the
|
|
9
|
+
# whole Perf comment workflow's that `duel init --standalone` writes do,
|
|
10
|
+
# and the caller grants what they need (actions: read, or write to run
|
|
11
|
+
# the Duel job again, and pull-requests: write), which each job narrows.
|
|
12
|
+
# See Duel's docs/ci.md.
|
|
13
13
|
on:
|
|
14
14
|
workflow_call:
|
|
15
15
|
inputs:
|
|
@@ -77,7 +77,7 @@ jobs:
|
|
|
77
77
|
- name: Get the report
|
|
78
78
|
id: report
|
|
79
79
|
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
80
|
-
uses: actions/download-artifact@v8
|
|
80
|
+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
81
81
|
with:
|
|
82
82
|
<% if reusable? -%>
|
|
83
83
|
# (Every call's, where the Perf workflow called Duel's more than
|
|
@@ -89,7 +89,7 @@ jobs:
|
|
|
89
89
|
path: ${{ runner.temp }}/report
|
|
90
90
|
run-id: ${{ github.event.workflow_run.id }}
|
|
91
91
|
github-token: ${{ github.token }}
|
|
92
|
-
- uses: ruby/setup-ruby@v1
|
|
92
|
+
- uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
|
|
93
93
|
if: steps.report.outcome == 'success'
|
|
94
94
|
with:
|
|
95
95
|
ruby-version: ruby
|
|
@@ -104,7 +104,7 @@ jobs:
|
|
|
104
104
|
MARKER: ${{ inputs.marker }}
|
|
105
105
|
<% end -%>
|
|
106
106
|
run: |
|
|
107
|
-
gem install duel --version '
|
|
107
|
+
gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
108
108
|
duel comment --report "$RUNNER_TEMP/report" --head "$HEAD_SHA" --marker <%= reusable? ? '"$MARKER"' : "'#{marker.gsub("'") { "'\\''" }}'" %>
|
|
109
109
|
<% if accept_label -%>
|
|
110
110
|
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# The report, as a new comment on a pushed commit, which notifies its
|
|
2
|
+
# author, when Duel says to, as for a pull request (see the Perf comment
|
|
3
|
+
# workflow). It does only where a regression in a push passes, with
|
|
4
|
+
# c.fail_on = :pull_requests in .duel.rb, so the comment says what the
|
|
5
|
+
# check doesn't. Its last comment is the last one on any commit. This
|
|
6
|
+
# job checks nothing out and runs none of the code.
|
|
7
|
+
commit-comment:
|
|
8
|
+
name: Comment on the commit
|
|
9
|
+
needs: duel
|
|
10
|
+
if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
permissions:
|
|
13
|
+
contents: write # to comment on the commit with the report
|
|
14
|
+
steps:
|
|
15
|
+
- name: Get the report
|
|
16
|
+
id: report
|
|
17
|
+
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
18
|
+
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
19
|
+
with:
|
|
20
|
+
name: duel-report<%= "${{ inputs.report != '' && format('-{0}', inputs.report) || '' }}" if reusable? %>
|
|
21
|
+
path: ${{ runner.temp }}/report
|
|
22
|
+
- uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
|
|
23
|
+
if: steps.report.outcome == 'success'
|
|
24
|
+
with:
|
|
25
|
+
ruby-version: ruby
|
|
26
|
+
# Duel's own code, from RubyGems (none of the push's).
|
|
27
|
+
- name: Comment on the commit
|
|
28
|
+
if: steps.report.outcome == 'success'
|
|
29
|
+
env:
|
|
30
|
+
GH_TOKEN: ${{ github.token }}
|
|
31
|
+
DUEL_COMMENT: ${{ needs.duel.outputs.comment }}
|
|
32
|
+
DUEL_STATUS: ${{ needs.duel.outputs.status }}
|
|
33
|
+
<% if reusable? -%>
|
|
34
|
+
# (duel, or for a project in a subdirectory, duel and its path.)
|
|
35
|
+
MARKER: ${{ inputs.working-directory == '.' && 'duel' || format('duel {0}', inputs.working-directory) }}
|
|
36
|
+
<% end -%>
|
|
37
|
+
run: |
|
|
38
|
+
gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
39
|
+
duel comment --report "$RUNNER_TEMP/report" --commit "$GITHUB_SHA" --marker <%= reusable? ? '"$MARKER"' : "'#{marker.gsub("'") { "'\\''" }}'" %>
|
|
40
|
+
|
|
41
|
+
# With c.push_base = :last_passed in .duel.rb, a push is compared with
|
|
42
|
+
# the last commit Duel passed, which a branch marks (each branch pushed
|
|
43
|
+
# has its own), so slowdowns too small for any one push to fail add up
|
|
44
|
+
# until they fail one. This job moves the branch (as Duel names it) to
|
|
45
|
+
# a push that passed (or makes it), but never back: where a newer
|
|
46
|
+
# push's run moved it first, it stays. It checks nothing out and runs
|
|
47
|
+
# none of the code. To accept a regression, move the branch yourself
|
|
48
|
+
# (for main, git push origin main:<%= passed_branch %>/main).
|
|
49
|
+
passed:
|
|
50
|
+
name: Mark the commit passed
|
|
51
|
+
needs: duel
|
|
52
|
+
# One at a time for each branch, so that two runs can't both find no
|
|
53
|
+
# branch and race to make it (a push's run doesn't wait for the one
|
|
54
|
+
# before it). One that's running is never cancelled, and one that's
|
|
55
|
+
# waiting gives way to a newer push's, which moves it further anyway.
|
|
56
|
+
concurrency:
|
|
57
|
+
group: ${{ github.workflow }}-passed-${{ github.ref_name }}
|
|
58
|
+
cancel-in-progress: false
|
|
59
|
+
if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed != '' }}
|
|
60
|
+
runs-on: ubuntu-latest
|
|
61
|
+
permissions:
|
|
62
|
+
contents: write # to move the branch
|
|
63
|
+
steps:
|
|
64
|
+
- uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
|
|
65
|
+
with:
|
|
66
|
+
ruby-version: ruby
|
|
67
|
+
# Duel's own code, from RubyGems (none of the push's).
|
|
68
|
+
- name: Move the branch to the commit
|
|
69
|
+
env:
|
|
70
|
+
GH_TOKEN: ${{ github.token }}
|
|
71
|
+
BRANCH: ${{ needs.duel.outputs.passed }}
|
|
72
|
+
run: |
|
|
73
|
+
gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
74
|
+
duel mark-passed "$BRANCH"
|
|
75
|
+
<% if badge? -%>
|
|
76
|
+
|
|
77
|
+
# Duel's badge, for the README: green (perf guarded), red (perf
|
|
78
|
+
# regressed) or grey (not measured), as the last push to the default
|
|
79
|
+
# branch that Duel compared left it. It's red where Duel failed the push
|
|
80
|
+
# or found a regression, even one that passed (with
|
|
81
|
+
# c.fail_on = :pull_requests in .duel.rb), and grey where the Duel job
|
|
82
|
+
# failed before Duel could judge the push (its runner ran out of memory,
|
|
83
|
+
# say). A push with nothing to compare leaves it as it was, and so does
|
|
84
|
+
# a run for an older push than the badge's. This job writes the badge,
|
|
85
|
+
# alone, to a branch of its own, which the README shows:
|
|
86
|
+
#
|
|
87
|
+
# [](https://github.com/OWNER/REPO/actions/workflows/<%= workflow_name %>)
|
|
88
|
+
#
|
|
89
|
+
# It checks nothing out, runs none of the code, and fetches nothing.
|
|
90
|
+
badge:
|
|
91
|
+
name: Update the badge
|
|
92
|
+
needs: duel
|
|
93
|
+
# One at a time, so that one run's check of which push the badge
|
|
94
|
+
# shows can't interleave with another run's write (a push's run
|
|
95
|
+
# doesn't wait for the one before it). A badge job that's running is
|
|
96
|
+
# never cancelled, and one that's waiting gives way to a newer push's,
|
|
97
|
+
# whose result the badge should show anyway.
|
|
98
|
+
concurrency:
|
|
99
|
+
group: ${{ github.workflow }}-badge
|
|
100
|
+
cancel-in-progress: false
|
|
101
|
+
if: ${{ <%= "inputs.badge && " if reusable? %>!cancelled() && github.event_name == 'push' && github.ref_name == github.event.repository.default_branch && (needs.duel.result == 'failure' || needs.duel.outputs.status == 'passed' || needs.duel.outputs.status == 'failed') }}
|
|
102
|
+
runs-on: ubuntu-latest
|
|
103
|
+
permissions:
|
|
104
|
+
contents: write # to move the badge's branch
|
|
105
|
+
steps:
|
|
106
|
+
- uses: ruby/setup-ruby@f9c4492e86def3e644ab32a0ef185eda5b0c20d0 # v1.342.0
|
|
107
|
+
with:
|
|
108
|
+
ruby-version: ruby
|
|
109
|
+
# Duel's own code, from RubyGems (none of the push's).
|
|
110
|
+
- name: Update the badge
|
|
111
|
+
env:
|
|
112
|
+
GH_TOKEN: ${{ github.token }}
|
|
113
|
+
<% if reusable? -%>
|
|
114
|
+
DIRECTORY: ${{ inputs.working-directory }}
|
|
115
|
+
<% end -%>
|
|
116
|
+
RESULT: ${{ needs.duel.result }}
|
|
117
|
+
STATUS: ${{ needs.duel.outputs.status }}
|
|
118
|
+
run: |
|
|
119
|
+
<% if reusable? -%>
|
|
120
|
+
# (duel/badge, or for a project in a subdirectory, duel/badge-<its
|
|
121
|
+
# path>.)
|
|
122
|
+
branch=duel/badge
|
|
123
|
+
if [ "$DIRECTORY" != . ]; then branch="duel/badge-$(printf %s "$DIRECTORY" | tr / -)"; fi
|
|
124
|
+
<% end -%>
|
|
125
|
+
gem install duel --version '<%= Duel::VERSION %>' --no-document
|
|
126
|
+
duel badge <%= reusable? ? '"$branch"' : badge_branch %> --status "$STATUS" --result "$RESULT"
|
|
127
|
+
<% end -%>
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
name: Duel push
|
|
2
|
+
|
|
3
|
+
# Duel's jobs for a push, reusable: a project's own Perf workflow (as
|
|
4
|
+
# `duel init` writes it) calls it once Duel's Perf workflow (duel.yml)
|
|
5
|
+
# has compared the push, with what its Duel job said (its outputs, and
|
|
6
|
+
# how it ended). They comment on the pushed commit, mark it passed, and
|
|
7
|
+
# keep the badge, as the whole Perf workflow's jobs that
|
|
8
|
+
# `duel init --standalone` writes do. They write to the repository, so
|
|
9
|
+
# they're apart from the Duel job, which runs the branch's code: the
|
|
10
|
+
# caller grants contents: write to this call alone, and only runs it for
|
|
11
|
+
# a push, so a pull request's run is granted nothing more than reading.
|
|
12
|
+
# None of them checks anything out or runs any of the code. See Duel's
|
|
13
|
+
# docs/ci.md.
|
|
14
|
+
on:
|
|
15
|
+
workflow_call:
|
|
16
|
+
inputs:
|
|
17
|
+
comment:
|
|
18
|
+
description: Whether Duel said to comment (post or change), or nothing, as duel.yml's comment output says
|
|
19
|
+
type: string
|
|
20
|
+
default: ""
|
|
21
|
+
status:
|
|
22
|
+
description: The run's status (passed, failed or neutral), or nothing, as duel.yml's status output says
|
|
23
|
+
type: string
|
|
24
|
+
default: ""
|
|
25
|
+
passed:
|
|
26
|
+
description: The branch to mark the push passed on, or nothing, as duel.yml's passed output says
|
|
27
|
+
type: string
|
|
28
|
+
default: ""
|
|
29
|
+
result:
|
|
30
|
+
description: How the call of duel.yml ended (success or failure, say), as needs.<job>.result says
|
|
31
|
+
type: string
|
|
32
|
+
required: true
|
|
33
|
+
working-directory:
|
|
34
|
+
description: The project's directory, from the repository's root, as given to duel.yml
|
|
35
|
+
type: string
|
|
36
|
+
default: .
|
|
37
|
+
report:
|
|
38
|
+
description: The name of the report, as given to duel.yml
|
|
39
|
+
type: string
|
|
40
|
+
default: ""
|
|
41
|
+
badge:
|
|
42
|
+
description: Whether to keep a badge for the README
|
|
43
|
+
type: boolean
|
|
44
|
+
default: false
|
|
45
|
+
|
|
46
|
+
# (Each job asks for what it needs.)
|
|
47
|
+
permissions: {}
|
|
48
|
+
|
|
49
|
+
jobs:
|
|
50
|
+
<%= push_jobs -%>
|