duel 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +67 -1
- data/README.md +20 -2
- data/docs/ci.md +495 -61
- data/docs/configuration.md +80 -0
- data/docs/how-it-works.md +4 -1
- data/lib/duel/baseline.rb +30 -4
- data/lib/duel/cli.rb +170 -28
- data/lib/duel/config.rb +71 -0
- data/lib/duel/init.rb +41 -7
- data/lib/duel/runner.rb +1 -1
- data/lib/duel/summary.rb +5 -3
- data/lib/duel/templates/duel.gitlab-ci.yml.erb +148 -0
- data/lib/duel/templates/duel.rb.erb +13 -0
- data/lib/duel/templates/perf-comment.yml.erb +59 -0
- data/lib/duel/templates/perf.yml.erb +116 -46
- data/lib/duel/version.rb +1 -1
- data/sig/duel/baseline.rbs +14 -1
- data/sig/duel/cli.rbs +64 -2
- data/sig/duel/config.rbs +36 -0
- data/sig/duel/init.rbs +13 -0
- data/sig/duel/runner.rbs +15 -3
- data/sig/duel/summary.rbs +3 -1
- metadata +3 -1
data/docs/ci.md
CHANGED
|
@@ -12,9 +12,10 @@ example, a series of changes that are each 1.5% slower would each pass.
|
|
|
12
12
|
To catch that kind of slow creep, also compare against your last release
|
|
13
13
|
on a schedule. The slowdowns add up there until they're a regression.
|
|
14
14
|
|
|
15
|
-
The following GitHub Actions workflow does both
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
The following GitHub Actions workflow does both. `duel init` writes it
|
|
16
|
+
for you, as `.github/workflows/perf.yml`, with a second workflow that
|
|
17
|
+
comments on a pull request when a benchmark fails (see
|
|
18
|
+
[Comment on a pull request](#comment-on-a-pull-request)):
|
|
18
19
|
|
|
19
20
|
```yaml
|
|
20
21
|
# .github/workflows/perf.yml
|
|
@@ -22,26 +23,39 @@ name: Perf
|
|
|
22
23
|
|
|
23
24
|
on:
|
|
24
25
|
pull_request:
|
|
26
|
+
merge_group: # (a merge queue's)
|
|
25
27
|
push:
|
|
26
|
-
branches: [main]
|
|
28
|
+
branches: [main] # (and any other branch to guard, a stable one, say)
|
|
27
29
|
schedule:
|
|
28
30
|
- cron: "0 4 * * *"
|
|
29
31
|
workflow_dispatch:
|
|
30
32
|
|
|
33
|
+
# A pull request's newer push cancels its run, if that's still going, as
|
|
34
|
+
# only its newest commit's result counts. Every other run (a push's, say)
|
|
35
|
+
# runs to the end, so that each push is compared.
|
|
36
|
+
concurrency:
|
|
37
|
+
group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
|
|
38
|
+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
39
|
+
|
|
31
40
|
# The Duel job runs the branch's code (its benchmarks too), so it only
|
|
32
|
-
# reads. Commenting is the Comment job's, which
|
|
41
|
+
# reads. Commenting on a commit is the Comment on the commit job's, which
|
|
42
|
+
# runs none of it, and commenting on a pull request is the Perf comment
|
|
43
|
+
# workflow's.
|
|
33
44
|
permissions:
|
|
34
45
|
contents: read
|
|
46
|
+
pull-requests: read # for the pull request's labels (see c.accept_label)
|
|
35
47
|
|
|
36
48
|
jobs:
|
|
37
49
|
duel:
|
|
38
50
|
name: Duel
|
|
39
51
|
runs-on: ubuntu-latest
|
|
40
|
-
# Whether
|
|
41
|
-
#
|
|
52
|
+
# Whether to comment (post, change or none), the run's status (passed,
|
|
53
|
+
# failed or neutral), and whether to mark a push passed (true, or
|
|
54
|
+
# nothing), from Duel.
|
|
42
55
|
outputs:
|
|
43
56
|
comment: ${{ steps.compare.outputs.comment }}
|
|
44
57
|
status: ${{ steps.compare.outputs.status }}
|
|
58
|
+
passed: ${{ steps.compare.outputs.passed }}
|
|
45
59
|
steps:
|
|
46
60
|
- uses: actions/checkout@v7
|
|
47
61
|
with:
|
|
@@ -52,50 +66,80 @@ jobs:
|
|
|
52
66
|
with:
|
|
53
67
|
ruby-version: "3.4" # (or none, with a .ruby-version)
|
|
54
68
|
bundler-cache: true
|
|
55
|
-
#
|
|
56
|
-
#
|
|
57
|
-
#
|
|
58
|
-
|
|
59
|
-
|
|
69
|
+
# The labels of the pull request (or for a push, of the one that
|
|
70
|
+
# merged it), for c.accept_label in .duel.rb, as a JSON array. They're
|
|
71
|
+
# read as the job runs, so after adding the label, run it again.
|
|
72
|
+
- name: Find the pull request's labels
|
|
73
|
+
id: labels
|
|
74
|
+
if: github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'push'
|
|
75
|
+
continue-on-error: true # (without them, nothing's accepted)
|
|
76
|
+
env:
|
|
77
|
+
GH_TOKEN: ${{ github.token }}
|
|
78
|
+
PR: ${{ github.event.pull_request.number }}
|
|
79
|
+
QUEUED: ${{ github.event.merge_group.head_ref }}
|
|
80
|
+
run: |
|
|
81
|
+
# (In a merge queue, the last pull request queued, which the queue's
|
|
82
|
+
# branch names: gh-readonly-queue/main/pr-123-<sha>.)
|
|
83
|
+
if [ -n "$QUEUED" ]; then
|
|
84
|
+
PR=${QUEUED##*/pr-}
|
|
85
|
+
PR=${PR%%-*}
|
|
86
|
+
fi
|
|
87
|
+
if [ -n "$PR" ]; then
|
|
88
|
+
labels=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR/labels" --jq '[.[].name]')
|
|
89
|
+
else
|
|
90
|
+
labels=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/pulls" --jq '[.[0].labels[]?.name]')
|
|
91
|
+
fi
|
|
92
|
+
echo "labels=$labels" >> "$GITHUB_OUTPUT"
|
|
93
|
+
# Against the branch it targets for a pull request, the queue's base
|
|
94
|
+
# for a merge queue, what was there before for a push (or as
|
|
95
|
+
# c.push_base in .duel.rb says, the last commit Duel passed), and the
|
|
96
|
+
# last release on a schedule (slow creep). Duel adds its report to
|
|
97
|
+
# the job's summary, writes it to duel.md for the comment, and sets
|
|
98
|
+
# the step's outputs to say whether to comment, as c.comment in
|
|
99
|
+
# .duel.rb says, whatever the outcome.
|
|
60
100
|
- name: Compare
|
|
61
101
|
id: compare
|
|
62
102
|
env:
|
|
63
|
-
DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || '' }}
|
|
103
|
+
DUEL_BASE: ${{ github.event_name == 'schedule' && 'last-release' || github.event_name == 'push' && github.event.before || github.event_name == 'merge_group' && github.event.merge_group.base_sha || '' }}
|
|
64
104
|
# (Run by hand, it compares anyway: with nothing changed, the code
|
|
65
105
|
# with itself, which checks the setup.)
|
|
66
106
|
DUEL_FORCE: ${{ github.event_name == 'workflow_dispatch' && '1' || '' }}
|
|
107
|
+
DUEL_LABELS: ${{ steps.labels.outputs.labels }}
|
|
108
|
+
run: bundle exec duel --markdown "$RUNNER_TEMP/duel/duel.md"
|
|
109
|
+
# For a pull request, what Duel said, for the Perf comment workflow,
|
|
110
|
+
# which can't see this job's outputs.
|
|
111
|
+
- name: Say whether to comment
|
|
112
|
+
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
|
|
113
|
+
env:
|
|
114
|
+
PR: ${{ github.event.pull_request.number }}
|
|
115
|
+
COMMENT: ${{ steps.compare.outputs.comment }}
|
|
116
|
+
STATUS: ${{ steps.compare.outputs.status }}
|
|
67
117
|
run: |
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
echo "::notice::No commit before this push to compare against"
|
|
72
|
-
exit 0
|
|
73
|
-
fi
|
|
74
|
-
bundle exec duel --markdown "$RUNNER_TEMP/duel.md"
|
|
75
|
-
# The report, for the Comment job.
|
|
118
|
+
mkdir -p "$RUNNER_TEMP/duel"
|
|
119
|
+
printf 'pr=%s\ncomment=%s\nstatus=%s\n' "$PR" "$COMMENT" "$STATUS" > "$RUNNER_TEMP/duel/comment.env"
|
|
120
|
+
# The report, for the comments.
|
|
76
121
|
- name: Keep the report
|
|
77
|
-
if: ${{ !cancelled() && github.event_name == 'pull_request' }}
|
|
122
|
+
if: ${{ !cancelled() && (github.event_name == 'pull_request' || github.event_name == 'push') }}
|
|
78
123
|
uses: actions/upload-artifact@v7
|
|
79
124
|
with:
|
|
80
125
|
name: duel-report
|
|
81
|
-
path: ${{ runner.temp }}/duel
|
|
126
|
+
path: ${{ runner.temp }}/duel/
|
|
82
127
|
if-no-files-found: ignore # none when Duel didn't get to compare
|
|
83
128
|
retention-days: 1
|
|
84
129
|
|
|
85
|
-
# The report, as a new comment on
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
name: Comment
|
|
130
|
+
# The report, as a new comment on a pushed commit, which notifies its
|
|
131
|
+
# author, when Duel says to, as for a pull request (see the Perf comment
|
|
132
|
+
# workflow). It does only where a regression in a push passes, with
|
|
133
|
+
# c.fail_on = :pull_requests in .duel.rb, so the comment says what the
|
|
134
|
+
# check doesn't. Its last comment is the last one on any commit. This
|
|
135
|
+
# job checks nothing out and runs none of the code.
|
|
136
|
+
commit-comment:
|
|
137
|
+
name: Comment on the commit
|
|
94
138
|
needs: duel
|
|
95
|
-
if: ${{ !cancelled() && github.event_name == '
|
|
139
|
+
if: ${{ !cancelled() && github.event_name == 'push' && (needs.duel.outputs.comment == 'post' || needs.duel.outputs.comment == 'change') }}
|
|
96
140
|
runs-on: ubuntu-latest
|
|
97
141
|
permissions:
|
|
98
|
-
|
|
142
|
+
contents: write # to comment on the commit with the report
|
|
99
143
|
steps:
|
|
100
144
|
- name: Get the report
|
|
101
145
|
id: report
|
|
@@ -104,31 +148,66 @@ jobs:
|
|
|
104
148
|
with:
|
|
105
149
|
name: duel-report
|
|
106
150
|
path: ${{ runner.temp }}/report
|
|
107
|
-
- name: Comment on the
|
|
151
|
+
- name: Comment on the commit
|
|
108
152
|
if: steps.report.outcome == 'success'
|
|
109
153
|
env:
|
|
110
154
|
GH_TOKEN: ${{ github.token }}
|
|
111
|
-
PR: ${{ github.event.pull_request.number }}
|
|
112
155
|
COMMENT: ${{ needs.duel.outputs.comment }}
|
|
113
156
|
STATUS: ${{ needs.duel.outputs.status }}
|
|
114
157
|
run: |
|
|
115
158
|
marker="<!-- duel"
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
119
|
-
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
159
|
+
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/comments" \
|
|
160
|
+
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\"))" | tail -n 1)
|
|
120
161
|
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
121
162
|
{ echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
|
|
122
|
-
gh api --method POST "repos/$GITHUB_REPOSITORY/
|
|
163
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
123
164
|
fi
|
|
165
|
+
|
|
166
|
+
# With c.push_base = :last_passed in .duel.rb, a push is compared with
|
|
167
|
+
# the last commit Duel passed, which a branch marks (each branch pushed
|
|
168
|
+
# has its own), so slowdowns too small for any one push to fail add up
|
|
169
|
+
# until they fail one. This job moves the branch to a push that passed
|
|
170
|
+
# (or makes it). It checks nothing out and runs none of the code. To
|
|
171
|
+
# accept a regression, move the branch yourself (for main, git push
|
|
172
|
+
# origin main:duel/passed/main).
|
|
173
|
+
passed:
|
|
174
|
+
name: Mark the commit passed
|
|
175
|
+
needs: duel
|
|
176
|
+
if: ${{ !cancelled() && github.event_name == 'push' && needs.duel.outputs.passed == 'true' }}
|
|
177
|
+
runs-on: ubuntu-latest
|
|
178
|
+
permissions:
|
|
179
|
+
contents: write # to move the branch
|
|
180
|
+
steps:
|
|
181
|
+
- name: Move the branch to the commit
|
|
182
|
+
env:
|
|
183
|
+
GH_TOKEN: ${{ github.token }}
|
|
184
|
+
BRANCH: duel/passed/${{ github.ref_name }}
|
|
185
|
+
run: |
|
|
186
|
+
gh api --method PATCH "repos/$GITHUB_REPOSITORY/git/refs/heads/$BRANCH" -f sha="$GITHUB_SHA" -F force=true > /dev/null 2>&1 ||
|
|
187
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/git/refs" -f ref="refs/heads/$BRANCH" -f sha="$GITHUB_SHA" > /dev/null
|
|
124
188
|
```
|
|
125
189
|
|
|
190
|
+
A newer push to a pull request cancels the pull request's run, if it's
|
|
191
|
+
still going, because only the newest commit's result counts. Every other
|
|
192
|
+
run, such as a push's to `main`, runs to the end, so each push is
|
|
193
|
+
compared.
|
|
194
|
+
|
|
126
195
|
The workflow compares the following:
|
|
127
196
|
|
|
128
|
-
- A pull request, against the main
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
197
|
+
- A pull request, against the branch that it targets: usually `main`,
|
|
198
|
+
but a stable branch, or another pull request's branch in a stack of
|
|
199
|
+
them, too.
|
|
200
|
+
- A merge queue's group, against the queue's base, the commit that the
|
|
201
|
+
group merges into. With a merge queue, a required Duel check must run
|
|
202
|
+
on `merge_group` too, or the queue waits for it forever. A regression
|
|
203
|
+
fails a merge queue's run as it fails a pull request's, and the run
|
|
204
|
+
reads the labels of the last pull request queued, for
|
|
205
|
+
[`c.accept_label`](#accept-a-regression-with-a-label).
|
|
206
|
+
- A push to `main`, against the commit before the push, or with
|
|
207
|
+
[`c.push_base = :last_passed`](#compare-a-push-with-the-last-commit-that-passed),
|
|
208
|
+
the last commit that Duel passed. If that commit isn't in the history,
|
|
209
|
+
such as after a force push or for a branch's first push, Duel reports
|
|
210
|
+
that there's nothing to compare against, and passes.
|
|
132
211
|
- On a schedule, the last release. Until you tag your first release,
|
|
133
212
|
Duel reports that there's nothing to compare against, and passes.
|
|
134
213
|
- When you run the workflow manually, your code against itself. This
|
|
@@ -175,20 +254,368 @@ GitHub Actions, the Compare step's `duel` sets two step outputs, which
|
|
|
175
254
|
the Duel job passes on as its own: `comment`, and `status` (`passed`,
|
|
176
255
|
`failed`, or `neutral`). `comment` is one of the following:
|
|
177
256
|
|
|
178
|
-
| `comment` | What the
|
|
257
|
+
| `comment` | What the Perf comment workflow does |
|
|
179
258
|
|---|---|
|
|
180
259
|
| `post` | Posts a new comment. |
|
|
181
260
|
| `change` | Posts a new comment only if the pull request has one from the workflow, and the last one's status isn't this run's. |
|
|
182
|
-
| `none` | Nothing
|
|
261
|
+
| `none` | Nothing. |
|
|
183
262
|
|
|
184
263
|
The Duel job runs the pull request's code, its benchmarks included, so
|
|
185
|
-
it only reads (`contents: read`)
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
264
|
+
it only reads (`contents: read`). It keeps the report as an artifact
|
|
265
|
+
for a day, with a file, `comment.env`, that says what Duel decided:
|
|
266
|
+
the pull request's number, `comment`, and `status`. A second workflow,
|
|
267
|
+
`.github/workflows/perf-comment.yml`, posts the report. It runs when
|
|
268
|
+
the Perf workflow completes (`workflow_run`), with your repository's
|
|
269
|
+
own copy of the workflow and a token that can write, so a pull request
|
|
270
|
+
from a fork gets a comment too. It checks nothing out and runs none of
|
|
271
|
+
the pull request's code. The pull request's code ran in the Duel job,
|
|
272
|
+
and could have written anything into the artifact, so the workflow
|
|
273
|
+
trusts only what Duel could have said there: a number that names the
|
|
274
|
+
pull request whose head the run compared, and a `comment` and a
|
|
275
|
+
`status` that Duel uses. The report's Markdown is the comment's body,
|
|
276
|
+
which can only say something on the pull request that the pull request
|
|
277
|
+
could say itself. When the Duel job fails before it compares, there's
|
|
278
|
+
no report, and no comment. GitHub runs a `workflow_run` workflow only
|
|
279
|
+
from your default branch, so the pull request that adds the workflow
|
|
280
|
+
gets no comment.
|
|
281
|
+
|
|
282
|
+
```yaml
|
|
283
|
+
# .github/workflows/perf-comment.yml
|
|
284
|
+
name: Perf comment
|
|
285
|
+
|
|
286
|
+
# Duel's report, as a new comment on the pull request, when Duel says to:
|
|
287
|
+
# always (post), or only when its last comment's status differs from the
|
|
288
|
+
# run's (change), so the latest comment is never stale. It never edits or
|
|
289
|
+
# deletes a comment. It runs once the Perf workflow has, with this
|
|
290
|
+
# repository's own workflow and a token that can write, so a pull request
|
|
291
|
+
# from a fork gets a comment too. It checks nothing out and runs none of
|
|
292
|
+
# the pull request's code, and of what the Duel job kept (where that code
|
|
293
|
+
# ran), it trusts only what that job could have said: the pull request's
|
|
294
|
+
# number, if it names the pull request the run compared, and the comment
|
|
295
|
+
# and the status, if they're ones Duel says. (The workflow must be on the
|
|
296
|
+
# default branch before it runs at all.)
|
|
297
|
+
on:
|
|
298
|
+
workflow_run:
|
|
299
|
+
workflows: ["Perf"]
|
|
300
|
+
types: [completed]
|
|
301
|
+
|
|
302
|
+
permissions: {}
|
|
303
|
+
|
|
304
|
+
jobs:
|
|
305
|
+
comment:
|
|
306
|
+
name: Comment
|
|
307
|
+
if: ${{ github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion != 'cancelled' }}
|
|
308
|
+
runs-on: ubuntu-latest
|
|
309
|
+
permissions:
|
|
310
|
+
actions: read # to get the report from the Perf run
|
|
311
|
+
pull-requests: write # to comment on the pull request with the report
|
|
312
|
+
steps:
|
|
313
|
+
- name: Get the report
|
|
314
|
+
id: report
|
|
315
|
+
continue-on-error: true # there's none when the Duel job failed before comparing
|
|
316
|
+
uses: actions/download-artifact@v8
|
|
317
|
+
with:
|
|
318
|
+
name: duel-report
|
|
319
|
+
path: ${{ runner.temp }}/report
|
|
320
|
+
run-id: ${{ github.event.workflow_run.id }}
|
|
321
|
+
github-token: ${{ github.token }}
|
|
322
|
+
- name: Comment on the pull request
|
|
323
|
+
if: steps.report.outcome == 'success'
|
|
324
|
+
env:
|
|
325
|
+
GH_TOKEN: ${{ github.token }}
|
|
326
|
+
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
327
|
+
run: |
|
|
328
|
+
said() { grep -xE "$1=($2)" "$RUNNER_TEMP/report/comment.env" | tail -n 1 | cut -d= -f2; }
|
|
329
|
+
PR=$(said pr '[0-9]+')
|
|
330
|
+
COMMENT=$(said comment 'post|change')
|
|
331
|
+
STATUS=$(said status 'passed|failed|neutral')
|
|
332
|
+
[ -n "$PR" ] && [ -n "$COMMENT" ] && [ -n "$STATUS" ] || exit 0
|
|
333
|
+
[ "$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq .head.sha)" = "$HEAD_SHA" ] || exit 0
|
|
334
|
+
marker="<!-- duel"
|
|
335
|
+
# The first line of the last comment Duel posted here: its marker,
|
|
336
|
+
# with its status (none in Duel 0.1.0's, which counts as another).
|
|
337
|
+
last=$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues/$PR/comments" \
|
|
338
|
+
--jq ".[] | select(.user.login == \"github-actions[bot]\") | .body | split(\"\n\")[0] | rtrimstr(\"\r\") | select(startswith(\"$marker status=\") or . == \"$marker -->\")" | tail -n 1)
|
|
339
|
+
if [ "$COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$STATUS -->" ]; }; then
|
|
340
|
+
{ echo "$marker status=$STATUS -->"; cat "$RUNNER_TEMP/report/duel.md"; } > "$RUNNER_TEMP/comment.md"
|
|
341
|
+
gh api --method POST "repos/$GITHUB_REPOSITORY/issues/$PR/comments" -F body=@"$RUNNER_TEMP/comment.md" > /dev/null
|
|
342
|
+
fi
|
|
343
|
+
```
|
|
344
|
+
|
|
345
|
+
## Comment on a pushed commit
|
|
346
|
+
|
|
347
|
+
With [`c.fail_on = :pull_requests`](configuration.md#fail-only-a-pull-request),
|
|
348
|
+
a regression in a push to `main` passes, and the workflow's Comment on
|
|
349
|
+
the commit job comments on the pushed commit with the report instead.
|
|
350
|
+
GitHub notifies the commit's author. It comments as `c.comment` says,
|
|
351
|
+
the way the Comment job does on a pull request, but its last comment is
|
|
352
|
+
the workflow's last one on any commit. So with the default, each push
|
|
353
|
+
that fails gets a comment, the first push after them that passes gets
|
|
354
|
+
one, and the passing pushes after it get none.
|
|
355
|
+
|
|
356
|
+
By default, a push gets no comment: Duel sets the `comment` output to
|
|
357
|
+
`none`, and a regression fails the push's check instead. The Comment on
|
|
358
|
+
the commit job checks nothing out either, and it alone gets the
|
|
359
|
+
`contents: write` permission that a commit comment takes.
|
|
360
|
+
|
|
361
|
+
## Accept a regression with a label
|
|
362
|
+
|
|
363
|
+
With [`c.accept_label`](configuration.md#accept-a-regression-with-a-label),
|
|
364
|
+
a pull request with that label passes a regression. The Duel job's Find
|
|
365
|
+
the pull request's labels step reads the labels of the pull request,
|
|
366
|
+
or for a push, of the pull request that merged it, and passes them to
|
|
367
|
+
`duel` as a JSON array, in `DUEL_LABELS`. That takes the
|
|
368
|
+
`pull-requests: read` permission. Without the labels, such as when
|
|
369
|
+
GitHub's API fails, nothing is accepted.
|
|
370
|
+
|
|
371
|
+
## Compare a push with the last commit that passed
|
|
372
|
+
|
|
373
|
+
By default, the workflow compares each push with the commit before it.
|
|
374
|
+
A series of pushes that are each a little slower each pass, and a push
|
|
375
|
+
that the workflow doesn't compare, such as one whose run was cancelled,
|
|
376
|
+
is never compared. With
|
|
377
|
+
[`c.push_base = :last_passed`](configuration.md#compare-a-push-with-the-last-commit-that-passed),
|
|
378
|
+
the workflow compares each push with the last commit that Duel passed,
|
|
379
|
+
so the slowdowns add up until they fail a push.
|
|
380
|
+
|
|
381
|
+
A branch, `duel/passed/main` for `main`, marks that commit. Each
|
|
382
|
+
branch that the workflow guards has its own (see
|
|
383
|
+
[Guard other branches](#guard-other-branches)). When a push passes, or has
|
|
384
|
+
nothing to compare, Duel sets the `passed` output to `true`, and the
|
|
385
|
+
Mark the commit passed job moves the branch to the push, or creates the
|
|
386
|
+
branch. That job checks nothing out either, and it gets the
|
|
387
|
+
`contents: write` permission that moving a branch takes. Until the
|
|
388
|
+
branch exists, and when the push's history doesn't have its commit,
|
|
389
|
+
Duel compares the push with the commit before it. For a project in a
|
|
390
|
+
subdirectory of its repository, the branch names the directory, such as
|
|
391
|
+
`duel/passed-gems-parser/main`.
|
|
392
|
+
|
|
393
|
+
After a regression, the branch stays where it is, so every push fails
|
|
394
|
+
(or with [`c.fail_on = :pull_requests`](#comment-on-a-pushed-commit),
|
|
395
|
+
is reported) until one is no slower. To accept a regression, move the
|
|
396
|
+
branch yourself:
|
|
397
|
+
|
|
398
|
+
```sh
|
|
399
|
+
git push origin main:duel/passed/main
|
|
400
|
+
```
|
|
401
|
+
|
|
402
|
+
If you set `c.push_base` back to `:before`, delete the branches too
|
|
403
|
+
(`git push origin --delete duel/passed/main`), because the step that
|
|
404
|
+
finds the baseline's Ruby uses one whenever it exists.
|
|
405
|
+
|
|
406
|
+
## Guard other branches
|
|
407
|
+
|
|
408
|
+
The workflow compares the pushes to `main`. To guard another branch,
|
|
409
|
+
such as a stable branch that gets backports, add it to the workflow's
|
|
410
|
+
`push` branches:
|
|
411
|
+
|
|
412
|
+
```yaml
|
|
413
|
+
push:
|
|
414
|
+
branches: [main, 2-x-stable]
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
Duel compares a push to each branch with the commit before it, or with
|
|
418
|
+
`c.push_base = :last_passed`, with the last commit that Duel passed on
|
|
419
|
+
that branch, which a branch of its own marks: `duel/passed/2-x-stable`.
|
|
420
|
+
A pull request into the branch is compared with the branch, whatever
|
|
421
|
+
the workflow guards.
|
|
422
|
+
|
|
423
|
+
## Run Duel on GitLab CI
|
|
424
|
+
|
|
425
|
+
`duel init --gitlab` writes GitLab CI jobs, in
|
|
426
|
+
`.gitlab/duel.gitlab-ci.yml`, instead of the GitHub Actions workflows.
|
|
427
|
+
Include them from your `.gitlab-ci.yml`:
|
|
428
|
+
|
|
429
|
+
```yaml
|
|
430
|
+
include:
|
|
431
|
+
- local: .gitlab/duel.gitlab-ci.yml
|
|
432
|
+
```
|
|
433
|
+
|
|
434
|
+
The Duel job compares a merge request with the branch that it targets,
|
|
435
|
+
a push to your default branch with the commit before it (or with
|
|
436
|
+
[`c.push_base = :last_passed`](#compare-a-push-with-the-last-commit-that-passed),
|
|
437
|
+
the last commit that Duel passed), and on a schedule, your last
|
|
438
|
+
release. It runs on the `ruby` image of the version that your
|
|
439
|
+
`.ruby-version` names, or else of your Ruby. On GitLab, a branch that
|
|
440
|
+
upgrades Ruby can't compare each version on its own Ruby, because the
|
|
441
|
+
image has only one. Duel reads GitLab's variables: a merge request's
|
|
442
|
+
target, and its labels, for
|
|
443
|
+
[`c.accept_label`](configuration.md#accept-a-regression-with-a-label).
|
|
444
|
+
A push's labels aren't available.
|
|
445
|
+
|
|
446
|
+
With [`c.fail_on = :pull_requests`](configuration.md#fail-only-a-pull-request),
|
|
447
|
+
a regression outside a merge request makes `duel` exit with status 3,
|
|
448
|
+
and the job lets that fail (`allow_failure: exit_codes: [3]`), so the
|
|
449
|
+
pipeline shows a warning, not a failure.
|
|
450
|
+
|
|
451
|
+
The Duel job writes what it decided to a dotenv report (`DUEL_OUTPUT`):
|
|
452
|
+
`DUEL_COMMENT`, `DUEL_STATUS`, and `DUEL_PASSED`. The jobs after it act
|
|
453
|
+
on them. They check nothing out and run none of the code. GitLab's own
|
|
454
|
+
job token can't comment, so they need project access tokens, in masked
|
|
455
|
+
CI/CD variables. Without them, the jobs don't run, and the Duel job's
|
|
456
|
+
log and its `duel.md` artifact have the report.
|
|
457
|
+
|
|
458
|
+
- `DUEL_COMMENT_TOKEN`, with the Reporter role and the `api` scope, and
|
|
459
|
+
not protected, so a merge request's pipeline has it, to comment on a
|
|
460
|
+
merge request. Every job in that pipeline has it, the Duel job too,
|
|
461
|
+
which runs the merge request's code, so give it no more than the
|
|
462
|
+
Reporter role.
|
|
463
|
+
- `DUEL_TOKEN`, with the Developer role and the `api` scope, and
|
|
464
|
+
protected, so only your protected branches' pipelines have it, to
|
|
465
|
+
comment on a pushed commit and to mark a push passed.
|
|
466
|
+
|
|
467
|
+
The comment on a pushed commit looks only at the comments on the commit
|
|
468
|
+
before it for its last status. A merge request from a fork runs its
|
|
469
|
+
pipeline in the fork, without your variables, so it gets no comment.
|
|
470
|
+
|
|
471
|
+
```yaml
|
|
472
|
+
# .gitlab/duel.gitlab-ci.yml
|
|
473
|
+
# Duel on GitLab CI: https://github.com/sferik/duel-ruby/blob/main/docs/ci.md#run-duel-on-gitlab-ci
|
|
474
|
+
#
|
|
475
|
+
# Include it from .gitlab-ci.yml:
|
|
476
|
+
#
|
|
477
|
+
# include:
|
|
478
|
+
# - local: .gitlab/duel.gitlab-ci.yml
|
|
479
|
+
#
|
|
480
|
+
# It compares a merge request with the branch it targets, a push to the
|
|
481
|
+
# default branch with the commit before it (or as c.push_base in .duel.rb
|
|
482
|
+
# says, the last commit Duel passed), and on a schedule, the last release.
|
|
483
|
+
# The Duel job's log, and its duel.md artifact, have the report. Comments
|
|
484
|
+
# need project access tokens, in masked CI/CD variables:
|
|
485
|
+
#
|
|
486
|
+
# - DUEL_COMMENT_TOKEN (the Reporter role, with the api scope, and not
|
|
487
|
+
# protected), to comment on a merge request. Every job of a merge
|
|
488
|
+
# request's pipeline has it, the Duel job too, which runs the merge
|
|
489
|
+
# request's code, so the token can do no more than comment.
|
|
490
|
+
# - DUEL_TOKEN (the Developer role, with the api scope, and protected, so
|
|
491
|
+
# only the default branch's pipelines have it), to comment on a pushed
|
|
492
|
+
# commit (see c.fail_on), and to mark a push passed (see c.push_base).
|
|
493
|
+
|
|
494
|
+
duel:
|
|
495
|
+
image: ruby:3.4
|
|
496
|
+
stage: test
|
|
497
|
+
variables:
|
|
498
|
+
GIT_DEPTH: 0 # the baseline comes from history, tags included
|
|
499
|
+
DUEL_OUTPUT: duel.env # whether to comment, and mark a push passed
|
|
500
|
+
rules:
|
|
501
|
+
- if: $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
502
|
+
- if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
503
|
+
variables:
|
|
504
|
+
DUEL_BASE: $CI_COMMIT_BEFORE_SHA
|
|
505
|
+
- if: $CI_PIPELINE_SOURCE == "schedule"
|
|
506
|
+
variables:
|
|
507
|
+
DUEL_BASE: last-release
|
|
508
|
+
# (Run by hand, it compares anyway: with nothing changed, the code
|
|
509
|
+
# with itself, which checks the setup.)
|
|
510
|
+
- if: $CI_PIPELINE_SOURCE == "web"
|
|
511
|
+
variables:
|
|
512
|
+
DUEL_FORCE: "1"
|
|
513
|
+
# A regression outside a merge request, where only a merge request
|
|
514
|
+
# fails (c.fail_on in .duel.rb), exits 3: a warning.
|
|
515
|
+
allow_failure:
|
|
516
|
+
exit_codes: [3]
|
|
517
|
+
before_script:
|
|
518
|
+
# The branch a merge request targets, the last commits Duel passed,
|
|
519
|
+
# and the releases.
|
|
520
|
+
- git fetch --quiet --tags origin "+refs/heads/duel/*:refs/remotes/origin/duel/*"
|
|
521
|
+
- if [ -n "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" ]; then git fetch --quiet origin "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"; fi
|
|
522
|
+
- bundle install
|
|
523
|
+
script:
|
|
524
|
+
- bundle exec duel --markdown duel.md
|
|
525
|
+
artifacts:
|
|
526
|
+
when: always
|
|
527
|
+
paths:
|
|
528
|
+
- duel.md
|
|
529
|
+
reports:
|
|
530
|
+
dotenv: duel.env
|
|
531
|
+
expire_in: 1 day
|
|
532
|
+
|
|
533
|
+
# The report, as a new comment on the merge request, when Duel says to:
|
|
534
|
+
# always (post), or only when its last comment's status differs from this
|
|
535
|
+
# run's (change), so the latest comment is never stale. It never edits or
|
|
536
|
+
# deletes a comment.
|
|
537
|
+
duel:comment:
|
|
538
|
+
image: alpine:3
|
|
539
|
+
stage: .post
|
|
540
|
+
needs: [duel]
|
|
541
|
+
rules:
|
|
542
|
+
- if: $DUEL_COMMENT_TOKEN && $CI_PIPELINE_SOURCE == "merge_request_event"
|
|
543
|
+
when: always
|
|
544
|
+
variables:
|
|
545
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
546
|
+
script:
|
|
547
|
+
- apk add --no-cache --quiet curl jq
|
|
548
|
+
- |
|
|
549
|
+
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
550
|
+
report=duel.md
|
|
551
|
+
[ -f "$report" ] || exit 0
|
|
552
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
553
|
+
token="PRIVATE-TOKEN: $DUEL_COMMENT_TOKEN"
|
|
554
|
+
marker="<!-- duel"
|
|
555
|
+
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
556
|
+
# The first line of the last comment Duel posted here: its marker,
|
|
557
|
+
# with its status.
|
|
558
|
+
last=$(curl -sf -H "$token" "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes?sort=desc&order_by=created_at&per_page=100" |
|
|
559
|
+
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .body | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | head -n 1)
|
|
560
|
+
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
561
|
+
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{body: .}' |
|
|
562
|
+
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/merge_requests/$CI_MERGE_REQUEST_IID/notes" > /dev/null
|
|
563
|
+
fi
|
|
564
|
+
|
|
565
|
+
# The report, as a new comment on a pushed commit, when Duel says to, as
|
|
566
|
+
# for a merge request: only where a regression in a push passes (see
|
|
567
|
+
# c.fail_on), so the comment says what the check doesn't. Its last comment
|
|
568
|
+
# is the commit before's.
|
|
569
|
+
duel:commit-comment:
|
|
570
|
+
image: alpine:3
|
|
571
|
+
stage: .post
|
|
572
|
+
needs: [duel]
|
|
573
|
+
rules:
|
|
574
|
+
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
575
|
+
when: always
|
|
576
|
+
variables:
|
|
577
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
578
|
+
script:
|
|
579
|
+
- apk add --no-cache --quiet curl jq
|
|
580
|
+
- |
|
|
581
|
+
case "$DUEL_COMMENT" in post|change) ;; *) exit 0 ;; esac
|
|
582
|
+
report=duel.md
|
|
583
|
+
[ -f "$report" ] || exit 0
|
|
584
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
585
|
+
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
586
|
+
marker="<!-- duel"
|
|
587
|
+
me=$(curl -sf -H "$token" "$CI_API_V4_URL/user" | jq .id)
|
|
588
|
+
last=$(curl -sf -H "$token" "$api/repository/commits/$CI_COMMIT_BEFORE_SHA/comments?per_page=100" |
|
|
589
|
+
jq -r --argjson me "$me" --arg marker "$marker status=" '.[] | select(.author.id == $me) | .note | split("\n")[0] | rtrimstr("\r") | select(startswith($marker))' | tail -n 1)
|
|
590
|
+
if [ "$DUEL_COMMENT" = post ] || { [ -n "$last" ] && [ "$last" != "$marker status=$DUEL_STATUS -->" ]; }; then
|
|
591
|
+
{ echo "$marker status=$DUEL_STATUS -->"; cat "$report"; } | jq -Rs '{note: .}' |
|
|
592
|
+
curl -sf -H "$token" -H "Content-Type: application/json" --data @- "$api/repository/commits/$CI_COMMIT_SHA/comments" > /dev/null
|
|
593
|
+
fi
|
|
594
|
+
|
|
595
|
+
# With c.push_base = :last_passed in .duel.rb, a push is compared with the
|
|
596
|
+
# last commit Duel passed, which a branch marks (duel/passed/main for
|
|
597
|
+
# main), so slowdowns too small for any one push to fail add up until they
|
|
598
|
+
# fail one. This job moves the branch to a push that passed (or makes it).
|
|
599
|
+
# To accept a regression, move the branch yourself (for main, git push
|
|
600
|
+
# origin main:duel/passed/main).
|
|
601
|
+
duel:passed:
|
|
602
|
+
image: alpine:3
|
|
603
|
+
stage: .post
|
|
604
|
+
needs: [duel]
|
|
605
|
+
rules:
|
|
606
|
+
- if: $DUEL_TOKEN && $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
|
|
607
|
+
variables:
|
|
608
|
+
GIT_STRATEGY: none # it runs none of the code
|
|
609
|
+
script:
|
|
610
|
+
- apk add --no-cache --quiet curl
|
|
611
|
+
- |
|
|
612
|
+
[ "$DUEL_PASSED" = true ] || exit 0
|
|
613
|
+
api="$CI_API_V4_URL/projects/$CI_PROJECT_ID"
|
|
614
|
+
token="PRIVATE-TOKEN: $DUEL_TOKEN"
|
|
615
|
+
branch="duel/passed/$CI_COMMIT_BRANCH"
|
|
616
|
+
curl -s -o /dev/null -H "$token" -X DELETE "$api/repository/branches/$(echo "$branch" | sed 's|/|%2F|g')"
|
|
617
|
+
curl -sf -o /dev/null -H "$token" --data-urlencode "branch=$branch" --data-urlencode "ref=$CI_COMMIT_SHA" "$api/repository/branches"
|
|
618
|
+
```
|
|
192
619
|
|
|
193
620
|
## Run each version on its own Ruby on CI
|
|
194
621
|
|
|
@@ -222,20 +649,27 @@ match. `duel init` adds these steps for a project that has a
|
|
|
222
649
|
# .duel.rb). When the baseline's names another Ruby than this code's,
|
|
223
650
|
# that one is installed too, and linked where Duel finds it (as
|
|
224
651
|
# chruby would), before the project's own Ruby, which runs Duel. The
|
|
225
|
-
# baseline is the Compare step's: where HEAD left the
|
|
226
|
-
#
|
|
227
|
-
#
|
|
228
|
-
#
|
|
229
|
-
#
|
|
652
|
+
# baseline is the Compare step's: where HEAD left the branch a pull
|
|
653
|
+
# request targets (or run by hand, the main branch), a merge queue's
|
|
654
|
+
# base, the commit before a push (or the last commit Duel passed, PASSED, if there is
|
|
655
|
+
# one: see the Mark the commit passed job), or on a schedule the last
|
|
656
|
+
# release, the newest tag before HEAD that RELEASE_TAGS matches. (If
|
|
657
|
+
# .duel.rb sets c.main_branch or c.release_tags, set MAIN_BRANCH or
|
|
658
|
+
# RELEASE_TAGS to match.)
|
|
230
659
|
- name: Find the baseline's Ruby
|
|
231
660
|
id: baseline
|
|
232
661
|
env:
|
|
233
662
|
BEFORE: ${{ github.event.before }}
|
|
234
|
-
|
|
663
|
+
QUEUE_BASE: ${{ github.event.merge_group.base_sha }}
|
|
664
|
+
PASSED: origin/duel/passed/${{ github.ref_name }}
|
|
665
|
+
MAIN_BRANCH: origin/${{ github.base_ref || github.event.repository.default_branch }}
|
|
235
666
|
RELEASE_TAGS: "v*"
|
|
236
667
|
run: |
|
|
237
668
|
case "$GITHUB_EVENT_NAME" in
|
|
238
|
-
push)
|
|
669
|
+
push)
|
|
670
|
+
base=$BEFORE
|
|
671
|
+
if git merge-base --is-ancestor "$PASSED" HEAD 2>/dev/null; then base=$PASSED; fi ;;
|
|
672
|
+
merge_group) base=$QUEUE_BASE ;;
|
|
239
673
|
schedule) base=$(git describe --tags --abbrev=0 --match "$RELEASE_TAGS" HEAD^ 2>/dev/null) || base= ;;
|
|
240
674
|
*) base=$(git merge-base HEAD "$MAIN_BRANCH" 2>/dev/null) || base= ;;
|
|
241
675
|
esac
|